add built-in repo to dynamiclly load app into app-store

Signed-off-by: LiHui <andrewli@kubesphere.io>

pkg/controller/openpitrix/helmrepo/helm_repo_controller.go

@@ -294,7 +294,7 @@ func (r *ReconcileHelmRepo) syncRepo(instance *v1alpha1.HelmRepo) error {
 	}
 
 	// 2. merge new index with old index which is stored in crd
-	savedIndex := helmrepoindex.MergeRepoIndex(index, existsSavedIndex)
+	savedIndex := helmrepoindex.MergeRepoIndex(instance, index, existsSavedIndex)
 
 	// 3. save index in crd
 	data, err := savedIndex.Bytes()

pkg/kapis/openpitrix/v1/handler.go

@@ -341,7 +341,7 @@ func (h *openpitrixHandler) DoAppAction(req *restful.Request, resp *restful.Resp
 
 	if err != nil {
 		klog.Errorln(err)
-		handleOpenpitrixError(resp, err)
+		api.HandleError(resp, nil, err)
 		return
 	}
 
@@ -407,7 +407,7 @@ func (h *openpitrixHandler) ModifyApp(req *restful.Request, resp *restful.Respon
 
 	if err != nil {
 		klog.Errorln(err)
-		handleOpenpitrixError(resp, err)
+		api.HandleError(resp, nil, err)
 		return
 	}
 
@@ -589,7 +589,7 @@ func (h *openpitrixHandler) ModifyAppVersion(req *restful.Request, resp *restful
 
 	if err != nil {
 		klog.Errorln(err)
-		handleOpenpitrixError(resp, err)
+		api.HandleError(resp, nil, err)
 		return
 	}
 
@@ -684,7 +684,7 @@ func (h *openpitrixHandler) DoAppVersionAction(req *restful.Request, resp *restf
 
 	if err != nil {
 		klog.Errorln(err)
-		handleOpenpitrixError(resp, err)
+		api.HandleError(resp, nil, err)
 		return
 	}
 

pkg/models/openpitrix/applications.go

@@ -175,11 +175,16 @@ func (c *applicationOperator) ValidatePackage(request *ValidatePackageRequest) (
 
 func (c *applicationOperator) DoAppAction(appId string, request *ActionRequest) error {
 
-	app, err := c.appLister.Get(appId)
+	app, err := c.getHelmApplication(appId)
 	if err != nil {
 		return err
 	}
 
+	// All the app belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else.
+	if repoId, exist := app.Labels[constants.ChartRepoIdLabelKey]; exist && repoId != v1alpha1.AppStoreRepoId {
+		return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplication), app.Name, errors.New("application is immutable"))
+	}
+
 	var filterState string
 	switch request.Action {
 	case ActionSuspend:
@@ -393,12 +398,17 @@ func (c *applicationOperator) ModifyApp(appId string, request *ModifyAppRequest)
 		return invalidS3Config
 	}
 
-	app, err := c.appLister.Get(appId)
+	app, err := c.getHelmApplication(appId)
 	if err != nil {
 		klog.Error(err)
 		return err
 	}
 
+	// All the app belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else.
+	if repoId, exist := app.Labels[constants.ChartRepoIdLabelKey]; exist && repoId != v1alpha1.AppStoreRepoId {
+		return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplication), app.Name, errors.New("application is immutable"))
+	}
+
 	appCopy := app.DeepCopy()
 	// modify category
 	if request.CategoryID != nil {
@@ -602,16 +612,32 @@ func (c *applicationOperator) listApps(conditions *params.Conditions) (ret []*v1
 	repoId := conditions.Match[RepoId]
 	if repoId != "" && repoId != v1alpha1.AppStoreRepoId {
 		// get helm application from helm repo
-		if ret, exists := c.cachedRepos.ListApplicationsByRepoId(repoId); !exists {
+		if ret, exists := c.cachedRepos.ListApplicationsInRepo(repoId); !exists {
 			klog.Warningf("load repo failed, repo id: %s", repoId)
 			return nil, loadRepoInfoFailed
 		} else {
 			return ret, nil
 		}
+	} else if repoId == v1alpha1.AppStoreRepoId {
+		// List apps in the app-store and built-in repo
+		if c.backingStoreClient == nil {
+			return []*v1alpha1.HelmApplication{}, nil
+		}
+
+		ls := map[string]string{}
+		// We just care about the category label when listing apps in built-in repo.
+		if conditions.Match[CategoryId] != "" {
+			ls[constants.CategoryIdLabelKey] = conditions.Match[CategoryId]
+		}
+		appInRepo, _ := c.cachedRepos.ListApplicationsInBuiltinRepo(labels.SelectorFromSet(ls))
+
+		ret, err = c.appLister.List(labels.SelectorFromSet(buildLabelSelector(conditions)))
+		ret = append(ret, appInRepo...)
 	} else {
 		if c.backingStoreClient == nil {
 			return []*v1alpha1.HelmApplication{}, nil
 		}
+
 		ret, err = c.appLister.List(labels.SelectorFromSet(buildLabelSelector(conditions)))
 	}
 

pkg/models/openpitrix/applicationversions.go

@@ -141,7 +141,7 @@ func (c *applicationOperator) DeleteAppVersion(id string) error {
 }
 
 func (c *applicationOperator) DescribeAppVersion(id string) (*AppVersion, error) {
-	version, err := c.versionLister.Get(id)
+	version, err := c.getAppVersion(id)
 	if err != nil {
 		klog.Errorf("get app version [%s] failed, error: %s", id, err)
 		return nil, err
@@ -152,12 +152,17 @@ func (c *applicationOperator) DescribeAppVersion(id string) (*AppVersion, error)
 
 func (c *applicationOperator) ModifyAppVersion(id string, request *ModifyAppVersionRequest) error {
 
-	version, err := c.versionLister.Get(id)
+	version, err := c.getAppVersion(id)
 	if err != nil {
 		klog.Errorf("get app version [%s] failed, error: %s", id, err)
 		return err
 	}
 
+	// All the app versions belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else.
+	if repoId, exists := version.Labels[constants.ChartRepoIdLabelKey]; exists && repoId != v1alpha1.AppStoreRepoId {
+		return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplicationVersion), version.Name, errors.New("version is immutable"))
+	}
+
 	versionCopy := version.DeepCopy()
 	spec := &versionCopy.Spec
 
@@ -355,12 +360,17 @@ func (c *applicationOperator) DoAppVersionAction(versionId string, request *Acti
 	}
 	state := v1alpha1.StateDraft
 
-	version, err := c.versionLister.Get(versionId)
+	version, err := c.getAppVersion(versionId)
 	if err != nil {
 		klog.Errorf("get app version %s failed, error: %s", versionId, err)
 		return err
 	}
 
+	// All the app versions belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else.
+	if repoId, exists := version.Labels[constants.ChartRepoIdLabelKey]; exists && repoId != v1alpha1.AppStoreRepoId {
+		return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplicationVersion), version.Name, errors.New("version is immutable"))
+	}
+
 	switch request.Action {
 	case ActionCancel:
 		if version.Status.State != v1alpha1.StateSubmitted {
@@ -588,3 +598,13 @@ func (c *applicationOperator) getAppVersionsByAppId(appId string) (ret []*v1alph
 
 	return
 }
+
+// get app version from repo and helm application
+func (c *applicationOperator) getAppVersion(id string) (ret *v1alpha1.HelmApplicationVersion, err error) {
+	if ver, exists, _ := c.cachedRepos.GetAppVersion(id); exists {
+		return ver, nil
+	}
+
+	ret, err = c.versionLister.Get(id)
+	return
+}

pkg/models/openpitrix/interface.go

@@ -66,10 +66,9 @@ func NewOpenpitrixOperator(ksInformers ks_informers.InformerFactory, ksClient ve
 				cachedReposData.AddRepo(r)
 			},
 			UpdateFunc: func(oldObj, newObj interface{}) {
-				oldR := oldObj.(*v1alpha1.HelmRepo)
-				cachedReposData.DeleteRepo(oldR)
-				r := newObj.(*v1alpha1.HelmRepo)
-				cachedReposData.AddRepo(r)
+				oldRepo := oldObj.(*v1alpha1.HelmRepo)
+				newRepo := newObj.(*v1alpha1.HelmRepo)
+				cachedReposData.UpdateRepo(oldRepo, newRepo)
 			},
 			DeleteFunc: func(obj interface{}) {
 				r := obj.(*v1alpha1.HelmRepo)

pkg/simple/client/openpitrix/helmrepoindex/repo_index.go

@@ -78,8 +78,8 @@ func loadIndex(data []byte) (*helmrepo.IndexFile, error) {
 
 var empty = struct{}{}
 
-// merge new index with index from crd
-func MergeRepoIndex(index *helmrepo.IndexFile, existsSavedIndex *SavedIndex) *SavedIndex {
+// MergeRepoIndex merge new index with index from crd
+func MergeRepoIndex(repo *v1alpha1.HelmRepo, index *helmrepo.IndexFile, existsSavedIndex *SavedIndex) *SavedIndex {
 	saved := &SavedIndex{}
 	if index == nil {
 		return existsSavedIndex
@@ -102,20 +102,37 @@ func MergeRepoIndex(index *helmrepo.IndexFile, existsSavedIndex *SavedIndex) *Sa
 		// add new applications
 		if application, exists := saved.Applications[name]; !exists {
 			application = &Application{
-				Name:          name,
-				ApplicationId: idutils.GetUuid36(v1alpha1.HelmApplicationIdPrefix),
-				Description:   versions[0].Description,
-				Icon:          versions[0].Icon,
+				Name:        name,
+				Description: versions[0].Description,
+				Icon:        versions[0].Icon,
+				Created:     time.Now(),
+			}
+
+			// The app version will be added to the labels of the helm release.
+			// But the apps in the repos which are created by the user may contain malformed text, so we generate a random name for them.
+			// The apps in the system repo have been audited by the admin, so the name of the charts should not include malformed text.
+			// Then we can add the name string to the labels of the k8s object.
+			if IsBuiltInRepo(repo.Name) {
+				application.ApplicationId = fmt.Sprintf("%s%s-%s", v1alpha1.HelmApplicationIdPrefix, repo.Name, name)
+			} else {
+				application.ApplicationId = idutils.GetUuid36(v1alpha1.HelmApplicationIdPrefix)
 			}
 
 			charts := make([]*ChartVersion, 0, len(versions))
+
 			for ind := range versions {
 				chart := &ChartVersion{
-					ApplicationId:        application.ApplicationId,
-					ApplicationVersionId: idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix),
-					ChartVersion:         *versions[ind],
+					ApplicationId: application.ApplicationId,
+					ChartVersion:  *versions[ind],
 				}
+
+				chart.ApplicationVersionId = generateAppVersionId(repo, versions[ind].Name, versions[ind].Version)
 				charts = append(charts, chart)
+
+				// Use the creation time of the oldest chart as the creation time of the app.
+				if versions[ind].Created.Before(application.Created) {
+					application.Created = versions[ind].Created
+				}
 			}
 
 			application.Charts = charts
@@ -132,10 +149,11 @@ func MergeRepoIndex(index *helmrepo.IndexFile, existsSavedIndex *SavedIndex) *Sa
 				// add new chart version
 				if _, exists := savedChartVersion[ver.Version]; !exists {
 					chart := &ChartVersion{
-						ApplicationId:        application.ApplicationId,
-						ApplicationVersionId: idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix),
-						ChartVersion:         *ver,
+						ApplicationId: application.ApplicationId,
+						ChartVersion:  *ver,
 					}
+
+					chart.ApplicationVersionId = generateAppVersionId(repo, ver.Name, ver.Version)
 					charts = append(charts, chart)
 				}
 				newVersion[ver.Version] = empty
@@ -204,6 +222,26 @@ func (i *SavedIndex) GetApplicationVersion(appId, versionId string) *v1alpha1.He
 	return nil
 }
 
+// The app version name will be added to the labels of the helm release.
+// But the apps in the repos which are created by the user may contain malformed text, so we generate a random name for them.
+// The apps in the system repo have been audited by the admin, so the name of the charts should not include malformed text.
+// Then we can add the name string to the labels of the k8s object.
+func generateAppVersionId(repo *v1alpha1.HelmRepo, chartName, version string) string {
+	if IsBuiltInRepo(repo.Name) {
+		return fmt.Sprintf("%s%s-%s-%s", v1alpha1.HelmApplicationIdPrefix, repo.Name, chartName, version)
+	} else {
+		return idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix)
+	}
+
+}
+
+// IsBuiltInRepo checks whether a repo is a built-in repo.
+// All the built-in repos are located in the workspace system-workspace and the name starts with 'built-in'
+// to differentiate from the repos created by the user
+func IsBuiltInRepo(repoName string) bool {
+	return strings.HasPrefix(repoName, v1alpha1.BuiltinRepoPrefix)
+}
+
 type SavedIndex struct {
 	APIVersion   string                  `json:"apiVersion"`
 	Generated    time.Time               `json:"generated"`
@@ -290,7 +328,7 @@ type Application struct {
 	// application status
 	Status string `json:"status"`
 	// The URL to an icon file.
-	Icon string `json:"icon,omitempty"`
-
-	Charts []*ChartVersion `json:"charts"`
+	Icon    string          `json:"icon,omitempty"`
+	Created time.Time       `json:"created"`
+	Charts  []*ChartVersion `json:"charts"`
 }

pkg/utils/reposcache/repo_cahes.go

@@ -19,7 +19,6 @@ package reposcache
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -28,7 +27,9 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/Masterminds/semver/v3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/klog"
 
 	"kubesphere.io/api/application/v1alpha1"
@@ -52,13 +53,15 @@ func NewReposCache() ReposCache {
 type ReposCache interface {
 	AddRepo(repo *v1alpha1.HelmRepo) error
 	DeleteRepo(repo *v1alpha1.HelmRepo) error
+	UpdateRepo(old, new *v1alpha1.HelmRepo) error
 
 	GetApplication(string) (*v1alpha1.HelmApplication, bool)
 	GetAppVersion(string) (*v1alpha1.HelmApplicationVersion, bool, error)
 	GetAppVersionWithData(string) (*v1alpha1.HelmApplicationVersion, bool, error)
 
 	ListAppVersionsByAppId(appId string) (ret []*v1alpha1.HelmApplicationVersion, exists bool)
-	ListApplicationsByRepoId(repoId string) (ret []*v1alpha1.HelmApplication, exists bool)
+	ListApplicationsInRepo(repoId string) (ret []*v1alpha1.HelmApplication, exists bool)
+	ListApplicationsInBuiltinRepo(selector labels.Selector) (ret []*v1alpha1.HelmApplication, exists bool)
 }
 
 type workspace string
@@ -84,8 +87,7 @@ func (c *cachedRepos) deleteRepo(repo *v1alpha1.HelmRepo) {
 	}
 
 	klog.V(4).Infof("delete repo %s from cache", repo.Name)
-	c.Lock()
-	defer c.Unlock()
+
 	repoId := repo.GetHelmRepoId()
 	ws := workspace(repo.GetWorkspace())
 	if _, exists := c.chartsInRepo[ws]; exists {
@@ -118,6 +120,9 @@ func loadBuiltinChartData(name, version string) ([]byte, error) {
 }
 
 func (c *cachedRepos) DeleteRepo(repo *v1alpha1.HelmRepo) error {
+	c.Lock()
+	defer c.Unlock()
+
 	c.deleteRepo(repo)
 	return nil
 }
@@ -131,7 +136,20 @@ func (c *cachedRepos) GetApplication(appId string) (app *v1alpha1.HelmApplicatio
 	return
 }
 
+func (c *cachedRepos) UpdateRepo(old, new *v1alpha1.HelmRepo) error {
+	if old.Status.Data == new.Status.Data {
+		return nil
+	}
+	c.Lock()
+	defer c.Unlock()
+
+	c.deleteRepo(old)
+	return c.addRepo(new, false)
+}
+
 func (c *cachedRepos) AddRepo(repo *v1alpha1.HelmRepo) error {
+	c.Lock()
+	defer c.Unlock()
 	return c.addRepo(repo, false)
 }
 
@@ -146,10 +164,7 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 		return err
 	}
 
-	klog.V(4).Infof("add repo %s to cache", repo.Name)
-
-	c.Lock()
-	defer c.Unlock()
+	klog.V(2).Infof("add repo %s to cache", repo.Name)
 
 	ws := workspace(repo.GetWorkspace())
 	if _, exists := c.chartsInRepo[ws]; !exists {
@@ -158,7 +173,6 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 
 	repoId := repo.GetHelmRepoId()
 	c.repos[repoId] = repo
-	//c.repoCtgCounts[repo.GetHelmRepoId()] = make(map[string]int)
 	if _, exists := c.repoCtgCounts[repoId]; !exists {
 		c.repoCtgCounts[repoId] = map[string]int{}
 	}
@@ -166,21 +180,23 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 
 	chartsCount := 0
 	for key, app := range index.Applications {
-		if builtin {
-			appName = v1alpha1.HelmApplicationIdPrefix + app.Name
-		} else {
-			appName = app.ApplicationId
+		appName = app.ApplicationId
+
+		appLabels := make(map[string]string)
+		if helmrepoindex.IsBuiltInRepo(repo.Name) {
+			appLabels[constants.WorkspaceLabelKey] = "system-workspace"
 		}
 
+		appLabels[constants.ChartRepoIdLabelKey] = repoId
+
 		HelmApp := v1alpha1.HelmApplication{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: appName,
 				Annotations: map[string]string{
 					constants.CreatorAnnotationKey: repo.GetCreator(),
 				},
-				Labels: map[string]string{
-					constants.ChartRepoIdLabelKey: repo.GetHelmRepoId(),
-				},
+				Labels:            appLabels,
+				CreationTimestamp: metav1.Time{Time: app.Created},
 			},
 			Spec: v1alpha1.HelmApplicationSpec{
 				Name:        key,
@@ -195,21 +211,15 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 
 		var ctg, appVerName string
 		var chartData []byte
+		var latestVersionName string
+		var latestSemver *semver.Version
+
+		// build all the versions of this app
 		for _, ver := range app.Charts {
 			chartsCount += 1
-			if ver.Annotations != nil && ver.Annotations["category"] != "" {
-				ctg = ver.Annotations["category"]
-			}
-			if builtin {
-				appVerName = base64.StdEncoding.EncodeToString([]byte(ver.Name + ver.Version))
-				chartData, err = loadBuiltinChartData(ver.Name, ver.Version)
-				if err != nil {
-					return err
-				}
-			} else {
-				appVerName = ver.ApplicationVersionId
-			}
+			ctg = ver.Annotations["category"]
 
+			appVerName = ver.ApplicationVersionId
 			version := &v1alpha1.HelmApplicationVersion{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:        appVerName,
@@ -235,8 +245,27 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 				},
 			}
 			c.versions[ver.ApplicationVersionId] = version
+
+			// Find the latest version.
+			currSemver, err := semver.NewVersion(version.GetSemver())
+			if err == nil {
+				if latestSemver == nil {
+					// the first valid semver
+					latestSemver = currSemver
+					latestVersionName = version.GetVersionName()
+				} else if latestSemver.LessThan(currSemver) {
+					// find a newer valid semver
+					latestSemver = currSemver
+					latestVersionName = version.GetVersionName()
+				}
+			} else {
+				// If the semver is invalid, just ignore it.
+				klog.V(2).Infof("parse version failed, id: %s, err: %s", version.Name, err)
+			}
 		}
 
+		HelmApp.Status.LatestVersion = latestVersionName
+
 		//modify application category
 		ctgId := ""
 		if ctg != "" {
@@ -262,7 +291,7 @@ func (c *cachedRepos) addRepo(repo *v1alpha1.HelmRepo, builtin bool) error {
 	return nil
 }
 
-func (c *cachedRepos) ListApplicationsByRepoId(repoId string) (ret []*v1alpha1.HelmApplication, exists bool) {
+func (c *cachedRepos) ListApplicationsInRepo(repoId string) (ret []*v1alpha1.HelmApplication, exists bool) {
 	c.RLock()
 	defer c.RUnlock()
 
@@ -279,6 +308,23 @@ func (c *cachedRepos) ListApplicationsByRepoId(repoId string) (ret []*v1alpha1.H
 	return ret, true
 }
 
+func (c *cachedRepos) ListApplicationsInBuiltinRepo(selector labels.Selector) (ret []*v1alpha1.HelmApplication, exists bool) {
+	c.RLock()
+	defer c.RUnlock()
+
+	ret = make([]*v1alpha1.HelmApplication, 0, 20)
+	for _, app := range c.apps {
+		if strings.HasPrefix(app.GetHelmRepoId(), v1alpha1.BuiltinRepoPrefix) {
+			if selector != nil && !selector.Empty() &&
+				(app.Labels == nil || !selector.Matches(labels.Set(app.Labels))) { // If the selector is not empty, we must check whether the labels of the app match the selector.
+				continue
+			}
+			ret = append(ret, app)
+		}
+	}
+	return ret, true
+}
+
 func (c *cachedRepos) ListAppVersionsByAppId(appId string) (ret []*v1alpha1.HelmApplicationVersion, exists bool) {
 	c.RLock()
 	defer c.RUnlock()

staging/src/kubesphere.io/api/application/v1alpha1/constants.go

@@ -49,6 +49,7 @@ const (
 	HelmApplicationAppStoreSuffix  = "-store"
 	HelmApplicationIdPrefix        = "app-"
 	HelmRepoIdPrefix               = "repo-"
+	BuiltinRepoPrefix              = "builtin-"
 	HelmApplicationVersionIdPrefix = "appv-"
 	HelmCategoryIdPrefix           = "ctg-"
 	HelmAttachmentPrefix           = "att-"