feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

pkg/simple/client/application/helper.go

@@ -0,0 +1,594 @@
+package application
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"crypto/md5"
+	"crypto/tls"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"kubesphere.io/utils/helm"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
+
+	pkgconstants "kubesphere.io/kubesphere/pkg/constants"
+
+	k8serr "k8s.io/apimachinery/pkg/api/errors"
+
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/kube"
+	helmrelease "helm.sh/helm/v3/pkg/release"
+	"helm.sh/helm/v3/pkg/storage"
+	"helm.sh/helm/v3/pkg/storage/driver"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/cli-runtime/pkg/resource"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/golang/example/stringutil"
+	"github.com/speps/go-hashids"
+
+	"kubesphere.io/kubesphere/pkg/utils/idutils"
+
+	"k8s.io/klog/v2"
+
+	"helm.sh/helm/v3/pkg/chart"
+
+	"helm.sh/helm/v3/pkg/getter"
+	helmrepo "helm.sh/helm/v3/pkg/repo"
+
+	"kubesphere.io/utils/s3"
+
+	"github.com/blang/semver/v4"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	appv2 "kubesphere.io/api/application/v2"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/scheme"
+)
+
+type AppRequest struct {
+	RepoName     string                       `json:"repoName,omitempty"`
+	AppName      string                       `json:"appName,omitempty"`
+	OriginalName string                       `json:"originalName,omitempty"`
+	AliasName    string                       `json:"aliasName,omitempty"`
+	VersionName  string                       `json:"versionName,omitempty"`
+	AppHome      string                       `json:"appHome,omitempty"`
+	Url          string                       `json:"url,omitempty"`
+	Icon         string                       `json:"icon,omitempty"`
+	Digest       string                       `json:"digest,omitempty"`
+	Workspace    string                       `json:"workspace,omitempty"`
+	Description  string                       `json:"description,omitempty"`
+	CategoryName string                       `json:"categoryName,omitempty"`
+	AppType      string                       `json:"appType,omitempty"`
+	Package      []byte                       `json:"package,omitempty"`
+	PullUrl      string                       `json:"pullUrl,omitempty"`
+	Credential   appv2.RepoCredential         `json:"credential,omitempty"`
+	Maintainers  []appv2.Maintainer           `json:"maintainers,omitempty"`
+	Abstraction  string                       `json:"abstraction,omitempty"`
+	Attachments  []string                     `json:"attachments,omitempty"`
+	FromRepo     bool                         `json:"fromRepo,omitempty"`
+	Resources    []appv2.GroupVersionResource `json:"resources,omitempty"`
+}
+
+func GetMaintainers(maintainers []*chart.Maintainer) []appv2.Maintainer {
+	result := make([]appv2.Maintainer, len(maintainers))
+	for i, maintainer := range maintainers {
+		result[i] = appv2.Maintainer{
+			Name:  maintainer.Name,
+			Email: maintainer.Email,
+			URL:   maintainer.URL,
+		}
+	}
+	return result
+}
+
+func CreateOrUpdateApp(client runtimeclient.Client, vRequests []AppRequest, cmStore, ossStore s3.Interface, owns ...metav1.OwnerReference) error {
+	ctx := context.Background()
+	if len(vRequests) == 0 {
+		return errors.New("version request is empty")
+	}
+	request := vRequests[0]
+
+	app := appv2.Application{}
+	app.Name = request.AppName
+
+	operationResult, err := controllerutil.CreateOrUpdate(ctx, client, &app, func() error {
+		app.Spec = appv2.ApplicationSpec{
+			Icon:        request.Icon,
+			AppHome:     request.AppHome,
+			AppType:     request.AppType,
+			Abstraction: request.Abstraction,
+			Attachments: request.Attachments,
+		}
+		if len(owns) > 0 {
+			app.OwnerReferences = owns
+		}
+
+		labels := app.GetLabels()
+		if labels == nil {
+			labels = make(map[string]string)
+		}
+		labels[appv2.RepoIDLabelKey] = request.RepoName
+		labels[appv2.AppTypeLabelKey] = request.AppType
+
+		if request.CategoryName != "" {
+			labels[appv2.AppCategoryNameKey] = request.CategoryName
+		} else {
+			labels[appv2.AppCategoryNameKey] = appv2.UncategorizedCategoryID
+		}
+
+		labels[constants.WorkspaceLabelKey] = request.Workspace
+		app.SetLabels(labels)
+
+		ant := app.GetAnnotations()
+		if ant == nil {
+			ant = make(map[string]string)
+		}
+		ant[constants.DisplayNameAnnotationKey] = request.AliasName
+		ant[constants.DescriptionAnnotationKey] = request.Description
+		ant[appv2.AppOriginalNameLabelKey] = request.OriginalName
+
+		if len(request.Maintainers) > 0 {
+			ant[appv2.AppMaintainersKey] = request.Maintainers[0].Name
+		}
+		app.SetAnnotations(ant)
+
+		return nil
+	})
+	if err != nil {
+		klog.Errorf("failed create or update app %s, err:%v", app.Name, err)
+		return err
+	}
+
+	if operationResult == controllerutil.OperationResultCreated {
+		if request.FromRepo {
+			app.Status.State = appv2.ReviewStatusActive
+		} else {
+			app.Status.State = appv2.ReviewStatusDraft
+		}
+	}
+
+	app.Status.UpdateTime = &metav1.Time{Time: time.Now()}
+	patch, _ := json.Marshal(app)
+	if err = client.Status().Patch(ctx, &app, runtimeclient.RawPatch(runtimeclient.Merge.Type(), patch)); err != nil {
+		klog.Errorf("failed to update app status, err:%v", err)
+		return err
+	}
+
+	for _, vRequest := range vRequests {
+		if err = CreateOrUpdateAppVersion(ctx, client, app, vRequest, cmStore, ossStore); err != nil {
+			klog.Errorf("failed to create or update app version, err:%v", err)
+			return err
+		}
+	}
+
+	err = UpdateLatestAppVersion(ctx, client, app)
+	if err != nil {
+		klog.Errorf("failed to update latest app version, err:%v", err)
+		return err
+	}
+	return nil
+}
+
+func CreateOrUpdateAppVersion(ctx context.Context, client runtimeclient.Client, app appv2.Application, vRequest AppRequest, cmStore, ossStore s3.Interface) error {
+
+	//1. create or update app version
+	appVersion := appv2.ApplicationVersion{}
+	appVersion.Name = fmt.Sprintf("%s-%s", app.Name, vRequest.VersionName)
+
+	mutateFn := func() error {
+		if err := controllerutil.SetControllerReference(&app, &appVersion, scheme.Scheme); err != nil {
+			klog.Errorf("%s SetControllerReference failed, err:%v", appVersion.Name, err)
+			return err
+		}
+		appVersion.Spec = appv2.ApplicationVersionSpec{
+			VersionName: vRequest.VersionName,
+			AppHome:     vRequest.AppHome,
+			Icon:        vRequest.Icon,
+			Created:     &metav1.Time{Time: time.Now()},
+			Digest:      vRequest.Digest,
+			AppType:     vRequest.AppType,
+			Maintainer:  vRequest.Maintainers,
+			PullUrl:     vRequest.PullUrl,
+		}
+		appVersion.Finalizers = []string{appv2.StoreCleanFinalizer}
+
+		labels := appVersion.GetLabels()
+		if labels == nil {
+			labels = make(map[string]string)
+		}
+		labels[appv2.RepoIDLabelKey] = vRequest.RepoName
+		labels[appv2.AppIDLabelKey] = vRequest.AppName
+		labels[appv2.AppTypeLabelKey] = vRequest.AppType
+		labels[constants.WorkspaceLabelKey] = vRequest.Workspace
+		appVersion.SetLabels(labels)
+
+		ant := appVersion.GetAnnotations()
+		if ant == nil {
+			ant = make(map[string]string)
+		}
+		ant[constants.DisplayNameAnnotationKey] = vRequest.AliasName
+		ant[constants.DescriptionAnnotationKey] = vRequest.Description
+		if len(vRequest.Maintainers) > 0 {
+			ant[appv2.AppMaintainersKey] = vRequest.Maintainers[0].Name
+		}
+		appVersion.SetAnnotations(ant)
+		return nil
+	}
+	_, err := controllerutil.CreateOrUpdate(ctx, client, &appVersion, mutateFn)
+
+	if err != nil {
+		klog.Errorf("failed create or update app version %s, err:%v", appVersion.Name, err)
+		return err
+	}
+
+	if !vRequest.FromRepo {
+		err = FailOverUpload(cmStore, ossStore, appVersion.Name, bytes.NewReader(vRequest.Package), len(vRequest.Package))
+		if err != nil {
+			klog.Errorf("upload package failed, error: %s", err)
+			return err
+		}
+	}
+
+	//3. update app version status
+	if vRequest.FromRepo {
+		appVersion.Status.State = appv2.ReviewStatusActive
+	} else {
+		appVersion.Status.State = appv2.ReviewStatusDraft
+	}
+	appVersion.Status.Updated = &metav1.Time{Time: time.Now()}
+	patch, _ := json.Marshal(appVersion)
+	if err = client.Status().Patch(ctx, &appVersion, runtimeclient.RawPatch(runtimeclient.Merge.Type(), patch)); err != nil {
+		klog.Errorf("failed to update app version status, err:%v", err)
+		return err
+	}
+
+	return err
+}
+
+func UpdateLatestAppVersion(ctx context.Context, client runtimeclient.Client, app appv2.Application) (err error) {
+	//4. update app latest version
+	err = client.Get(ctx, runtimeclient.ObjectKey{Name: app.Name, Namespace: app.Namespace}, &app)
+	if err != nil {
+		klog.Errorf("failed to get app, err:%v", err)
+		return err
+	}
+
+	appVersionList := appv2.ApplicationVersionList{}
+	lbs := labels.SelectorFromSet(labels.Set{appv2.AppIDLabelKey: app.Name})
+	opt := runtimeclient.ListOptions{LabelSelector: lbs}
+	err = client.List(ctx, &appVersionList, &opt)
+	if err != nil {
+		klog.Errorf("failed to list app version, err:%v", err)
+		return err
+	}
+	if len(appVersionList.Items) == 0 {
+		return nil
+	}
+
+	latestAppVersion := appVersionList.Items[0].Spec.VersionName
+	for _, v := range appVersionList.Items {
+		parsedVersion, err := semver.Make(strings.TrimPrefix(v.Spec.VersionName, "v"))
+		if err != nil {
+			klog.Warningf("failed to parse version: %s, use first version %s", v.Spec.VersionName, latestAppVersion)
+			continue
+		}
+		if parsedVersion.GT(semver.MustParse(strings.TrimPrefix(latestAppVersion, "v"))) {
+			latestAppVersion = v.Spec.VersionName
+		}
+	}
+
+	ant := app.GetAnnotations()
+	ant[appv2.LatestAppVersionKey] = latestAppVersion
+	app.SetAnnotations(ant)
+	err = client.Update(ctx, &app)
+	return err
+}
+
+func HelmPull(u string, cred appv2.RepoCredential) (*bytes.Buffer, error) {
+	parsedURL, err := url.Parse(u)
+	if err != nil {
+		return nil, err
+	}
+	var resp *bytes.Buffer
+
+	skipTLS := true
+	if cred.InsecureSkipTLSVerify != nil && !*cred.InsecureSkipTLSVerify {
+		skipTLS = false
+	}
+
+	indexURL := parsedURL.String()
+	g, _ := getter.NewHTTPGetter()
+	options := []getter.Option{
+		getter.WithTimeout(5 * time.Minute),
+		getter.WithURL(u),
+		getter.WithInsecureSkipVerifyTLS(skipTLS),
+		getter.WithTLSClientConfig(cred.CertFile, cred.KeyFile, cred.CAFile),
+		getter.WithBasicAuth(cred.Username, cred.Password)}
+
+	if skipTLS {
+		options = append(options, getter.WithTransport(
+			&http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
+		))
+	}
+
+	resp, err = g.Get(indexURL, options...)
+	return resp, err
+}
+
+func LoadRepoIndex(u string, cred appv2.RepoCredential) (idx helmrepo.IndexFile, err error) {
+	if !strings.HasSuffix(u, "/") {
+		u = fmt.Sprintf("%s/index.yaml", u)
+	} else {
+		u = fmt.Sprintf("%sindex.yaml", u)
+	}
+
+	resp, err := HelmPull(u, cred)
+	if err != nil {
+		return idx, err
+	}
+	if err = yaml.Unmarshal(resp.Bytes(), &idx); err != nil {
+		return idx, err
+	}
+	idx.SortEntries()
+
+	return idx, nil
+}
+
+func ReadYaml(data []byte) (jsonList []json.RawMessage, err error) {
+	reader := bytes.NewReader(data)
+	bufReader := bufio.NewReader(reader)
+	r := yaml.NewYAMLReader(bufReader)
+	for {
+		d, err := r.Read()
+		if err != nil && err == io.EOF {
+			break
+		}
+		jsonData, err := yaml.ToJSON(d)
+		if err != nil {
+			return nil, err
+		}
+		_, _, err = Decode(jsonData)
+		if err != nil {
+			return nil, err
+		}
+		jsonList = append(jsonList, jsonData)
+	}
+	return jsonList, nil
+}
+
+func Decode(data []byte) (obj runtime.Object, gvk *schema.GroupVersionKind, err error) {
+	decoder := unstructured.UnstructuredJSONScheme
+	obj, gvk, err = decoder.Decode(data, nil, nil)
+	return obj, gvk, err
+}
+
+func UpdateHelmStatus(kubeConfig []byte, release *helmrelease.Release) (deployed bool, err error) {
+	config, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
+	if err != nil {
+		klog.Errorf("failed to get rest config, err:%v", err)
+		return deployed, err
+	}
+	clientSet, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		klog.Errorf("failed to get kubernetes client, err:%v", err)
+		return deployed, err
+	}
+
+	actionConfig := new(action.Configuration)
+	store := storage.Init(driver.NewSecrets(clientSet.CoreV1().Secrets(release.Namespace)))
+	actionConfig.Releases = store
+
+	deployed, err = checkReady(release, clientSet, kubeConfig)
+	if err != nil {
+		klog.Errorf("failed to check helm ready, err:%v", err)
+		return deployed, err
+	}
+	if !deployed {
+		klog.Infof("helm release %s not ready", release.Name)
+		return deployed, nil
+	}
+
+	klog.Infof("helm release %s now ready", release.Name)
+	release.SetStatus("deployed", "Successfully deployed")
+
+	if err = actionConfig.Releases.Update(release); err != nil {
+		klog.Errorf("failed to update release: %v", err)
+		return deployed, err
+	}
+	klog.Infof("update release %s status successfully", release.Name)
+	return true, err
+}
+
+func checkReady(release *helmrelease.Release, clientSet *kubernetes.Clientset, kubeConfig []byte) (allReady bool, err error) {
+
+	checker := kube.NewReadyChecker(clientSet, nil, kube.PausedAsReady(true), kube.CheckJobs(true))
+
+	helmConf, err := helm.InitHelmConf(kubeConfig, release.Namespace)
+	if err != nil {
+		klog.Errorf("failed to init helm conf, err:%v", err)
+		return allReady, err
+	}
+
+	allResources := make([]*resource.Info, 0)
+	for _, i := range release.Hooks {
+		hookResources, err := helmConf.KubeClient.Build(bytes.NewBufferString(i.Manifest), false)
+		if err != nil {
+			klog.Errorf("failed to get helm hookResources, err:%v", err)
+			return allReady, err
+		}
+		allResources = append(allResources, hookResources...)
+	}
+	klog.Infof("%s get helm hookResources %d", release.Name, len(allResources))
+
+	chartResources, err := helmConf.KubeClient.Build(bytes.NewBufferString(release.Manifest), false)
+	if err != nil {
+		klog.Errorf("failed to get helm resources, err:%v", err)
+		return allReady, err
+	}
+	allResources = append(allResources, chartResources...)
+	klog.Infof("%s get helm chartResources %d", release.Name, len(chartResources))
+
+	for idx, j := range allResources {
+		kind := j.Object.GetObjectKind().GroupVersionKind().Kind
+		klog.Infof("[%d/%d] check helm release %s  %s: %s/%s", idx+1, len(allResources),
+			release.Name, kind, j.Namespace, j.Name)
+		ready, err := checker.IsReady(context.Background(), j)
+		if k8serr.IsNotFound(err) {
+			//pre-job-->chart-resource-->post-job
+			//If a certain step times out, the subsequent steps will not be created,
+			//and the status is considered failed, no repair will be made.
+			klog.Warningf("[%d/%d] helm release %s resource %s: %s/%s not found", idx+1, len(allResources), release.Name, kind, j.Namespace, j.Name)
+			return false, nil
+		}
+
+		if err != nil {
+			klog.Errorf("failed to check resource ready, err:%v", err)
+			return allReady, err
+		}
+		if !ready {
+			klog.Infof("[%d/%d] helm release %s resource %s: %s/%s not ready", idx+1, len(allResources), release.Name, kind, j.Namespace, j.Name)
+			return allReady, nil
+		}
+	}
+	return true, nil
+}
+
+func GvkToGvr(gvk *schema.GroupVersionKind, mapper meta.RESTMapper) (schema.GroupVersionResource, error) {
+	mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
+	if meta.IsNoMatchError(err) || err != nil {
+		return schema.GroupVersionResource{}, err
+	}
+	return mapping.Resource, nil
+}
+func GetInfoFromBytes(bytes json.RawMessage, mapper meta.RESTMapper) (gvr schema.GroupVersionResource, utd *unstructured.Unstructured, err error) {
+	obj, gvk, err := Decode(bytes)
+	if err != nil {
+		return gvr, utd, err
+	}
+	gvr, err = GvkToGvr(gvk, mapper)
+	if err != nil {
+		return gvr, utd, err
+	}
+	utd, err = ConvertToUnstructured(obj)
+	return gvr, utd, err
+}
+func ConvertToUnstructured(obj any) (*unstructured.Unstructured, error) {
+	objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
+	return &unstructured.Unstructured{Object: objMap}, err
+}
+
+func ComplianceCheck(values, tempLate []byte, mapper meta.RESTMapper, ns string) (result []json.RawMessage, err error) {
+	yamlList, err := ReadYaml(values)
+	if err != nil {
+		return nil, err
+	}
+	yamlTempList, err := ReadYaml(tempLate)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(yamlTempList) != len(yamlList) {
+		return nil, errors.New("yamlList and yamlTempList length not equal")
+	}
+	for idx := range yamlTempList {
+		_, utd, err := GetInfoFromBytes(yamlList[idx], mapper)
+		if err != nil {
+			return nil, err
+		}
+		_, utdTemp, err := GetInfoFromBytes(yamlTempList[idx], mapper)
+		if err != nil {
+			return nil, err
+		}
+		if utdTemp.GetKind() != utd.GetKind() || utdTemp.GetAPIVersion() != utd.GetAPIVersion() {
+			return nil, errors.New("yamlList and yamlTempList not equal")
+		}
+		if utd.GetNamespace() != ns {
+			return nil, errors.New("subresource must have same namespace with app release")
+		}
+	}
+	return yamlList, nil
+}
+
+func GetUuid36(prefix string) string {
+	id := idutils.GetIntId()
+	hd := hashids.NewData()
+	hd.Alphabet = idutils.Alphabet36
+	h, err := hashids.NewWithData(hd)
+	if err != nil {
+		panic(err)
+	}
+	i, err := h.Encode([]int{int(id)})
+	if err != nil {
+		panic(err)
+	}
+	//hashids.minAlphabetLength = 16
+	add := stringutil.Reverse(i)[:5]
+
+	return prefix + add
+}
+
+func GenerateShortNameMD5Hash(input string) string {
+	input = strings.ToLower(input)
+	errs := validation.IsDNS1123Subdomain(input)
+	if len(input) > 14 || len(errs) != 0 {
+		hash := md5.New()
+		hash.Write([]byte(input))
+		hashInBytes := hash.Sum(nil)
+		hashString := hex.EncodeToString(hashInBytes)
+		return hashString[:10]
+	}
+	return input
+}
+
+func FormatVersion(input string) string {
+	re := regexp.MustCompile(`[^a-z0-9-.]`)
+	errs := validation.IsDNS1123Subdomain(input)
+	if len(errs) != 0 {
+		klog.Warningf("Version %s does not meet the Kubernetes naming standard, replacing invalid characters with '-'", input)
+		input = re.ReplaceAllStringFunc(input, func(s string) string {
+			return "-"
+		})
+	}
+	return input
+}
+
+func GetHelmKubeConfig(ctx context.Context, cluster *clusterv1alpha1.Cluster, runClient client.Client) (config []byte, err error) {
+
+	if cluster.Spec.Connection.Type == clusterv1alpha1.ConnectionTypeProxy {
+		klog.Infof("cluster %s is proxy cluster", cluster.Name)
+		secret := &corev1.Secret{}
+		key := types.NamespacedName{Namespace: pkgconstants.KubeSphereNamespace, Name: "kubeconfig-admin"}
+		err = runClient.Get(ctx, key, secret)
+		if err != nil {
+			klog.Errorf("failed to get kubeconfig-admin secret: %v", err)
+			return nil, err
+		}
+		config = secret.Data["config"]
+		return config, err
+	}
+	return cluster.Spec.Connection.KubeConfig, nil
+}

pkg/simple/client/application/store.go

@@ -0,0 +1,171 @@
+package application
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"k8s.io/klog/v2"
+
+	"io"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	s3lib "github.com/aws/aws-sdk-go/service/s3"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	appv2 "kubesphere.io/api/application/v2"
+	"kubesphere.io/utils/s3"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type CmStore struct {
+	Client runtimeclient.Client
+}
+
+var _ s3.Interface = CmStore{}
+
+func InitStore(s3opts *s3.Options, Client client.Client) (cmStore, ossStore s3.Interface, err error) {
+	if s3opts != nil && len(s3opts.Endpoint) != 0 {
+		klog.Infof("init s3 client with endpoint: %s", s3opts.Endpoint)
+		var err error
+		ossStore, err = s3.NewS3Client(s3opts)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to create s3 client: %v", err)
+		}
+	}
+	klog.Infof("init configmap store")
+	cmStore = CmStore{
+		Client: Client,
+	}
+	return cmStore, ossStore, nil
+}
+
+func (c CmStore) Read(key string) ([]byte, error) {
+	cm := corev1.ConfigMap{}
+	nameKey := runtimeclient.ObjectKey{Name: key, Namespace: appv2.ApplicationNamespace}
+	err := c.Client.Get(context.TODO(), nameKey, &cm)
+	if err != nil {
+		return nil, err
+	}
+	return cm.BinaryData[appv2.BinaryKey], nil
+}
+
+func (c CmStore) Upload(key, fileName string, body io.Reader, size int) error {
+	data, _ := io.ReadAll(body)
+	obj := corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      key,
+			Namespace: appv2.ApplicationNamespace,
+		},
+		BinaryData: map[string][]byte{appv2.BinaryKey: data},
+	}
+	err := c.Client.Create(context.TODO(), &obj)
+	//ignore already exists error
+	if apierrors.IsAlreadyExists(err) {
+		klog.Warningf("save to store ignore already exists %s", key)
+		return nil
+	}
+
+	return err
+}
+
+func (c CmStore) Delete(ids []string) error {
+	for _, id := range ids {
+		obj := corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      id,
+				Namespace: appv2.ApplicationNamespace,
+			},
+		}
+		err := c.Client.Delete(context.TODO(), &obj)
+		if err != nil && apierrors.IsNotFound(err) {
+			continue
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func FailOverGet(cm, oss s3.Interface, key string, cli client.Client, isApp bool) (data []byte, err error) {
+
+	if isApp {
+		fromRepo, pullUrl, repoName, err := FromRepo(cli, key)
+		if err != nil {
+			klog.Errorf("failed to get app version, err: %v", err)
+			return nil, err
+		}
+		if fromRepo {
+			return DownLoadChart(cli, pullUrl, repoName)
+		}
+	}
+
+	if oss == nil {
+		klog.Infof("read from configMap %s", key)
+		return cm.Read(key)
+	}
+	klog.Infof("read from oss %s", key)
+	data, err = oss.Read(key)
+	if err != nil {
+		var aerr awserr.Error
+		if errors.As(err, &aerr) && aerr.Code() == s3lib.ErrCodeNoSuchKey {
+			klog.Infof("FailOver read from configMap %s", key)
+			return cm.Read(key)
+		}
+	}
+	return data, err
+}
+
+func FromRepo(cli runtimeclient.Client, key string) (fromRepo bool, url, repoName string, err error) {
+
+	appVersion := appv2.ApplicationVersion{}
+	err = cli.Get(context.TODO(), client.ObjectKey{Name: key}, &appVersion)
+	if err != nil {
+		klog.Errorf("failed to get app version, err: %v", err)
+		return fromRepo, url, repoName, err
+	}
+	if appVersion.Spec.PullUrl != "" {
+		klog.Infof("load chart from pull url: %s", appVersion.Spec.PullUrl)
+	} else {
+		klog.Infof("load chart from local store")
+	}
+	fromRepo = appVersion.GetLabels()[appv2.RepoIDLabelKey] != appv2.UploadRepoKey
+	repoName = appVersion.GetLabels()[appv2.RepoIDLabelKey]
+	return fromRepo, appVersion.Spec.PullUrl, repoName, nil
+}
+
+func DownLoadChart(cli runtimeclient.Client, pullUrl, repoName string) (data []byte, err error) {
+
+	repo := appv2.Repo{}
+	err = cli.Get(context.TODO(), client.ObjectKey{Name: repoName}, &repo)
+	if err != nil {
+		klog.Errorf("failed to get app repo, err: %v", err)
+		return data, err
+	}
+	buf, err := HelmPull(pullUrl, repo.Spec.Credential)
+	if err != nil {
+		klog.Errorf("load chart failed, error: %s", err)
+		return data, err
+	}
+	return buf.Bytes(), nil
+}
+
+func FailOverUpload(cm, oss s3.Interface, key string, body io.Reader, size int) error {
+	if oss == nil {
+		klog.Infof("upload to cm %s", key)
+		return cm.Upload(key, key, body, size)
+	}
+	klog.Infof("upload to oss %s", key)
+	return oss.Upload(key, key, body, size)
+}
+func FailOverDelete(cm, oss s3.Interface, key []string) error {
+	if oss == nil {
+		klog.Infof("delete from cm %v", key)
+		return cm.Delete(key)
+	}
+	klog.Infof("delete from oss %v", key)
+	return oss.Delete(key)
+}

pkg/simple/client/application/yaml.go

@@ -0,0 +1,97 @@
+package application
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	helmrelease "helm.sh/helm/v3/pkg/release"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/klog/v2"
+	"kubesphere.io/utils/helm"
+)
+
+var _ helm.Executor = &YamlInstaller{}
+
+type YamlInstaller struct {
+	Mapper      meta.RESTMapper
+	DynamicCli  *dynamic.DynamicClient
+	GvrListInfo []InsInfo
+	Namespace   string
+}
+type InsInfo struct {
+	schema.GroupVersionResource
+	Name      string
+	Namespace string
+}
+
+func (t YamlInstaller) Install(ctx context.Context, release, chart string, values []byte, options ...helm.HelmOption) (string, error) {
+	return "", nil
+}
+
+func (t YamlInstaller) Upgrade(ctx context.Context, release, chart string, values []byte, options ...helm.HelmOption) (string, error) {
+	yamlList, err := ReadYaml(values)
+	if err != nil {
+		return "", err
+	}
+	klog.Infof("attempting to apply %d yaml files", len(yamlList))
+
+	err = t.ForApply(yamlList)
+
+	return "", err
+}
+
+func (t YamlInstaller) Uninstall(ctx context.Context, release string, options ...helm.HelmOption) (string, error) {
+	for _, i := range t.GvrListInfo {
+		err := t.DynamicCli.Resource(i.GroupVersionResource).Namespace(i.Namespace).
+			Delete(ctx, i.Name, metav1.DeleteOptions{})
+		if apierrors.IsNotFound(err) {
+			continue
+		}
+		if err != nil {
+			return "", err
+		}
+	}
+	return "", nil
+}
+
+func (t YamlInstaller) ForceDelete(ctx context.Context, release string, options ...helm.HelmOption) error {
+	_, err := t.Uninstall(ctx, release, options...)
+	return err
+}
+
+func (t YamlInstaller) Get(ctx context.Context, releaseName string, options ...helm.HelmOption) (*helmrelease.Release, error) {
+	rv := &helmrelease.Release{}
+	rv.Info = &helmrelease.Info{Status: helmrelease.StatusDeployed}
+	return rv, nil
+}
+
+func (t YamlInstaller) WaitingForResourcesReady(ctx context.Context, release string, timeout time.Duration, options ...helm.HelmOption) (bool, error) {
+	return true, nil
+}
+
+func (t YamlInstaller) ForApply(tasks []json.RawMessage) (err error) {
+
+	for idx, js := range tasks {
+
+		gvr, utd, err := GetInfoFromBytes(js, t.Mapper)
+		if err != nil {
+			return err
+		}
+		opt := metav1.PatchOptions{FieldManager: "v1.FieldManager"}
+		_, err = t.DynamicCli.Resource(gvr).
+			Namespace(utd.GetNamespace()).
+			Patch(context.TODO(), utd.GetName(), types.ApplyPatchType, js, opt)
+
+		if err != nil {
+			return err
+		}
+		klog.Infof("[%d/%d] %s/%s applied", idx+1, len(tasks), gvr.Resource, utd.GetName())
+	}
+	return nil
+}