kubesphere/pkg/simple/client/application/helper.go

package application

import (
	"bufio"
	"bytes"
	"context"
	"crypto/md5"
	"crypto/sha1"
	"crypto/tls"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"

	"kubesphere.io/utils/helm"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/types"
	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"

	pkgconstants "kubesphere.io/kubesphere/pkg/constants"

	k8serr "k8s.io/apimachinery/pkg/api/errors"

	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/kube"
	"helm.sh/helm/v3/pkg/registry"
	helmrelease "helm.sh/helm/v3/pkg/release"
	"helm.sh/helm/v3/pkg/storage"
	"helm.sh/helm/v3/pkg/storage/driver"
	"k8s.io/apimachinery/pkg/util/validation"
	"k8s.io/cli-runtime/pkg/resource"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"sigs.k8s.io/controller-runtime/pkg/client"

	"github.com/golang/example/stringutil"
	"github.com/speps/go-hashids"

	"kubesphere.io/kubesphere/pkg/utils/idutils"

	"k8s.io/klog/v2"

	"helm.sh/helm/v3/pkg/chart"

	"helm.sh/helm/v3/pkg/getter"
	helmrepo "helm.sh/helm/v3/pkg/repo"

	"kubesphere.io/utils/s3"

	"github.com/blang/semver/v4"
	"k8s.io/apimachinery/pkg/labels"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/yaml"
	appv2 "kubesphere.io/api/application/v2"
	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

	"kubesphere.io/kubesphere/pkg/constants"
	"kubesphere.io/kubesphere/pkg/scheme"
)

type AppRequest struct {
	RepoName     string                       `json:"repoName,omitempty"`
	AppName      string                       `json:"appName,omitempty"`
	OriginalName string                       `json:"originalName,omitempty"`
	AliasName    string                       `json:"aliasName,omitempty"`
	VersionName  string                       `json:"versionName,omitempty"`
	AppHome      string                       `json:"appHome,omitempty"`
	Url          string                       `json:"url,omitempty"`
	Icon         string                       `json:"icon,omitempty"`
	Digest       string                       `json:"digest,omitempty"`
	Workspace    string                       `json:"workspace,omitempty"`
	Description  string                       `json:"description,omitempty"`
	CategoryName string                       `json:"categoryName,omitempty"`
	AppType      string                       `json:"appType,omitempty"`
	Package      []byte                       `json:"package,omitempty"`
	PullUrl      string                       `json:"pullUrl,omitempty"`
	Credential   appv2.RepoCredential         `json:"credential,omitempty"`
	Maintainers  []appv2.Maintainer           `json:"maintainers,omitempty"`
	Abstraction  string                       `json:"abstraction,omitempty"`
	Attachments  []string                     `json:"attachments,omitempty"`
	FromRepo     bool                         `json:"fromRepo,omitempty"`
	Resources    []appv2.GroupVersionResource `json:"resources,omitempty"`
}

func GetMaintainers(maintainers []*chart.Maintainer) []appv2.Maintainer {
	result := make([]appv2.Maintainer, len(maintainers))
	for i, maintainer := range maintainers {
		result[i] = appv2.Maintainer{
			Name:  maintainer.Name,
			Email: maintainer.Email,
			URL:   maintainer.URL,
		}
	}
	return result
}

func CreateOrUpdateApp(client runtimeclient.Client, vRequests []AppRequest, cmStore, ossStore s3.Interface, owns ...metav1.OwnerReference) error {
	ctx := context.Background()
	if len(vRequests) == 0 {
		return errors.New("version request is empty")
	}
	request := vRequests[0]

	app := appv2.Application{}
	app.Name = request.AppName

	operationResult, err := controllerutil.CreateOrUpdate(ctx, client, &app, func() error {
		app.Spec = appv2.ApplicationSpec{
			Icon:        request.Icon,
			AppHome:     request.AppHome,
			AppType:     request.AppType,
			Abstraction: request.Abstraction,
			Attachments: request.Attachments,
		}
		if len(owns) > 0 {
			app.OwnerReferences = owns
		}

		labels := app.GetLabels()
		if labels == nil {
			labels = make(map[string]string)
		}
		labels[appv2.RepoIDLabelKey] = request.RepoName
		labels[appv2.AppTypeLabelKey] = request.AppType

		if request.CategoryName != "" {
			labels[appv2.AppCategoryNameKey] = request.CategoryName
		} else {
			labels[appv2.AppCategoryNameKey] = appv2.UncategorizedCategoryID
		}

		labels[constants.WorkspaceLabelKey] = request.Workspace
		app.SetLabels(labels)

		ant := app.GetAnnotations()
		if ant == nil {
			ant = make(map[string]string)
		}
		ant[constants.DisplayNameAnnotationKey] = request.AliasName
		ant[constants.DescriptionAnnotationKey] = request.Description
		ant[appv2.AppOriginalNameLabelKey] = request.OriginalName

		if len(request.Maintainers) > 0 {
			ant[appv2.AppMaintainersKey] = request.Maintainers[0].Name
		}
		app.SetAnnotations(ant)

		return nil
	})
	if err != nil {
		klog.Errorf("failed create or update app %s, err:%v", app.Name, err)
		return err
	}

	if operationResult == controllerutil.OperationResultCreated {
		if request.FromRepo {
			app.Status.State = appv2.ReviewStatusActive
		} else {
			app.Status.State = appv2.ReviewStatusDraft
		}
	}

	app.Status.UpdateTime = &metav1.Time{Time: time.Now()}
	patch, _ := json.Marshal(app)
	if err = client.Status().Patch(ctx, &app, runtimeclient.RawPatch(runtimeclient.Merge.Type(), patch)); err != nil {
		klog.Errorf("failed to update app status, err:%v", err)
		return err
	}

	for _, vRequest := range vRequests {
		if err = CreateOrUpdateAppVersion(ctx, client, app, vRequest, cmStore, ossStore); err != nil {
			klog.Errorf("failed to create or update app version, err:%v", err)
			return err
		}
	}

	err = UpdateLatestAppVersion(ctx, client, app)
	if err != nil {
		klog.Errorf("failed to update latest app version, err:%v", err)
		return err
	}
	return nil
}

func CreateOrUpdateAppVersion(ctx context.Context, client runtimeclient.Client, app appv2.Application, vRequest AppRequest, cmStore, ossStore s3.Interface) error {

	//1. create or update app version
	appVersion := appv2.ApplicationVersion{}
	legalVersion := FormatVersion(vRequest.VersionName)
	appVersion.Name = fmt.Sprintf("%s-%s", app.Name, legalVersion)

	mutateFn := func() error {
		if err := controllerutil.SetControllerReference(&app, &appVersion, scheme.Scheme); err != nil {
			klog.Errorf("%s SetControllerReference failed, err:%v", appVersion.Name, err)
			return err
		}
		appVersion.Spec = appv2.ApplicationVersionSpec{
			VersionName: vRequest.VersionName,
			AppHome:     vRequest.AppHome,
			Icon:        vRequest.Icon,
			Created:     &metav1.Time{Time: time.Now()},
			Digest:      vRequest.Digest,
			AppType:     vRequest.AppType,
			Maintainer:  vRequest.Maintainers,
			PullUrl:     vRequest.PullUrl,
		}
		appVersion.Finalizers = []string{appv2.StoreCleanFinalizer}

		labels := appVersion.GetLabels()
		if labels == nil {
			labels = make(map[string]string)
		}
		labels[appv2.RepoIDLabelKey] = vRequest.RepoName
		labels[appv2.AppIDLabelKey] = vRequest.AppName
		labels[appv2.AppTypeLabelKey] = vRequest.AppType
		labels[constants.WorkspaceLabelKey] = vRequest.Workspace
		appVersion.SetLabels(labels)

		ant := appVersion.GetAnnotations()
		if ant == nil {
			ant = make(map[string]string)
		}
		ant[constants.DisplayNameAnnotationKey] = vRequest.AliasName
		ant[constants.DescriptionAnnotationKey] = vRequest.Description
		if len(vRequest.Maintainers) > 0 {
			ant[appv2.AppMaintainersKey] = vRequest.Maintainers[0].Name
		}
		appVersion.SetAnnotations(ant)
		return nil
	}
	_, err := controllerutil.CreateOrUpdate(ctx, client, &appVersion, mutateFn)

	if err != nil {
		klog.Errorf("failed create or update app version %s, err:%v", appVersion.Name, err)
		return err
	}

	if !vRequest.FromRepo {
		err = FailOverUpload(cmStore, ossStore, appVersion.Name, bytes.NewReader(vRequest.Package), len(vRequest.Package))
		if err != nil {
			klog.Errorf("upload package failed, error: %s", err)
			return err
		}
	}

	//3. update app version status
	if vRequest.FromRepo {
		appVersion.Status.State = appv2.ReviewStatusActive
	} else {
		appVersion.Status.State = appv2.ReviewStatusDraft
	}
	appVersion.Status.Updated = &metav1.Time{Time: time.Now()}
	patch, _ := json.Marshal(appVersion)
	if err = client.Status().Patch(ctx, &appVersion, runtimeclient.RawPatch(runtimeclient.Merge.Type(), patch)); err != nil {
		klog.Errorf("failed to update app version status, err:%v", err)
		return err
	}

	return err
}

func UpdateLatestAppVersion(ctx context.Context, client runtimeclient.Client, app appv2.Application) (err error) {
	//4. update app latest version
	err = client.Get(ctx, runtimeclient.ObjectKey{Name: app.Name, Namespace: app.Namespace}, &app)
	if err != nil {
		klog.Errorf("failed to get app, err:%v", err)
		return err
	}

	appVersionList := appv2.ApplicationVersionList{}
	lbs := labels.SelectorFromSet(labels.Set{appv2.AppIDLabelKey: app.Name})
	opt := runtimeclient.ListOptions{LabelSelector: lbs}
	err = client.List(ctx, &appVersionList, &opt)
	if err != nil {
		klog.Errorf("failed to list app version, err:%v", err)
		return err
	}
	if len(appVersionList.Items) == 0 {
		return nil
	}

	latestAppVersion := appVersionList.Items[0].Spec.VersionName
	for _, v := range appVersionList.Items {
		parsedVersion, err := semver.Make(strings.TrimPrefix(v.Spec.VersionName, "v"))
		if err != nil {
			klog.Warningf("failed to parse version: %s, use first version %s", v.Spec.VersionName, latestAppVersion)
			continue
		}
		if parsedVersion.GT(semver.MustParse(strings.TrimPrefix(latestAppVersion, "v"))) {
			latestAppVersion = v.Spec.VersionName
		}
	}

	ant := app.GetAnnotations()
	ant[appv2.LatestAppVersionKey] = latestAppVersion
	app.SetAnnotations(ant)
	err = client.Update(ctx, &app)
	return err
}

func HelmPull(u string, cred appv2.RepoCredential) (*bytes.Buffer, error) {
	parsedURL, err := url.Parse(u)
	if err != nil {
		return nil, err
	}
	var resp *bytes.Buffer

	skipTLS := true
	if cred.InsecureSkipTLSVerify != nil && !*cred.InsecureSkipTLSVerify {
		skipTLS = false
	}

	indexURL := parsedURL.String()
	g, _ := getter.NewHTTPGetter()
	options := []getter.Option{
		getter.WithTimeout(5 * time.Minute),
		getter.WithURL(u),
		getter.WithInsecureSkipVerifyTLS(skipTLS),
		getter.WithTLSClientConfig(cred.CertFile, cred.KeyFile, cred.CAFile),
		getter.WithBasicAuth(cred.Username, cred.Password)}

	if skipTLS {
		options = append(options, getter.WithTransport(
			&http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}},
		))
	}

	resp, err = g.Get(indexURL, options...)
	return resp, err
}

func LoadRepoIndex(u string, cred appv2.RepoCredential) (idx helmrepo.IndexFile, err error) {
	if registry.IsOCI(u) {
		return LoadRepoIndexFromOci(u, cred)
	}

	if !strings.HasSuffix(u, "/") {
		u = fmt.Sprintf("%s/index.yaml", u)
	} else {
		u = fmt.Sprintf("%sindex.yaml", u)
	}

	resp, err := HelmPull(u, cred)
	if err != nil {
		return idx, err
	}
	if err = yaml.Unmarshal(resp.Bytes(), &idx); err != nil {
		return idx, err
	}
	idx.SortEntries()

	return idx, nil
}

func ReadYaml(data []byte) (jsonList []json.RawMessage, err error) {
	reader := bytes.NewReader(data)
	bufReader := bufio.NewReader(reader)
	r := yaml.NewYAMLReader(bufReader)
	for {
		d, err := r.Read()
		if err != nil && err == io.EOF {
			break
		}
		jsonData, err := yaml.ToJSON(d)
		if err != nil {
			return nil, err
		}
		_, _, err = Decode(jsonData)
		if err != nil {
			return nil, err
		}
		jsonList = append(jsonList, jsonData)
	}
	return jsonList, nil
}

func Decode(data []byte) (obj runtime.Object, gvk *schema.GroupVersionKind, err error) {
	decoder := unstructured.UnstructuredJSONScheme
	obj, gvk, err = decoder.Decode(data, nil, nil)
	return obj, gvk, err
}

func UpdateHelmStatus(kubeConfig []byte, release *helmrelease.Release) (deployed bool, err error) {
	config, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig)
	if err != nil {
		klog.Errorf("failed to get rest config, err:%v", err)
		return deployed, err
	}
	clientSet, err := kubernetes.NewForConfig(config)
	if err != nil {
		klog.Errorf("failed to get kubernetes client, err:%v", err)
		return deployed, err
	}

	actionConfig := new(action.Configuration)
	store := storage.Init(driver.NewSecrets(clientSet.CoreV1().Secrets(release.Namespace)))
	actionConfig.Releases = store

	deployed, err = checkReady(release, clientSet, kubeConfig)
	if err != nil {
		klog.Errorf("failed to check helm ready, err:%v", err)
		return deployed, err
	}
	if !deployed {
		klog.Infof("helm release %s not ready", release.Name)
		return deployed, nil
	}

	klog.Infof("helm release %s now ready", release.Name)
	release.SetStatus("deployed", "Successfully deployed")

	if err = actionConfig.Releases.Update(release); err != nil {
		klog.Errorf("failed to update release: %v", err)
		return deployed, err
	}
	klog.Infof("update release %s status successfully", release.Name)
	return true, err
}

func checkReady(release *helmrelease.Release, clientSet *kubernetes.Clientset, kubeConfig []byte) (allReady bool, err error) {

	checker := kube.NewReadyChecker(clientSet, nil, kube.PausedAsReady(true), kube.CheckJobs(true))

	helmConf, err := helm.InitHelmConf(kubeConfig, release.Namespace)
	if err != nil {
		klog.Errorf("failed to init helm conf, err:%v", err)
		return allReady, err
	}

	allResources := make([]*resource.Info, 0)
	for _, i := range release.Hooks {
		hookResources, err := helmConf.KubeClient.Build(bytes.NewBufferString(i.Manifest), false)
		if err != nil {
			klog.Errorf("failed to get helm hookResources, err:%v", err)
			return allReady, err
		}
		allResources = append(allResources, hookResources...)
	}
	klog.Infof("%s get helm hookResources %d", release.Name, len(allResources))

	chartResources, err := helmConf.KubeClient.Build(bytes.NewBufferString(release.Manifest), false)
	if err != nil {
		klog.Errorf("failed to get helm resources, err:%v", err)
		return allReady, err
	}
	allResources = append(allResources, chartResources...)
	klog.Infof("%s get helm chartResources %d", release.Name, len(chartResources))

	for idx, j := range allResources {
		kind := j.Object.GetObjectKind().GroupVersionKind().Kind
		klog.Infof("[%d/%d] check helm release %s  %s: %s/%s", idx+1, len(allResources),
			release.Name, kind, j.Namespace, j.Name)
		ready, err := checker.IsReady(context.Background(), j)
		if k8serr.IsNotFound(err) {
			//pre-job-->chart-resource-->post-job
			//If a certain step times out, the subsequent steps will not be created,
			//and the status is considered failed, no repair will be made.
			klog.Warningf("[%d/%d] helm release %s resource %s: %s/%s not found", idx+1, len(allResources), release.Name, kind, j.Namespace, j.Name)
			return false, nil
		}

		if err != nil {
			klog.Errorf("failed to check resource ready, err:%v", err)
			return allReady, err
		}
		if !ready {
			klog.Infof("[%d/%d] helm release %s resource %s: %s/%s not ready", idx+1, len(allResources), release.Name, kind, j.Namespace, j.Name)
			return allReady, nil
		}
	}
	return true, nil
}

func GvkToGvr(gvk *schema.GroupVersionKind, mapper meta.RESTMapper) (schema.GroupVersionResource, error) {
	mapping, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
	if meta.IsNoMatchError(err) || err != nil {
		return schema.GroupVersionResource{}, err
	}
	return mapping.Resource, nil
}
func GetInfoFromBytes(bytes json.RawMessage, mapper meta.RESTMapper) (gvr schema.GroupVersionResource, utd *unstructured.Unstructured, err error) {
	obj, gvk, err := Decode(bytes)
	if err != nil {
		return gvr, utd, err
	}
	gvr, err = GvkToGvr(gvk, mapper)
	if err != nil {
		return gvr, utd, err
	}
	utd, err = ConvertToUnstructured(obj)
	return gvr, utd, err
}
func ConvertToUnstructured(obj any) (*unstructured.Unstructured, error) {
	objMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
	return &unstructured.Unstructured{Object: objMap}, err
}

func ComplianceCheck(values, tempLate []byte, mapper meta.RESTMapper, ns string) (result []json.RawMessage, err error) {
	yamlList, err := ReadYaml(values)
	if err != nil {
		return nil, err
	}
	yamlTempList, err := ReadYaml(tempLate)
	if err != nil {
		return nil, err
	}

	if len(yamlTempList) != len(yamlList) {
		return nil, errors.New("yamlList and yamlTempList length not equal")
	}
	for idx := range yamlTempList {
		_, utd, err := GetInfoFromBytes(yamlList[idx], mapper)
		if err != nil {
			return nil, err
		}
		_, utdTemp, err := GetInfoFromBytes(yamlTempList[idx], mapper)
		if err != nil {
			return nil, err
		}
		if utdTemp.GetKind() != utd.GetKind() || utdTemp.GetAPIVersion() != utd.GetAPIVersion() {
			return nil, errors.New("yamlList and yamlTempList not equal")
		}
		if utd.GetNamespace() != ns {
			return nil, errors.New("subresource must have same namespace with app release")
		}
	}
	return yamlList, nil
}

func GetUuid36(prefix string) string {
	id := idutils.GetIntId()
	hd := hashids.NewData()
	hd.Alphabet = idutils.Alphabet36
	h, err := hashids.NewWithData(hd)
	if err != nil {
		panic(err)
	}
	i, err := h.Encode([]int{int(id)})
	if err != nil {
		panic(err)
	}
	//hashids.minAlphabetLength = 16
	add := stringutil.Reverse(i)[:5]

	return prefix + add
}

func GenerateShortNameMD5Hash(input string) string {
	input = strings.ToLower(input)
	errs := validation.IsDNS1123Subdomain(input)
	if len(input) > 14 || len(errs) != 0 {
		hash := md5.New()
		hash.Write([]byte(input))
		hashInBytes := hash.Sum(nil)
		hashString := hex.EncodeToString(hashInBytes)
		return hashString[:10]
	}
	return input
}

func FormatVersion(input string) string {
	if len(validation.IsDNS1123Subdomain(input)) == 0 {
		return input
	}
	hash := sha1.Sum([]byte(input))
	formattedVersion := hex.EncodeToString(hash[:])[:12]
	return formattedVersion
}

func GetHelmKubeConfig(ctx context.Context, cluster *clusterv1alpha1.Cluster, runClient client.Client) (config []byte, err error) {

	if cluster.Spec.Connection.Type == clusterv1alpha1.ConnectionTypeProxy {
		klog.Infof("cluster %s is proxy cluster", cluster.Name)
		secret := &corev1.Secret{}
		key := types.NamespacedName{Namespace: pkgconstants.KubeSphereNamespace, Name: "kubeconfig-admin"}
		err = runClient.Get(ctx, key, secret)
		if err != nil {
			klog.Errorf("failed to get kubeconfig-admin secret: %v", err)
			return nil, err
		}
		config = secret.Data["config"]
		return config, err
	}
	return cluster.Spec.Connection.KubeConfig, nil
}