/* Copyright 2020 The KubeSphere Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package openpitrix import ( "bytes" "context" "encoding/base64" "errors" "fmt" "reflect" "sort" "helm.sh/helm/v3/pkg/chart/loader" "kubesphere.io/kubesphere/pkg/apiserver/query" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/s3" "github.com/go-openapi/strfmt" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" "kubesphere.io/api/application/v1alpha1" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmrepoindex" "kubesphere.io/kubesphere/pkg/utils/stringutils" ) func (c *applicationOperator) GetAppVersionPackage(appId, versionId string) (*GetAppVersionPackageResponse, error) { var version *v1alpha1.HelmApplicationVersion var err error version, err = c.getAppVersionByVersionIdWithData(versionId) if err != nil { return nil, err } return &GetAppVersionPackageResponse{ AppId: appId, VersionId: versionId, Package: version.Spec.Data, }, nil } // check helm package and create helm app version if not exist func (c *applicationOperator) CreateAppVersion(request *CreateAppVersionRequest) (*CreateAppVersionResponse, error) { if c.backingStoreClient == nil { return nil, invalidS3Config } chrt, err := helmrepoindex.LoadPackage(request.Package) if err != nil { klog.Errorf("load package failed, error: %s", err) return nil, err } app, err := c.appLister.Get(request.AppId) if err != nil { klog.Errorf("get app %s failed, error: %s", request.AppId, err) return nil, err } chartPackage := request.Package.String() version := buildApplicationVersion(app, chrt, &chartPackage, request.Username) version, err = c.createApplicationVersion(version) if err != nil { klog.Errorf("create helm app version failed, error: %s", err) return nil, err } klog.V(4).Infof("create helm app version %s success", request.Name) return &CreateAppVersionResponse{ VersionId: version.GetHelmApplicationVersionId(), }, nil } func (c *applicationOperator) DeleteAppVersion(id string) error { appVersion, err := c.versionLister.Get(id) if err != nil { if apierrors.IsNotFound(err) { return nil } else { klog.Infof("get app version %s failed, error: %s", id, err) return err } } switch appVersion.Status.State { case v1alpha1.StateActive: klog.Warningf("delete app version %s/%s not permitted, current state:%s", appVersion.GetWorkspace(), appVersion.GetTrueName(), appVersion.Status.State) return actionNotPermitted } // Delete data in storage err = c.backingStoreClient.Delete(dataKeyInStorage(appVersion.GetWorkspace(), id)) if err != nil { if aerr, ok := err.(awserr.Error); ok && aerr.Code() != s3.ErrCodeNoSuchKey { klog.Errorf("delete app version %s/%s data failed, error: %s", appVersion.GetWorkspace(), appVersion.Name, err) return deleteDataInStorageFailed } } // delete app version in etcd err = c.appVersionClient.Delete(context.TODO(), id, metav1.DeleteOptions{}) if err != nil { if apierrors.IsNotFound(err) { return nil } klog.Errorf("delete app version %s failed", err) return err } else { klog.Infof("app version %s deleted", id) } return nil } func (c *applicationOperator) DescribeAppVersion(id string) (*AppVersion, error) { version, err := c.getAppVersion(id) if err != nil { klog.Errorf("get app version [%s] failed, error: %s", id, err) return nil, err } app := convertAppVersion(version) return app, nil } func (c *applicationOperator) ModifyAppVersion(id string, request *ModifyAppVersionRequest) error { version, err := c.getAppVersion(id) if err != nil { klog.Errorf("get app version [%s] failed, error: %s", id, err) return err } // All the app versions belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else. if repoId, exists := version.Labels[constants.ChartRepoIdLabelKey]; exists && repoId != v1alpha1.AppStoreRepoId { return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplicationVersion), version.Name, errors.New("version is immutable")) } versionCopy := version.DeepCopy() spec := &versionCopy.Spec // extract information from chart package if len(request.Package) > 0 { if version.Status.State != v1alpha1.StateDraft { return actionNotPermitted } // 1. Parse the chart package chart, err := helmrepoindex.LoadPackage(request.Package) if err != nil { klog.Errorf("load package failed, error: %s", err) return err } // chart name must match with the original one if spec.Name != chart.GetName() { return fmt.Errorf("chart name not match, current name: %s, original name: %s", chart.GetName(), spec.Name) } // new version name if chart.GetVersionName() != version.GetVersionName() { existsVersion, err := c.getAppVersionByVersionName(version.GetHelmApplicationId(), chart.GetVersionName()) if err != nil { return err } if existsVersion != nil { return appVersionItemExists } } // 2. update crd info spec.Version = chart.GetVersion() spec.AppVersion = chart.GetAppVersion() spec.Icon = chart.GetIcon() spec.Home = chart.GetHome() spec.Description = stringutils.ShortenString(chart.GetDescription(), v1alpha1.MsgLen) now := metav1.Now() spec.Created = &now // 3. save chart data to s3 storage, just overwrite the legacy data err = c.backingStoreClient.Upload(dataKeyInStorage(versionCopy.GetWorkspace(), versionCopy.Name), versionCopy.Name, bytes.NewBuffer(request.Package), len(request.Package)) if err != nil { klog.Errorf("upload chart for app version: %s/%s failed, error: %s", versionCopy.GetWorkspace(), versionCopy.GetTrueName(), err) return uploadChartDataFailed } else { klog.V(4).Infof("chart data uploaded for app version: %s/%s", versionCopy.GetWorkspace(), versionCopy.GetTrueName()) } } else { // new version name if request.Name != nil && *request.Name != "" && version.GetVersionName() != *request.Name { spec.Version, spec.AppVersion = parseChartVersionName(*request.Name) existsVersion, err := c.getAppVersionByVersionName(version.GetHelmApplicationId(), *request.Name) if err != nil { return err } if existsVersion != nil { return appVersionItemExists } } if request.Description != nil && *request.Description != "" { spec.Description = stringutils.ShortenString(*request.Description, v1alpha1.MsgLen) } } patch := client.MergeFrom(version) data, err := patch.Data(versionCopy) if err != nil { klog.Error("create patch failed", err) return err } // data == "{}", need not to patch if len(data) == 2 { return nil } _, err = c.appVersionClient.Patch(context.TODO(), id, patch.Type(), data, metav1.PatchOptions{}) if err != nil { klog.Error(err) return err } return nil } func (c *applicationOperator) ListAppVersions(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { versions, err := c.getAppVersionsByAppId(conditions.Match[AppId]) if err != nil { klog.Error(err) return nil, err } versions = filterAppVersions(versions, conditions) if reverse { sort.Sort(sort.Reverse(AppVersions(versions))) } else { sort.Sort(AppVersions(versions)) } totalCount := len(versions) start, end := (&query.Pagination{Limit: limit, Offset: offset}).GetValidPagination(totalCount) versions = versions[start:end] items := make([]interface{}, 0, len(versions)) for i := range versions { items = append(items, convertAppVersion(versions[i])) } return &models.PageableResponse{Items: items, TotalCount: totalCount}, nil } func (c *applicationOperator) ListAppVersionReviews(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { appVersions, err := c.versionLister.List(labels.Everything()) if err != nil { klog.Error(err) return nil, err } filtered := filterAppReviews(appVersions, conditions) if reverse { sort.Sort(sort.Reverse(AppVersionReviews(filtered))) } else { sort.Sort(AppVersionReviews(filtered)) } totalCount := len(filtered) start, end := (&query.Pagination{Limit: limit, Offset: offset}).GetValidPagination(totalCount) filtered = filtered[start:end] items := make([]interface{}, 0, len(filtered)) for i := range filtered { app, err := c.appLister.Get(filtered[i].GetHelmApplicationId()) if err != nil { return nil, err } review := convertAppVersionReview(app, filtered[i]) items = append(items, review) } return &models.PageableResponse{Items: items, TotalCount: totalCount}, nil } func (c *applicationOperator) ListAppVersionAudits(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { appId := conditions.Match[AppId] versionId := conditions.Match[VersionId] var versions []*v1alpha1.HelmApplicationVersion var err error if versionId == "" { ls := map[string]string{ constants.ChartApplicationIdLabelKey: appId, } versions, err = c.versionLister.List(labels.SelectorFromSet(ls)) if err != nil { klog.Errorf("get app %s failed, error: %s", appId, err) } } else { version, err := c.versionLister.Get(versionId) if err != nil { klog.Errorf("get app version %s failed, error: %s", versionId, err) } versions = []*v1alpha1.HelmApplicationVersion{version} } var allAudits []*AppVersionAudit for _, item := range versions { audits := convertAppVersionAudit(item) allAudits = append(allAudits, audits...) } sort.Sort(AppVersionAuditList(allAudits)) totalCount := len(allAudits) start, end := (&query.Pagination{Limit: limit, Offset: offset}).GetValidPagination(totalCount) allAudits = allAudits[start:end] items := make([]interface{}, 0, len(allAudits)) for i := range allAudits { items = append(items, allAudits[i]) } return &models.PageableResponse{Items: items, TotalCount: totalCount}, nil } func (c *applicationOperator) DoAppVersionAction(versionId string, request *ActionRequest) error { var err error t := metav1.Now() var audit = v1alpha1.Audit{ Message: request.Message, Operator: request.Username, Time: t, } state := v1alpha1.StateDraft version, err := c.getAppVersion(versionId) if err != nil { klog.Errorf("get app version %s failed, error: %s", versionId, err) return err } // All the app versions belonging to a built-in repo have a label `application.kubesphere.io/repo-id`, and the value should be `builtin-stable` or else. if repoId, exists := version.Labels[constants.ChartRepoIdLabelKey]; exists && repoId != v1alpha1.AppStoreRepoId { return apierrors.NewForbidden(v1alpha1.Resource(v1alpha1.ResourcePluralHelmApplicationVersion), version.Name, errors.New("version is immutable")) } switch request.Action { case ActionCancel: if version.Status.State != v1alpha1.StateSubmitted { } state = v1alpha1.StateDraft audit.State = v1alpha1.StateDraft case ActionPass: if version.Status.State != v1alpha1.StateSubmitted { } state = v1alpha1.StatePassed audit.State = v1alpha1.StatePassed case ActionRecover: if version.Status.State != v1alpha1.StateSuspended { } state = v1alpha1.StateActive audit.State = v1alpha1.StateActive case ActionReject: if version.Status.State != v1alpha1.StateSubmitted { // todo check status } state = v1alpha1.StateRejected audit.State = v1alpha1.StateRejected case ActionSubmit: if version.Status.State != v1alpha1.StateDraft { // todo check status } state = v1alpha1.StateSubmitted audit.State = v1alpha1.StateSubmitted case ActionSuspend: if version.Status.State != v1alpha1.StateActive { // todo check status } state = v1alpha1.StateSuspended audit.State = v1alpha1.StateSuspended case ActionRelease: // release to app store if version.Status.State != v1alpha1.StatePassed { // todo check status } state = v1alpha1.StateActive audit.State = v1alpha1.StateActive default: err = errors.New("action not support") } _ = state if err != nil { klog.Error(err) return err } version, err = c.updateAppVersionStatus(version, state, &audit) if err != nil { klog.Errorf("update app version audit [%s] failed, error: %s", versionId, err) return err } if request.Action == ActionRelease || request.Action == ActionRecover { // if we release a new helm application version, we need update the spec in helm application copy app, err := c.appLister.Get(version.GetHelmApplicationId()) if err != nil { return err } appInStore, err := c.appLister.Get(fmt.Sprintf("%s%s", version.GetHelmApplicationId(), v1alpha1.HelmApplicationAppStoreSuffix)) if err != nil { if apierrors.IsNotFound(err) { // controller-manager will create application in app store return nil } return err } if !reflect.DeepEqual(&app.Spec, &appInStore.Spec) { appCopy := appInStore.DeepCopy() appCopy.Spec = app.Spec patch := client.MergeFrom(appInStore) data, _ := patch.Data(appCopy) _, err = c.appClient.Patch(context.TODO(), appCopy.Name, patch.Type(), data, metav1.PatchOptions{}) if err != nil { return err } } } return nil } func (c *applicationOperator) getAppVersionByVersionName(appId, verName string) (*v1alpha1.HelmApplicationVersion, error) { ls := map[string]string{ constants.ChartApplicationIdLabelKey: appId, } versions, err := c.versionLister.List(labels.SelectorFromSet(ls)) if err != nil && !apierrors.IsNotFound(err) { return nil, err } for _, ver := range versions { if verName == ver.GetVersionName() { return ver, nil } } return nil, nil } // Create helmApplicationVersion and helmAudit func (c *applicationOperator) createApplicationVersion(ver *v1alpha1.HelmApplicationVersion) (*v1alpha1.HelmApplicationVersion, error) { existsVersion, err := c.getAppVersionByVersionName(ver.GetHelmApplicationId(), ver.GetVersionName()) if err != nil { return nil, err } if existsVersion != nil { klog.V(2).Infof("helm application version: %s exist", ver.GetVersionName()) return nil, appVersionItemExists } // save chart data to s3 storage _, err = base64.StdEncoding.Decode(ver.Spec.Data, ver.Spec.Data) if err != nil { klog.Errorf("decode error: %s", err) return nil, err } else { err = c.backingStoreClient.Upload(dataKeyInStorage(ver.GetWorkspace(), ver.Name), ver.Name, bytes.NewBuffer(ver.Spec.Data), len(ver.Spec.Data)) if err != nil { klog.Errorf("upload chart for app version: %s/%s failed, error: %s", ver.GetWorkspace(), ver.GetTrueName(), err) return nil, uploadChartDataFailed } else { klog.V(4).Infof("chart data uploaded for app version: %s/%s", ver.GetWorkspace(), ver.GetTrueName()) } } // data will not save to etcd ver.Spec.Data = nil ver.Spec.DataKey = ver.Name version, err := c.appVersionClient.Create(context.TODO(), ver, metav1.CreateOptions{}) if err == nil { klog.V(4).Infof("create helm application %s version success", version.Name) } return version, err } func (c *applicationOperator) updateAppVersionStatus(version *v1alpha1.HelmApplicationVersion, state string, status *v1alpha1.Audit) (*v1alpha1.HelmApplicationVersion, error) { version.Status.State = state states := append([]v1alpha1.Audit{*status}, version.Status.Audit...) if len(version.Status.Audit) >= v1alpha1.HelmRepoSyncStateLen { // strip the last item states = states[:v1alpha1.HelmRepoSyncStateLen:v1alpha1.HelmRepoSyncStateLen] } version.Status.Audit = states version, err := c.appVersionClient.UpdateStatus(context.TODO(), version, metav1.UpdateOptions{}) return version, err } func (c *applicationOperator) GetAppVersionFiles(versionId string, request *GetAppVersionFilesRequest) (*GetAppVersionPackageFilesResponse, error) { var version *v1alpha1.HelmApplicationVersion var err error // get chart data version, err = c.getAppVersionByVersionIdWithData(versionId) if err != nil { klog.Errorf("get app version %s chart data failed: %v", versionId, err) return nil, err } // parse chart data chartData, err := loader.LoadArchive(bytes.NewReader(version.Spec.Data)) if err != nil { klog.Errorf("Failed to load package for app version: %s, error: %+v", versionId, err) return nil, err } res := &GetAppVersionPackageFilesResponse{Files: map[string]strfmt.Base64{}, VersionId: versionId} for _, f := range chartData.Raw { res.Files[f.Name] = f.Data } return res, nil } func (c *applicationOperator) getAppVersionByVersionIdWithData(versionId string) (*v1alpha1.HelmApplicationVersion, error) { if version, exists, err := c.cachedRepos.GetAppVersionWithData(versionId); exists { if err != nil { return nil, err } return version, nil } version, err := c.versionLister.Get(versionId) if err != nil { return nil, err } data, err := c.backingStoreClient.Read(dataKeyInStorage(version.GetWorkspace(), versionId)) if err != nil { klog.Errorf("load chart data for app version: %s/%s failed, error : %s", version.GetTrueName(), version.GetTrueName(), err) return nil, downloadFileFailed } version.Spec.Data = data return version, nil } func (c *applicationOperator) getAppVersionsByAppId(appId string) (ret []*v1alpha1.HelmApplicationVersion, err error) { if ret, exists := c.cachedRepos.ListAppVersionsByAppId(appId); exists { return ret, nil } // list app version from client-go ret, err = c.versionLister.List(labels.SelectorFromSet(map[string]string{constants.ChartApplicationIdLabelKey: appId})) if err != nil && !apierrors.IsNotFound(err) { klog.Error(err) return nil, err } return } // get app version from repo and helm application func (c *applicationOperator) getAppVersion(id string) (ret *v1alpha1.HelmApplicationVersion, err error) { if ver, exists, _ := c.cachedRepos.GetAppVersion(id); exists { return ver, nil } ret, err = c.versionLister.Get(id) return }