openpitrix crd

Signed-off-by: LiHui <andrewli@yunify.com>

delete helm repo, release and app

Signed-off-by: LiHui <andrewli@yunify.com>

Fix Dockerfile

Signed-off-by: LiHui <andrewli@yunify.com>

add unit test for category controller

Signed-off-by: LiHui <andrewli@yunify.com>

resource api

Signed-off-by: LiHui <andrewli@yunify.com>

miscellaneous

Signed-off-by: LiHui <andrewli@yunify.com>

resource api

Signed-off-by: LiHui <andrewli@yunify.com>

add s3 repo indx

Signed-off-by: LiHui <andrewli@yunify.com>

attachment api

Signed-off-by: LiHui <andrewli@yunify.com>

repo controller test

Signed-off-by: LiHui <andrewli@yunify.com>

application controller test

Signed-off-by: LiHui <andrewli@yunify.com>

release metric

Signed-off-by: LiHui <andrewli@yunify.com>

helm release controller test

Signed-off-by: LiHui <andrewli@yunify.com>

move constants to /pkg/apis/application

Signed-off-by: LiHui <andrewli@yunify.com>

remove unused code

Signed-off-by: LiHui <andrewli@yunify.com>

add license header

Signed-off-by: LiHui <andrewli@yunify.com>

Fix bugs

Signed-off-by: LiHui <andrewli@yunify.com>

cluster cluent

Signed-off-by: LiHui <andrewli@yunify.com>

format code

Signed-off-by: LiHui <andrewli@yunify.com>

move workspace,cluster from spec to labels

Signed-off-by: LiHui <andrewli@yunify.com>

add license header

Signed-off-by: LiHui <andrewli@yunify.com>

openpitrix test

Signed-off-by: LiHui <andrewli@yunify.com>

add worksapce labels for app in appstore

Signed-off-by: LiHui <andrewli@yunify.com>
This commit is contained in:
LiHui
2020-12-23 15:24:30 +08:00
parent 737639020b
commit 83e6221f3a
193 changed files with 19634 additions and 6039 deletions

View File

@@ -0,0 +1,36 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helmrepoindex
import "time"
type VersionInterface interface {
GetName() string
GetVersion() string
GetAppVersion() string
GetDescription() string
GetUrls() string
GetVersionName() string
GetIcon() string
GetHome() string
GetSources() string
GetKeywords() string
GetMaintainers() string
GetScreenshots() string
GetPackageName() string
GetCreateTime() time.Time
}

View File

@@ -0,0 +1,104 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helmrepoindex
import (
"bytes"
"context"
"fmt"
"helm.sh/helm/v3/pkg/getter"
"kubesphere.io/kubesphere/pkg/apis/application/v1alpha1"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"net/url"
"strings"
"time"
)
func parseS3Url(parse *url.URL) (region, endpoint, bucket, path string) {
if strings.HasPrefix(parse.Host, "s3.") {
region = strings.Split(parse.Host, ".")[1]
endpoint = fmt.Sprintf("https://%s", parse.Host)
} else {
region = "us-east-1"
endpoint = fmt.Sprintf("http://%s", parse.Host)
}
parts := strings.Split(strings.TrimPrefix(parse.Path, "/"), "/")
if len(parts) > 0 {
bucket = parts[0]
path = strings.Join(parts[1:], "/")
} else {
bucket = parse.Path
}
return region, endpoint, bucket, path
}
func loadData(ctx context.Context, u string, cred *v1alpha1.HelmRepoCredential) (*bytes.Buffer, error) {
parsedURL, err := url.Parse(u)
if err != nil {
return nil, err
}
var resp *bytes.Buffer
if strings.HasPrefix(u, "s3://") {
region, endpoint, bucket, p := parseS3Url(parsedURL)
client, err := s3.NewS3Client(&s3.Options{
Endpoint: endpoint,
Bucket: bucket,
Region: region,
AccessKeyID: cred.AccessKeyID,
SecretAccessKey: cred.SecretAccessKey,
DisableSSL: !strings.HasPrefix(region, "https://"),
ForcePathStyle: true,
})
if err != nil {
return nil, err
}
data, err := client.Read(p)
if err != nil {
return nil, err
}
resp = bytes.NewBuffer(data)
} else {
skipTLS := true
if cred.InsecureSkipTLSVerify != nil && !*cred.InsecureSkipTLSVerify {
skipTLS = false
}
indexURL := parsedURL.String()
// TODO add user-agent
g, _ := getter.NewHTTPGetter()
resp, err = g.Get(indexURL,
getter.WithTimeout(5*time.Minute),
getter.WithURL(u),
getter.WithInsecureSkipVerifyTLS(skipTLS),
getter.WithTLSClientConfig(cred.CertFile, cred.KeyFile, cred.CAFile),
getter.WithBasicAuth(cred.Username, cred.Password),
)
if err != nil {
return nil, err
}
}
return resp, nil
}
func LoadChart(ctx context.Context, u string, cred *v1alpha1.HelmRepoCredential) (*bytes.Buffer, error) {
return loadData(ctx, u, cred)
}

View File

@@ -0,0 +1,96 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helmrepoindex
import (
"bytes"
"encoding/json"
"fmt"
"helm.sh/helm/v3/pkg/repo"
"k8s.io/klog"
"strings"
"time"
"helm.sh/helm/v3/pkg/chart/loader"
)
func LoadPackage(pkg []byte) (VersionInterface, error) {
p, err := loader.LoadArchive(bytes.NewReader(pkg))
if err != nil {
klog.Errorf("Failed to load package, error: %+v", err)
return nil, err
}
return HelmVersionWrapper{ChartVersion: &repo.ChartVersion{Metadata: p.Metadata}}, nil
}
type HelmVersionWrapper struct {
*repo.ChartVersion
}
func (h HelmVersionWrapper) GetIcon() string { return h.ChartVersion.Icon }
func (h HelmVersionWrapper) GetName() string { return h.ChartVersion.Name }
func (h HelmVersionWrapper) GetHome() string { return h.ChartVersion.Home }
func (h HelmVersionWrapper) GetVersion() string { return h.ChartVersion.Version }
func (h HelmVersionWrapper) GetAppVersion() string { return h.ChartVersion.AppVersion }
func (h HelmVersionWrapper) GetDescription() string { return h.ChartVersion.Description }
func (h HelmVersionWrapper) GetCreateTime() time.Time { return h.ChartVersion.Created }
func (h HelmVersionWrapper) GetUrls() string {
if len(h.ChartVersion.URLs) == 0 {
return ""
}
return h.ChartVersion.URLs[0]
}
func (h HelmVersionWrapper) GetSources() string {
if len(h.ChartVersion.Sources) == 0 {
return ""
}
s, _ := json.Marshal(h.ChartVersion.Sources)
return string(s)
}
func (h HelmVersionWrapper) GetKeywords() string {
return strings.Join(h.ChartVersion.Keywords, ",")
}
func (h HelmVersionWrapper) GetMaintainers() string {
if len(h.ChartVersion.Maintainers) == 0 {
return ""
}
s, _ := json.Marshal(h.ChartVersion.Maintainers)
return string(s)
}
func (h HelmVersionWrapper) GetScreenshots() string {
return ""
}
func (h HelmVersionWrapper) GetVersionName() string {
versionName := h.GetVersion()
if h.GetAppVersion() != "" {
versionName += fmt.Sprintf(" [%s]", h.GetAppVersion())
}
return versionName
}
func (h HelmVersionWrapper) GetPackageName() string {
file := h.GetUrls()
if len(file) == 0 {
return fmt.Sprintf("%s-%s.tgz", h.Name, h.Version)
}
return file
}

View File

@@ -0,0 +1,270 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helmrepoindex
import (
"bytes"
"compress/zlib"
"context"
"encoding/base64"
"encoding/json"
"fmt"
helmrepo "helm.sh/helm/v3/pkg/repo"
"io"
"io/ioutil"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"kubesphere.io/kubesphere/pkg/apis/application/v1alpha1"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/utils/idutils"
"sigs.k8s.io/yaml"
"strings"
"time"
)
const IndexYaml = "index.yaml"
func LoadRepoIndex(ctx context.Context, u string, cred *v1alpha1.HelmRepoCredential) (*helmrepo.IndexFile, error) {
if !strings.HasSuffix(u, "/") {
u = fmt.Sprintf("%s/%s", u, IndexYaml)
} else {
u = fmt.Sprintf("%s%s", u, IndexYaml)
}
resp, err := loadData(ctx, u, cred)
if err != nil {
return nil, err
}
indexFile, err := loadIndex(resp.Bytes())
if err != nil {
return nil, err
}
return indexFile, nil
}
// loadIndex loads an index file and does minimal validity checking.
//
// This will fail if API Version is not set (ErrNoAPIVersion) or if the unmarshal fails.
func loadIndex(data []byte) (*helmrepo.IndexFile, error) {
i := &helmrepo.IndexFile{}
if err := yaml.UnmarshalStrict(data, i); err != nil {
return i, err
}
i.SortEntries()
if i.APIVersion == "" {
return i, helmrepo.ErrNoAPIVersion
}
return i, nil
}
// merge new index with index from crd
func MergeRepoIndex(index *helmrepo.IndexFile, existsSavedIndex *SavedIndex) *SavedIndex {
saved := &SavedIndex{}
if index == nil {
return existsSavedIndex
}
saved.Applications = make(map[string]*Application)
if existsSavedIndex != nil {
for name := range existsSavedIndex.Applications {
saved.Applications[name] = existsSavedIndex.Applications[name]
}
}
// just copy fields from index
saved.APIVersion = index.APIVersion
saved.Generated = index.Generated
saved.PublicKeys = index.PublicKeys
allNames := make(map[string]bool, len(index.Entries))
for name, versions := range index.Entries {
// add new applications
if application, exists := saved.Applications[name]; !exists {
application = &Application{
Name: name,
ApplicationId: idutils.GetUuid36(v1alpha1.HelmApplicationIdPrefix),
Description: versions[0].Description,
Icon: versions[0].Icon,
}
charts := make([]*ChartVersion, 0, len(versions))
for ind := range versions {
chart := &ChartVersion{
ApplicationId: application.ApplicationId,
ApplicationVersionId: idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix),
ChartVersion: *versions[ind],
}
charts = append(charts, chart)
}
application.Charts = charts
saved.Applications[name] = application
} else {
// update exists applications
savedChartVersion := make(map[string]struct{})
for _, ver := range application.Charts {
savedChartVersion[ver.Version] = struct{}{}
}
charts := application.Charts
for _, ver := range versions {
// add new chart version
if _, exists := savedChartVersion[ver.Version]; !exists {
chart := &ChartVersion{
ApplicationId: application.ApplicationId,
ApplicationVersionId: idutils.GetUuid36(v1alpha1.HelmApplicationVersionIdPrefix),
ChartVersion: *ver,
}
charts = append(charts, chart)
}
application.Charts = charts
saved.Applications[name] = application
}
}
allNames[name] = true
}
for name := range saved.Applications {
if _, exists := allNames[name]; !exists {
delete(saved.Applications, name)
}
}
return saved
}
func (i *SavedIndex) GetApplicationVersion(appId, versionId string) *v1alpha1.HelmApplicationVersion {
for _, app := range i.Applications {
if app.ApplicationId == appId {
for _, ver := range app.Charts {
if ver.ApplicationVersionId == versionId {
version := &v1alpha1.HelmApplicationVersion{
ObjectMeta: metav1.ObjectMeta{
Name: versionId,
Labels: map[string]string{
constants.ChartApplicationIdLabelKey: appId,
},
},
Spec: v1alpha1.HelmApplicationVersionSpec{
URLs: ver.URLs,
Digest: ver.Digest,
Metadata: &v1alpha1.Metadata{
Name: ver.Name,
AppVersion: ver.AppVersion,
Version: ver.Version,
Annotations: ver.Annotations,
},
},
}
return version
}
}
}
}
return nil
}
type SavedIndex struct {
APIVersion string `json:"apiVersion"`
Generated time.Time `json:"generated"`
Applications map[string]*Application `json:"apps"`
PublicKeys []string `json:"publicKeys,omitempty"`
// Annotations are additional mappings uninterpreted by Helm. They are made available for
// other applications to add information to the index file.
Annotations map[string]string `json:"annotations,omitempty"`
}
func ByteArrayToSavedIndex(data []byte) (*SavedIndex, error) {
ret := &SavedIndex{}
if len(data) == 0 {
return ret, nil
}
enc := base64.URLEncoding
buf := make([]byte, enc.DecodedLen(len(data)))
n, err := enc.Decode(buf, data)
if err != nil {
return nil, err
}
buf = buf[:n]
r, err := zlib.NewReader(bytes.NewBuffer(buf))
if err != nil {
return nil, err
}
r.Close()
b, err := ioutil.ReadAll(r)
if err != nil && err != io.EOF {
return nil, err
}
err = json.Unmarshal(b, ret)
if err != nil {
return nil, err
}
return ret, nil
}
func (i *SavedIndex) Bytes() ([]byte, error) {
d, err := json.Marshal(i)
if err != nil {
return nil, err
}
buf := &bytes.Buffer{}
w := zlib.NewWriter(buf)
_, err = w.Write(d)
if err != nil {
return nil, err
}
err = w.Close()
if err != nil {
return nil, err
}
encSrc := buf.Bytes()
enc := base64.URLEncoding
ret := make([]byte, enc.EncodedLen(len(encSrc)))
enc.Encode(ret, encSrc)
return ret, nil
}
// chart version with app id and app version id
type ChartVersion struct {
// Do not save ApplicationId into crd
ApplicationId string `json:"-"`
ApplicationVersionId string `json:"verId"`
helmrepo.ChartVersion `json:",inline"`
}
type Application struct {
// application name
Name string `json:"name"`
ApplicationId string `json:"appId"`
// chart description
Description string `json:"desc"`
// application status
Status string `json:"status"`
// The URL to an icon file.
Icon string `json:"icon,omitempty"`
Charts []*ChartVersion `json:"charts"`
}

View File

@@ -0,0 +1,53 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package helmrepoindex
import (
"context"
"fmt"
"kubesphere.io/kubesphere/pkg/apis/application/v1alpha1"
"strings"
"testing"
)
func TestLoadRepo(t *testing.T) {
u := "https://charts.kubesphere.io/main"
index, err := LoadRepoIndex(context.TODO(), u, &v1alpha1.HelmRepoCredential{})
if err != nil {
t.Errorf("load repo failed, err: %s", err)
t.Failed()
return
}
for _, entry := range index.Entries {
chartUrl := entry[0].URLs[0]
if !(strings.HasPrefix(chartUrl, "https://") || strings.HasPrefix(chartUrl, "http://")) {
chartUrl = fmt.Sprintf("%s/%s", u, chartUrl)
}
chartData, err := LoadChart(context.TODO(), chartUrl, &v1alpha1.HelmRepoCredential{})
if err != nil {
t.Errorf("load chart data failed, err: %s", err)
t.Failed()
}
_ = chartData
break
}
}