fix: add tls when get repository index. (#6195)

* fix: add tls when get repository index.

Signed-off-by: joyceliu <joyceliu@yunify.com>

* Update staging/src/kubesphere.io/utils/helm/repo_index.go

Signed-off-by: hongming <coder.scala@gmail.com>

* fix: add tls when get repository index.

Signed-off-by: joyceliu <joyceliu@yunify.com>

---------

Signed-off-by: joyceliu <joyceliu@yunify.com>
Signed-off-by: hongming <coder.scala@gmail.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
Co-authored-by: hongming <coder.scala@gmail.com>

Makefile

@@ -2,9 +2,6 @@
 CRD_OPTIONS ?= "crd:allowDangerousTypes=true"
 MANIFESTS="cluster/v1alpha1 iam/... quota/v1alpha2 storage/v1alpha1 tenant/... extensions/v1alpha1 core/v1alpha1 gateway/v1alpha2 application/v2"
 
-# App Version
-APP_VERSION = v3.2.0
-
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -112,19 +109,15 @@ container-cross-push: ; $(info $(M)...Begin to build and push.)  @ ## Build and
 	hack/docker_build_multiarch.sh
 
 helm-package: ; $(info $(M)...Begin to helm-package.)  @ ## Helm-package.
-	ls config/crds/ | xargs -i cp -r config/crds/{} config/ks-core/crds/
-	helm package config/ks-core --app-version=${APP_VERSION} --version=0.1.0 -d ./bin
+	helm package config/ks-core -d ./bin
 
 helm-deploy: ; $(info $(M)...Begin to helm-deploy.)  @ ## Helm-deploy.
-	ls config/crds/ | xargs -i cp -r config/crds/{} config/ks-core/crds/
 	- kubectl create ns kubesphere-controls-system
 	helm upgrade --install ks-core ./config/ks-core -n kubesphere-system --create-namespace
-	kubectl apply -f https://raw.githubusercontent.com/kubesphere/ks-installer/master/roles/ks-core/prepare/files/ks-init/role-templates.yaml
 
 helm-uninstall: ; $(info $(M)...Begin to helm-uninstall.)  @ ## Helm-uninstall.
 	- kubectl delete ns kubesphere-controls-system
 	helm uninstall ks-core -n kubesphere-system
-	kubectl delete -f https://raw.githubusercontent.com/kubesphere/ks-installer/master/roles/ks-core/prepare/files/ks-init/role-templates.yaml
 
 # Run tests
 test: vet test-env ;$(info $(M)...Begin to run tests.)  @ ## Run tests.

config/ks-core/charts/ks-crds/crds/application.kubesphere.io_applicationreleases.yaml

@@ -73,6 +73,8 @@ spec:
                 type: string
               appVersionID:
                 type: string
+              icon:
+                type: string
               values:
                 format: byte
                 type: string

config/ks-core/charts/ks-crds/crds/kubesphere.io_repositories.yaml

@@ -50,12 +50,18 @@ spec:
                     type: string
                 type: object
               caBundle:
-                description: if the caBundle is empty, use --insecure-skip-tls-verify.
+                description: The caBundle (base64 string) is used in helmExecutor
+                  to verify the helm server.
                 type: string
               description:
                 type: string
               image:
+                description: 'DEPRECATED: the field will remove in future versions,
+                  please use url.'
                 type: string
+              insecure:
+                description: --insecure-skip-tls-verify. default false
+                type: boolean
               updateStrategy:
                 properties:
                   registryPoll:

config/ks-core/templates/extension-museum.yaml

@@ -26,8 +26,38 @@ spec:
             - "/charts"
             - "--storage"
             - "local"
+            - "--tls-cert"
+            - "/etc/certs/tls.crt"
+            - "--tls-key"
+            - "/etc/certs/tls.key"
           ports:
             - containerPort: 8080
+          volumeMounts:
+            - name: certs
+              mountPath: /etc/certs/
+      volumes:
+        - name: certs
+          secret:
+            secretName: extensions-museum-certs
+---
+
+{{- $ca := genCA "self-signed-ca" 3650 }}
+{{- $cn := printf "%s-extensions-museum" .Release.Name }}
+{{- $altName1 := printf "extensions-museum.%s" .Release.Namespace }}
+{{- $altName2 := printf "extensions-museum.%s.svc" .Release.Namespace }}
+{{- $cert := genSignedCert $cn nil (list $altName1 $altName2) 3650 $ca }}
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: extensions-museum-certs
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/tls
+data:
+  ca.crt: {{ b64enc $ca.Cert }}
+  tls.crt: {{ b64enc $cert.Cert }}
+  tls.key: {{ b64enc $cert.Key }}
+
 ---
 apiVersion: v1
 kind: Service
@@ -39,7 +69,7 @@ spec:
     app: extensions-museum
   ports:
     - protocol: TCP
-      port: 80
+      port: 443
       targetPort: 8080
 
 ---
@@ -48,7 +78,8 @@ kind: Repository
 metadata:
   name: extensions-museum
 spec:
-  url: http://extensions-museum.{{ .Release.Namespace }}.svc
+  url: https://extensions-museum.{{ .Release.Namespace }}.svc
+  caBundle: {{ b64enc $ca.Cert }}
 
 ---
 apiVersion: {{ if semverCompare ">=1.20.0" .Capabilities.KubeVersion.Version }}batch/v1{{ else }}batch/v1beta1{{end}}

hack/generate_manifests.sh

@@ -17,6 +17,6 @@ for PKG in "${PKGS[@]}"; do
     go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=./staging/src/kubesphere.io/api/"${PKG}"
   else
     echo "Generating manifests for ${PKG}"
-    go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=./staging/src/kubesphere.io/api/"${PKG}" rbac:roleName=controller-perms "${CRD_OPTIONS}" output:crd:artifacts:config=config/ks-core/crds
+    go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=./staging/src/kubesphere.io/api/"${PKG}" rbac:roleName=controller-perms "${CRD_OPTIONS}" output:crd:artifacts:config=config/ks-core/charts/ks-crds/crds
   fi
 done

pkg/controller/core/installplan_controller.go

@@ -456,27 +456,41 @@ func (r *InstallPlanReconciler) loadChartData(ctx context.Context) ([]byte, stri
 	switch chartURL.Scheme {
 	case registry.OCIScheme:
 		opts := make([]getter.Option, 0)
-		opts = append(opts, getter.WithInsecureSkipVerifyTLS(true))
+		if extensionVersion.Spec.Repository != "" {
+			opts = append(opts, getter.WithInsecureSkipVerifyTLS(repo.Spec.Insecure))
+		}
 		if repo.Spec.BasicAuth != nil {
 			opts = append(opts, getter.WithBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password))
 		}
 		chartGetter, err = getter.NewOCIGetter(opts...)
+		if err != nil {
+			return nil, "", fmt.Errorf("failed to create chart getter: %v", err)
+		}
 	case "http", "https":
 		opts := make([]getter.Option, 0)
+		if chartURL.Scheme == "https" && extensionVersion.Spec.Repository != "" {
+			opts = append(opts, getter.WithInsecureSkipVerifyTLS(repo.Spec.Insecure))
+		}
+		if repo.Spec.CABundle != "" {
+			caFile, err := storeCAFile(repo.Spec.CABundle, repo.Name)
+			if err != nil {
+				return nil, "", fmt.Errorf("failed to store CABundle to local file: %s", err)
+			}
+			opts = append(opts, getter.WithTLSClientConfig("", "", caFile))
+		}
 		if chartURL.Scheme == "https" {
-			opts = append(opts, getter.WithInsecureSkipVerifyTLS(true))
+			opts = append(opts, getter.WithInsecureSkipVerifyTLS(repo.Spec.Insecure))
 		}
 		if repo.Spec.BasicAuth != nil {
 			opts = append(opts, getter.WithBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password))
 		}
 		chartGetter, err = getter.NewHTTPGetter(opts...)
-	default:
-		err = fmt.Errorf("unsupported scheme: %s", chartURL.Scheme)
-	}
-
 		if err != nil {
 			return nil, "", fmt.Errorf("failed to create chart getter: %v", err)
 		}
+	default:
+		return nil, "", fmt.Errorf("unsupported scheme: %s", chartURL.Scheme)
+	}
 
 	buffer, err := chartGetter.Get(chartURL.String())
 	if err != nil {

pkg/controller/core/repository_controller.go

@@ -11,7 +11,6 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"io"
 	"mime"
 	"net/http"
 	"net/url"
@@ -19,10 +18,6 @@ import (
 	"strings"
 	"time"
 
-	kscontroller "kubesphere.io/kubesphere/pkg/controller"
-
-	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
-
 	"github.com/go-logr/logr"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	appsv1 "k8s.io/api/apps/v1"
@@ -35,6 +30,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/retry"
 	"k8s.io/klog/v2"
+	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
 	corev1alpha1 "kubesphere.io/api/core/v1alpha1"
 	"kubesphere.io/utils/helm"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -43,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"kubesphere.io/kubesphere/pkg/constants"
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
 )
 
 const (
@@ -52,6 +49,8 @@ const (
 	defaultRequeueInterval      = 15 * time.Second
 	generateNameFormat          = "repository-%s"
 	extensionFileName           = "extension.yaml"
+	// caTemplate store repository.spec.caBound in local dir.
+	caTemplate = "{{ .TempDIR }}/repository/{{ .RepositoryName }}/ssl/ca.crt"
 )
 
 var extensionRepoConflict = fmt.Errorf("extension repo mismatch")
@@ -178,10 +177,10 @@ func (r *RepositoryReconciler) syncExtensionsFromURL(ctx context.Context, repo *
 	logger := klog.FromContext(ctx)
 	ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
 	defer cancel()
-	cred := helm.RepoCredential{}
-	if repo.Spec.BasicAuth != nil {
-		cred.Username = repo.Spec.BasicAuth.Username
-		cred.Password = repo.Spec.BasicAuth.Password
+
+	cred, err := newHelmCred(repo)
+	if err != nil {
+		return err
 	}
 	index, err := helm.LoadRepoIndex(ctx, repoURL, cred)
 	if err != nil {
@@ -216,7 +215,7 @@ func (r *RepositoryReconciler) syncExtensionsFromURL(ctx context.Context, repo *
 				}
 			}
 
-			extensionVersionSpec, err := r.loadExtensionVersionSpecFrom(ctx, chartURL, repo)
+			extensionVersionSpec, err := r.loadExtensionVersionSpecFrom(ctx, chartURL, repo, cred)
 			if err != nil {
 				return fmt.Errorf("failed to load extension version spec: %s", err)
 			}
@@ -435,37 +434,19 @@ func (r *RepositoryReconciler) deployRepository(ctx context.Context, repo *corev
 	return nil
 }
 
-func (r *RepositoryReconciler) loadExtensionVersionSpecFrom(ctx context.Context, chartURL string, repo *corev1alpha1.Repository) (*corev1alpha1.ExtensionVersionSpec, error) {
+func (r *RepositoryReconciler) loadExtensionVersionSpecFrom(ctx context.Context, chartURL string, repo *corev1alpha1.Repository, cred helm.RepoCredential) (*corev1alpha1.ExtensionVersionSpec, error) {
 	logger := klog.FromContext(ctx)
 	var result *corev1alpha1.ExtensionVersionSpec
 
 	err := retry.OnError(retry.DefaultRetry, func(err error) bool {
 		return true
 	}, func() error {
-		req, err := http.NewRequest(http.MethodGet, chartURL, nil)
+		data, err := helm.LoadData(ctx, chartURL, cred)
 		if err != nil {
 			return err
 		}
 
-		if repo.Spec.BasicAuth != nil {
-			req.SetBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password)
-		}
-
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-
-		if resp.StatusCode != http.StatusOK {
-			data, err := io.ReadAll(resp.Body)
-			if err != nil {
-				return err
-			}
-			return fmt.Errorf(string(data))
-		}
-
-		files, err := loader.LoadArchiveFiles(resp.Body)
+		files, err := loader.LoadArchiveFiles(data)
 		if err != nil {
 			return err
 		}

pkg/controller/core/util.go

@@ -7,15 +7,18 @@ package core
 
 import (
 	"bytes"
+	"encoding/base64"
 	goerrors "errors"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"sort"
 	"strings"
-
-	yaml3 "gopkg.in/yaml.v3"
+	"text/template"
 
 	"github.com/Masterminds/semver/v3"
+	yaml3 "gopkg.in/yaml.v3"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/storage/driver"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -23,9 +26,9 @@ import (
 	"k8s.io/klog/v2"
 	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
 	corev1alpha1 "kubesphere.io/api/core/v1alpha1"
+	"kubesphere.io/utils/helm"
 
 	"kubesphere.io/kubesphere/pkg/utils/hashutil"
-
 	"kubesphere.io/kubesphere/pkg/version"
 )
 
@@ -264,3 +267,58 @@ func configChanged(sub *corev1alpha1.InstallPlan, cluster string) bool {
 	}
 	return newConfigHash != oldConfigHash
 }
+
+// newHelmCred from Repository
+func newHelmCred(repo *corev1alpha1.Repository) (helm.RepoCredential, error) {
+	cred := helm.RepoCredential{
+		InsecureSkipTLSVerify: repo.Spec.Insecure,
+	}
+	if repo.Spec.CABundle != "" {
+		caFile, err := storeCAFile(repo.Spec.CABundle, repo.Name)
+		if err != nil {
+			return cred, err
+		}
+		cred.CAFile = caFile
+	}
+	if repo.Spec.BasicAuth != nil {
+		cred.Username = repo.Spec.BasicAuth.Username
+		cred.Password = repo.Spec.BasicAuth.Password
+	}
+	return cred, nil
+}
+
+// storeCAFile in local file from caTemplate.
+func storeCAFile(caBundle string, repoName string) (string, error) {
+	var buff = &bytes.Buffer{}
+	tmpl, err := template.New("repositoryCABundle").Parse(caTemplate)
+	if err != nil {
+		return "", err
+	}
+	if err := tmpl.Execute(buff, map[string]string{
+		"TempDIR":        os.TempDir(),
+		"RepositoryName": repoName,
+	}); err != nil {
+		return "", err
+	}
+	caFile := buff.String()
+	if _, err := os.Stat(filepath.Dir(caFile)); err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+
+		if err := os.MkdirAll(filepath.Dir(caFile), os.ModePerm); err != nil {
+			return "", err
+		}
+	}
+
+	data, err := base64.StdEncoding.DecodeString(caBundle)
+	if err != nil {
+		return "", err
+	}
+
+	if err := os.WriteFile(caFile, data, os.ModePerm); err != nil {
+		return "", err
+	}
+
+	return caFile, nil
+}

pkg/kapis/package/v1alpha1/handler.go

@@ -7,8 +7,12 @@ package v1alpha1
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"net/url"
+	"os"
+	"path/filepath"
+	"text/template"
 
 	"github.com/emicklei/go-restful/v3"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -22,6 +26,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/api"
 )
 
+var caTemplate = "{{ .TempDIR }}/repository/{{ .RepositoryName }}/ssl/ca.crt"
+
 type handler struct {
 	cache runtimeclient.Reader
 }
@@ -71,25 +77,42 @@ func (h *handler) ListFiles(request *restful.Request, response *restful.Response
 	switch chartURL.Scheme {
 	case registry.OCIScheme:
 		opts := make([]getter.Option, 0)
-		opts = append(opts, getter.WithInsecureSkipVerifyTLS(true))
+		if extensionVersion.Spec.Repository != "" {
+			opts = append(opts, getter.WithInsecureSkipVerifyTLS(repo.Spec.Insecure))
+		}
 		if repo.Spec.BasicAuth != nil {
 			opts = append(opts, getter.WithBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password))
 		}
 		chartGetter, err = getter.NewOCIGetter(opts...)
-	case "http", "https":
-		options := make([]getter.Option, 0)
-		if chartURL.Scheme == "https" {
-			options = append(options, getter.WithInsecureSkipVerifyTLS(true))
-		}
-		if repo.Spec.BasicAuth != nil {
-			options = append(options, getter.WithBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password))
-		}
-		chartGetter, err = getter.NewHTTPGetter(options...)
-	}
 		if err != nil {
 			api.HandleInternalError(response, request, fmt.Errorf("failed to create chart getter: %v", err))
 			return
 		}
+	case "http", "https":
+		opts := make([]getter.Option, 0)
+		if chartURL.Scheme == "https" && extensionVersion.Spec.Repository != "" {
+			opts = append(opts, getter.WithInsecureSkipVerifyTLS(repo.Spec.Insecure))
+		}
+		if repo.Spec.CABundle != "" {
+			caFile, err := storeCAFile(repo.Spec.CABundle, repo.Name)
+			if err != nil {
+				api.HandleInternalError(response, request, fmt.Errorf("failed to store CABundle to local file: %s", err))
+				return
+			}
+			opts = append(opts, getter.WithTLSClientConfig("", "", caFile))
+		}
+		if repo.Spec.BasicAuth != nil {
+			opts = append(opts, getter.WithBasicAuth(repo.Spec.BasicAuth.Username, repo.Spec.BasicAuth.Password))
+		}
+		chartGetter, err = getter.NewHTTPGetter(opts...)
+		if err != nil {
+			api.HandleInternalError(response, request, fmt.Errorf("failed to create chart getter: %v", err))
+			return
+		}
+	default:
+		api.HandleInternalError(response, request, fmt.Errorf("cannot support chartURL %s, it's schame should be: oci,http,https", extensionVersion.Spec.ChartURL))
+		return
+	}
 
 	data, err := chartGetter.Get(extensionVersion.Spec.ChartURL)
 	if err != nil {
@@ -105,3 +128,39 @@ func (h *handler) ListFiles(request *restful.Request, response *restful.Response
 
 	_ = response.WriteEntity(files)
 }
+
+// storeCAFile in local file from caTemplate.
+func storeCAFile(caBundle string, repoName string) (string, error) {
+	var buff = &bytes.Buffer{}
+	tmpl, err := template.New("repositoryCABundle").Parse(caTemplate)
+	if err != nil {
+		return "", err
+	}
+	if err := tmpl.Execute(buff, map[string]string{
+		"TempDIR":        os.TempDir(),
+		"RepositoryName": repoName,
+	}); err != nil {
+		return "", err
+	}
+	caFile := buff.String()
+	if _, err := os.Stat(filepath.Dir(caFile)); err != nil {
+		if !os.IsNotExist(err) {
+			return "", err
+		}
+
+		if err := os.MkdirAll(filepath.Dir(caFile), os.ModePerm); err != nil {
+			return "", err
+		}
+	}
+
+	data, err := base64.StdEncoding.DecodeString(caBundle)
+	if err != nil {
+		return "", err
+	}
+
+	if err := os.WriteFile(caFile, data, os.ModePerm); err != nil {
+		return "", err
+	}
+
+	return caFile, nil
+}

staging/src/kubesphere.io/api/core/v1alpha1/repository_types.go

@@ -18,14 +18,17 @@ type BasicAuth struct {
 }
 
 type RepositorySpec struct {
+	// DEPRECATED: the field will remove in future versions, please use url.
 	Image          string          `json:"image,omitempty"`
 	URL            string          `json:"url,omitempty"`
 	Description    string          `json:"description,omitempty"`
 	BasicAuth      *BasicAuth      `json:"basicAuth,omitempty"`
 	UpdateStrategy *UpdateStrategy `json:"updateStrategy,omitempty"`
-	// +optional The caBundle (base64 string) is used in helmExecutor to verify the helm server.
-	// if the caBundle is empty, use --insecure-skip-tls-verify.
+	// The caBundle (base64 string) is used in helmExecutor to verify the helm server.
+	// +optional
 	CABundle string `json:"caBundle,omitempty"`
+	// --insecure-skip-tls-verify. default false
+	Insecure bool `json:"insecure,omitempty"`
 }
 
 type RepositoryStatus struct {

staging/src/kubesphere.io/utils/helm/repo_index.go

@@ -24,7 +24,7 @@ func LoadRepoIndex(ctx context.Context, u string, cred RepoCredential) (*helmrep
 		u = fmt.Sprintf("%s%s", u, IndexYaml)
 	}
 
-	resp, err := loadData(ctx, u, cred)
+	resp, err := LoadData(ctx, u, cred)
 	if err != nil {
 		return nil, err
 	}
@@ -52,7 +52,7 @@ func loadIndex(data []byte) (*helmrepo.IndexFile, error) {
 	return i, nil
 }
 
-func loadData(ctx context.Context, u string, cred RepoCredential) (*bytes.Buffer, error) {
+func LoadData(ctx context.Context, u string, cred RepoCredential) (*bytes.Buffer, error) {
 	parsedURL, err := url.Parse(u)
 	if err != nil {
 		return nil, err
@@ -81,18 +81,11 @@ func loadData(ctx context.Context, u string, cred RepoCredential) (*bytes.Buffer
 
 		resp = bytes.NewBuffer(data)
 	} else {
-		skipTLS := true
-		if cred.InsecureSkipTLSVerify != nil && !*cred.InsecureSkipTLSVerify {
-			skipTLS = false
-		}
-
-		indexURL := parsedURL.String()
 		// TODO add user-agent
 		g, _ := getter.NewHTTPGetter()
-		resp, err = g.Get(indexURL,
+		resp, err = g.Get(parsedURL.String(),
 			getter.WithTimeout(5*time.Minute),
-			getter.WithURL(u),
-			getter.WithInsecureSkipVerifyTLS(skipTLS),
+			getter.WithInsecureSkipVerifyTLS(cred.InsecureSkipTLSVerify),
 			getter.WithTLSClientConfig(cred.CertFile, cred.KeyFile, cred.CAFile),
 			getter.WithBasicAuth(cred.Username, cred.Password),
 		)
@@ -135,7 +128,7 @@ type RepoCredential struct {
 	// verify certificates of HTTPS-enabled servers using this CA bundle
 	CAFile string `json:"caFile,omitempty"`
 	// skip tls certificate checks for the repository, default is ture
-	InsecureSkipTLSVerify *bool `json:"insecureSkipTLSVerify,omitempty"`
+	InsecureSkipTLSVerify bool `json:"insecureSkipTLSVerify,omitempty"`
 
 	S3Config `json:",inline"`
 }