diff --git a/build/ks-apiserver/Dockerfile b/build/ks-apiserver/Dockerfile index be3530ba4..19236fae4 100644 --- a/build/ks-apiserver/Dockerfile +++ b/build/ks-apiserver/Dockerfile @@ -1,12 +1,17 @@ # Copyright 2020 The KubeSphere Authors. All rights reserved. # Use of this source code is governed by an Apache license # that can be found in the LICENSE file. -FROM alpine/helm:3.4.2 as helm-base - FROM alpine:3.11 +ARG HELM_VERSION=v3.5.2 + RUN apk add --no-cache ca-certificates -COPY --from=helm-base /usr/bin/helm /usr/bin/helm +# install helm +RUN wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + mv linux-amd64/helm /usr/bin/ && \ + rm -rf linux-amd64 # To speed up building process, we copy binary directly from make # result instead of building it again, so make sure you run the # following command first before building docker image diff --git a/build/ks-controller-manager/Dockerfile b/build/ks-controller-manager/Dockerfile index e0f2e853e..dc0049d91 100644 --- a/build/ks-controller-manager/Dockerfile +++ b/build/ks-controller-manager/Dockerfile @@ -1,12 +1,24 @@ # Copyright 2020 The KubeSphere Authors. All rights reserved. # Use of this source code is governed by an Apache license # that can be found in the LICENSE file. -FROM alpine/helm:3.4.2 as helm-base - FROM alpine:3.11 +ARG HELM_VERSION=v3.5.2 +ARG KUSTOMIZE_VERSION=v4.0.5 + RUN apk add --no-cache ca-certificates -COPY --from=helm-base /usr/bin/helm /usr/bin/helm +# install helm +RUN wget https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + tar xvf helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + rm helm-${HELM_VERSION}-linux-amd64.tar.gz && \ + mv linux-amd64/helm /usr/bin/ && \ + rm -rf linux-amd64 +# install kustomize +RUN wget https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + tar xvf kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + rm kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz && \ + mv kustomize /usr/bin + COPY /bin/cmd/controller-manager /usr/local/bin/ EXPOSE 8443 8080 diff --git a/go.mod b/go.mod index 302f8574f..b0b00721c 100644 --- a/go.mod +++ b/go.mod @@ -109,6 +109,7 @@ require ( sigs.k8s.io/controller-runtime v0.6.4 sigs.k8s.io/controller-tools v0.4.0 sigs.k8s.io/kubefed v0.4.0 + sigs.k8s.io/kustomize v2.0.3+incompatible sigs.k8s.io/yaml v1.2.0 ) diff --git a/pkg/controller/openpitrix/helmrelease/helm_release_controller.go b/pkg/controller/openpitrix/helmrelease/helm_release_controller.go index f1febe8ab..6e42ab2f7 100644 --- a/pkg/controller/openpitrix/helmrelease/helm_release_controller.go +++ b/pkg/controller/openpitrix/helmrelease/helm_release_controller.go @@ -31,6 +31,7 @@ import ( "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/application/v1alpha1" "kubesphere.io/kubesphere/pkg/client/informers/externalversions" + "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmrepoindex" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix/helmwrapper" "kubesphere.io/kubesphere/pkg/simple/client/s3" @@ -309,7 +310,10 @@ func (r *ReconcileHelmRelease) createOrUpgradeHelmRelease(rls *v1alpha1.HelmRele } // If clusterConfig is empty, this application will be installed in current host. - hw := helmwrapper.NewHelmWrapper(clusterConfig, rls.GetRlsNamespace(), rls.Spec.Name, helmwrapper.SetMock(r.helmMock)) + hw := helmwrapper.NewHelmWrapper(clusterConfig, rls.GetRlsNamespace(), rls.Spec.Name, + // We just add kubesphere.io/creator annotation now. + helmwrapper.SetAnnotations(map[string]string{constants.CreatorAnnotationKey: rls.GetCreator()}), + helmwrapper.SetMock(r.helmMock)) var res helmwrapper.HelmRes if upgrade { res, err = hw.Upgrade(rls.Spec.ChartName, string(chartData), string(rls.Spec.Values)) diff --git a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper.go b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper.go index 86d1d3c5d..d91f83b05 100644 --- a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper.go +++ b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper.go @@ -20,20 +20,34 @@ import ( "bytes" "encoding/json" "fmt" + "gopkg.in/yaml.v3" "k8s.io/klog" kpath "k8s.io/utils/path" "kubesphere.io/kubesphere/pkg/utils/idutils" "os" "os/exec" - "path" "path/filepath" + "sigs.k8s.io/kustomize/pkg/types" "strings" "time" ) +const ( + workspaceBase = "/tmp/helm-operator" +) + var ( UninstallNotFoundFormat = "Error: uninstall: Release not loaded: %s: release: not found" StatusNotFoundFormat = "Error: release: not found" + + kustomizationFile = "kustomization.yaml" + postRenderExecFile = "helm-post-render.sh" + // kustomize cannot read stdio now, so we save helm stdout to file, then kustomize reads that file and build the resources + kustomizeBuild = `#!/bin/sh +# save helm stdout to file, then kustomize read this file +cat > ./.local-helm-output.yaml +kustomize build +` ) type HelmRes struct { @@ -131,9 +145,9 @@ func (c *helmWrapper) Status() (status *releaseStatus, err error) { func (c *helmWrapper) Workspace() string { if c.workspaceSuffix == "" { - return path.Join(c.base, fmt.Sprintf("%s_%s", c.Namespace, c.ReleaseName)) + return filepath.Join(c.base, fmt.Sprintf("%s_%s", c.Namespace, c.ReleaseName)) } else { - return path.Join(c.base, fmt.Sprintf("%s_%s_%s", c.Namespace, c.ReleaseName, c.workspaceSuffix)) + return filepath.Join(c.base, fmt.Sprintf("%s_%s_%s", c.Namespace, c.ReleaseName, c.workspaceSuffix)) } } @@ -145,6 +159,11 @@ type helmWrapper struct { ReleaseName string ChartName string + // add labels to helm chart + labels map[string]string + // add annotations to helm chart + annotations map[string]string + // helm cmd path cmdPath string // base should be /dev/shm on linux @@ -158,11 +177,16 @@ func (c *helmWrapper) kubeConfigPath() string { if len(c.Kubeconfig) == 0 { return "" } - return path.Join(c.Workspace(), "kube.config") + return filepath.Join(c.Workspace(), "kube.config") +} + +// The dir where chart saved +func (c *helmWrapper) chartDir() string { + return filepath.Join(c.Workspace(), "chart") } func (c *helmWrapper) chartPath() string { - return filepath.Join(c.Workspace(), fmt.Sprintf("%s.tgz", c.ChartName)) + return filepath.Join(c.chartDir(), fmt.Sprintf("%s.tgz", c.ChartName)) } func (c *helmWrapper) cleanup() { @@ -185,6 +209,13 @@ func SetDryRun(dryRun bool) Option { } } +// extra annotations added to all resources in chart +func SetAnnotations(annotations map[string]string) Option { + return func(wrapper *helmWrapper) { + wrapper.annotations = annotations + } +} + func SetMock(mock bool) Option { return func(wrapper *helmWrapper) { wrapper.mock = mock @@ -208,6 +239,43 @@ func NewHelmWrapper(kubeconfig, ns, rls string, options ...Option) *helmWrapper return c } +func (c *helmWrapper) setupPostRenderEnvironment() error { + if len(c.labels) == 0 && len(c.annotations) == 0 { + return nil + } + + // build the executable file + postRender, err := os.OpenFile(filepath.Join(c.Workspace(), postRenderExecFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return err + } + _, err = postRender.WriteString(kustomizeBuild) + if err != nil { + return err + } + postRender.Close() + + // create kustomization.yaml + kustomization, err := os.OpenFile(filepath.Join(c.Workspace(), kustomizationFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + return err + } + + kustomizationConfig := types.Kustomization{ + Resources: []string{"./.local-helm-output.yaml"}, + CommonAnnotations: c.annotations, // add extra annotations to output + CommonLabels: c.labels, // add extra labels to output + } + + err = yaml.NewEncoder(kustomization).Encode(kustomizationConfig) + if err != nil { + return err + } + kustomization.Close() + + return nil +} + // ensureWorkspace check whether workspace exists or not. // If not exists, create workspace dir. func (c *helmWrapper) ensureWorkspace() error { @@ -222,6 +290,12 @@ func (c *helmWrapper) ensureWorkspace() error { } } + err := os.MkdirAll(c.chartDir(), os.ModeDir|os.ModePerm) + if err != nil { + klog.Errorf("mkdir %s failed, error: %s", c.chartDir(), err) + return err + } + if len(c.Kubeconfig) > 0 { kubeFile, err := os.OpenFile(c.kubeConfigPath(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) if err != nil { @@ -243,7 +317,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error { c.ChartName = chartName // write chart - f, err := os.Create(path.Join(c.chartPath())) + f, err := os.Create(c.chartPath()) if err != nil { return err @@ -257,7 +331,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error { f.Close() // write values - f, err = os.Create(path.Join(c.Workspace(), "values.yaml")) + f, err = os.Create(filepath.Join(c.Workspace(), "values.yaml")) if err != nil { return err } @@ -271,6 +345,7 @@ func (c *helmWrapper) createChart(chartName, chartData, values string) error { return nil } +// helm uninstall func (c *helmWrapper) Uninstall() (res HelmRes, err error) { start := time.Now() defer func() { @@ -332,6 +407,7 @@ func (c *helmWrapper) Uninstall() (res HelmRes, err error) { return } +// helm upgrade func (c *helmWrapper) Upgrade(chartName, chartData, values string) (res HelmRes, err error) { // TODO: check release status first if true { @@ -342,6 +418,7 @@ func (c *helmWrapper) Upgrade(chartName, chartData, values string) (res HelmRes, } } +// helm install func (c *helmWrapper) Install(chartName, chartData, values string) (res HelmRes, err error) { return c.install(chartName, chartData, values, false) } @@ -359,6 +436,11 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool) } defer c.cleanup() + err = c.setupPostRenderEnvironment() + if err != nil { + return + } + if err = c.createChart(chartName, chartData, values); err != nil { return } @@ -393,7 +475,7 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool) cmd.Args = append(cmd.Args, c.ReleaseName, c.chartPath(), "--namespace", c.Namespace) if len(values) > 0 { - cmd.Args = append(cmd.Args, "--values", path.Join(c.Workspace(), "values.yaml")) + cmd.Args = append(cmd.Args, "--values", filepath.Join(c.Workspace(), "values.yaml")) } if c.dryRun { @@ -404,6 +486,11 @@ func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool) cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath()) } + // Post render, add annotations or labels to resources + if len(c.labels) > 0 || len(c.annotations) > 0 { + cmd.Args = append(cmd.Args, "--post-renderer", filepath.Join(c.Workspace(), postRenderExecFile)) + } + if klog.V(8) { // output debug info cmd.Args = append(cmd.Args, "--debug") diff --git a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_darwin.go b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_darwin.go index e3f988dae..0016f5290 100644 --- a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_darwin.go +++ b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_darwin.go @@ -17,6 +17,5 @@ limitations under the License. package helmwrapper const ( - workspaceBase = "/tmp/helm-operator" - helmPath = "/usr/local/bin/helm" + helmPath = "/usr/local/bin/helm" ) diff --git a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_linux.go b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_linux.go index 0688ac5f9..2ad48c7ef 100644 --- a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_linux.go +++ b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_linux.go @@ -17,6 +17,5 @@ limitations under the License. package helmwrapper const ( - workspaceBase = "/dev/shm/helm-operator" - helmPath = "/usr/bin/helm" + helmPath = "/usr/bin/helm" ) diff --git a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_test.go b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_test.go index 3f953a2c6..c431bf6b6 100644 --- a/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_test.go +++ b/pkg/simple/client/openpitrix/helmwrapper/helm_wrapper_test.go @@ -18,12 +18,15 @@ package helmwrapper import ( "fmt" + "kubesphere.io/kubesphere/pkg/constants" "os" "testing" ) func TestHelmInstall(t *testing.T) { - wr := NewHelmWrapper("", "dummy", "dummy", SetMock(true)) + wr := NewHelmWrapper("", "dummy", "dummy", + SetAnnotations(map[string]string{constants.CreatorAnnotationKey: "1234"}), + SetMock(true)) res, err := wr.Install("dummy-chart", "", "dummy-value") if err != nil {