606 lines
14 KiB
Go
606 lines
14 KiB
Go
/*
|
|
Copyright 2020 The KubeSphere Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package helmwrapper
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"helm.sh/helm/v3/pkg/cli"
|
|
|
|
"helm.sh/helm/v3/pkg/kube"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
helmrelease "helm.sh/helm/v3/pkg/release"
|
|
"k8s.io/klog"
|
|
kpath "k8s.io/utils/path"
|
|
"sigs.k8s.io/kustomize/pkg/types"
|
|
|
|
"kubesphere.io/kubesphere/pkg/server/errors"
|
|
"kubesphere.io/kubesphere/pkg/utils/idutils"
|
|
)
|
|
|
|
const (
|
|
workspaceBase = "/tmp/helm-operator"
|
|
)
|
|
|
|
var (
|
|
ErrorTimedOutToWaitResource = errors.New("timed out waiting for resources to be ready")
|
|
|
|
UninstallNotFoundFormat = "Error: uninstall: Release not loaded: %s: release: not found"
|
|
StatusNotFoundFormat = "Error: release: not found"
|
|
releaseExists = "release exists"
|
|
|
|
kustomizationFile = "kustomization.yaml"
|
|
postRenderExecFile = "helm-post-render.sh"
|
|
// kustomize cannot read stdio now, so we save helm stdout to file, then kustomize reads that file and build the resources
|
|
kustomizeBuild = `#!/bin/sh
|
|
# save helm stdout to file, then kustomize read this file
|
|
cat > ./.local-helm-output.yaml
|
|
kustomize build
|
|
`
|
|
)
|
|
|
|
type HelmRes struct {
|
|
Message string
|
|
}
|
|
|
|
var _ HelmWrapper = &helmWrapper{}
|
|
|
|
type HelmWrapper interface {
|
|
Install(chartName, chartData, values string) error
|
|
// upgrade a release
|
|
Upgrade(chartName, chartData, values string) error
|
|
Uninstall() error
|
|
// Get manifests
|
|
Manifest() (string, error)
|
|
|
|
// IsReleaseReady check helm release is ready or not
|
|
IsReleaseReady(timeout time.Duration) (bool, error)
|
|
}
|
|
|
|
// IsReleaseReady check helm releases is ready or not
|
|
// If the return values is (true, nil), then the resources are ready
|
|
func (c *helmWrapper) IsReleaseReady(waitTime time.Duration) (bool, error) {
|
|
|
|
// Get the manifest to build resources
|
|
manifest, err := c.Manifest()
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
var client *kube.Client
|
|
if c.Kubeconfig == "" {
|
|
client = kube.New(nil)
|
|
} else {
|
|
helmSettings := cli.New()
|
|
helmSettings.KubeConfig = c.kubeConfigPath()
|
|
client = kube.New(helmSettings.RESTClientGetter())
|
|
}
|
|
|
|
client.Namespace = c.Namespace
|
|
resources, err := client.Build(bytes.NewBufferString(manifest), true)
|
|
|
|
err = client.Wait(resources, waitTime)
|
|
|
|
if err == nil {
|
|
return true, nil
|
|
}
|
|
|
|
if err == wait.ErrWaitTimeout {
|
|
return false, ErrorTimedOutToWaitResource
|
|
}
|
|
|
|
return false, err
|
|
}
|
|
|
|
func (c *helmWrapper) Status() (status *helmrelease.Release, err error) {
|
|
if err = c.ensureWorkspace(); err != nil {
|
|
return nil, err
|
|
}
|
|
defer c.cleanup()
|
|
|
|
stdout := &bytes.Buffer{}
|
|
stderr := &bytes.Buffer{}
|
|
cmd := exec.Cmd{
|
|
Path: c.cmdPath,
|
|
Dir: c.Workspace(),
|
|
Args: []string{
|
|
c.cmdPath,
|
|
"status",
|
|
fmt.Sprintf("%s", c.ReleaseName),
|
|
"--namespace",
|
|
c.Namespace,
|
|
"--output",
|
|
"json",
|
|
},
|
|
Stderr: stderr,
|
|
Stdout: stdout,
|
|
}
|
|
|
|
if c.kubeConfigPath() != "" {
|
|
cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath())
|
|
}
|
|
|
|
err = cmd.Run()
|
|
|
|
if err != nil {
|
|
helmErr := strings.TrimSpace(stderr.String())
|
|
if helmErr == StatusNotFoundFormat {
|
|
klog.V(2).Infof("namespace: %s, name: %s, run command failed, stderr: %s, error: %v", c.Namespace, c.ReleaseName, stderr, err)
|
|
return nil, errors.New(helmErr)
|
|
}
|
|
klog.Errorf("namespace: %s, name: %s, run command failed, stderr: %s, error: %v", c.Namespace, c.ReleaseName, stderr, err)
|
|
return
|
|
} else {
|
|
klog.V(2).Infof("namespace: %s, name: %s, run command success", c.Namespace, c.ReleaseName)
|
|
klog.V(8).Infof("namespace: %s, name: %s, run command success, stdout: %s", c.Namespace, c.ReleaseName, stdout)
|
|
}
|
|
|
|
status = &helmrelease.Release{}
|
|
err = json.Unmarshal(stdout.Bytes(), status)
|
|
if err != nil {
|
|
klog.Errorf("namespace: %s, name: %s, json unmarshal failed, error: %s", c.Namespace, c.ReleaseName, err)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (c *helmWrapper) Workspace() string {
|
|
if c.workspaceSuffix == "" {
|
|
return filepath.Join(c.base, fmt.Sprintf("%s_%s", c.Namespace, c.ReleaseName))
|
|
} else {
|
|
return filepath.Join(c.base, fmt.Sprintf("%s_%s_%s", c.Namespace, c.ReleaseName, c.workspaceSuffix))
|
|
}
|
|
}
|
|
|
|
type helmWrapper struct {
|
|
// KubeConfig string
|
|
Kubeconfig string
|
|
Namespace string
|
|
// helm release name
|
|
ReleaseName string
|
|
ChartName string
|
|
|
|
// add labels to helm chart
|
|
labels map[string]string
|
|
// add annotations to helm chart
|
|
annotations map[string]string
|
|
|
|
// helm cmd path
|
|
cmdPath string
|
|
base string
|
|
workspaceSuffix string
|
|
dryRun bool
|
|
mock bool
|
|
}
|
|
|
|
func (c *helmWrapper) kubeConfigPath() string {
|
|
if len(c.Kubeconfig) == 0 {
|
|
return ""
|
|
}
|
|
return filepath.Join(c.Workspace(), "kube.config")
|
|
}
|
|
|
|
// The dir where chart saved
|
|
func (c *helmWrapper) chartDir() string {
|
|
return filepath.Join(c.Workspace(), "chart")
|
|
}
|
|
|
|
func (c *helmWrapper) chartPath() string {
|
|
return filepath.Join(c.chartDir(), fmt.Sprintf("%s.tgz", c.ChartName))
|
|
}
|
|
|
|
func (c *helmWrapper) cleanup() {
|
|
if err := os.RemoveAll(c.Workspace()); err != nil {
|
|
klog.Errorf("remove dir %s failed, error: %s", c.Workspace(), err)
|
|
}
|
|
}
|
|
|
|
func (c *helmWrapper) Set(options ...Option) {
|
|
for _, option := range options {
|
|
option(c)
|
|
}
|
|
}
|
|
|
|
type Option func(*helmWrapper)
|
|
|
|
func SetDryRun(dryRun bool) Option {
|
|
return func(wrapper *helmWrapper) {
|
|
wrapper.dryRun = dryRun
|
|
}
|
|
}
|
|
|
|
// extra annotations added to all resources in chart
|
|
func SetAnnotations(annotations map[string]string) Option {
|
|
return func(wrapper *helmWrapper) {
|
|
wrapper.annotations = annotations
|
|
}
|
|
}
|
|
|
|
// extra labels added to all resources in chart
|
|
func SetLabels(labels map[string]string) Option {
|
|
return func(wrapper *helmWrapper) {
|
|
wrapper.labels = labels
|
|
}
|
|
}
|
|
|
|
func SetMock(mock bool) Option {
|
|
return func(wrapper *helmWrapper) {
|
|
wrapper.mock = mock
|
|
}
|
|
}
|
|
|
|
func NewHelmWrapper(kubeconfig, ns, rls string, options ...Option) *helmWrapper {
|
|
c := &helmWrapper{
|
|
Kubeconfig: kubeconfig,
|
|
Namespace: ns,
|
|
ReleaseName: rls,
|
|
base: workspaceBase,
|
|
cmdPath: helmPath,
|
|
workspaceSuffix: idutils.GetUuid36(""),
|
|
}
|
|
|
|
for _, option := range options {
|
|
option(c)
|
|
}
|
|
|
|
return c
|
|
}
|
|
|
|
func (c *helmWrapper) setupPostRenderEnvironment() error {
|
|
if len(c.labels) == 0 && len(c.annotations) == 0 {
|
|
return nil
|
|
}
|
|
|
|
// build the executable file
|
|
postRender, err := os.OpenFile(filepath.Join(c.Workspace(), postRenderExecFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = postRender.WriteString(kustomizeBuild)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
postRender.Close()
|
|
|
|
// create kustomization.yaml
|
|
kustomization, err := os.OpenFile(filepath.Join(c.Workspace(), kustomizationFile), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
kustomizationConfig := types.Kustomization{
|
|
Resources: []string{"./.local-helm-output.yaml"},
|
|
CommonAnnotations: c.annotations, // add extra annotations to output
|
|
CommonLabels: c.labels, // add extra labels to output
|
|
}
|
|
|
|
err = yaml.NewEncoder(kustomization).Encode(kustomizationConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kustomization.Close()
|
|
|
|
return nil
|
|
}
|
|
|
|
// ensureWorkspace check whether workspace exists or not.
|
|
// If not exists, create workspace dir.
|
|
func (c *helmWrapper) ensureWorkspace() error {
|
|
if exists, err := kpath.Exists(kpath.CheckFollowSymlink, c.Workspace()); err != nil {
|
|
klog.Errorf("check dir %s failed, error: %s", c.Workspace(), err)
|
|
return err
|
|
} else if !exists {
|
|
err = os.MkdirAll(c.Workspace(), os.ModeDir|os.ModePerm)
|
|
if err != nil {
|
|
klog.Errorf("mkdir %s failed, error: %s", c.Workspace(), err)
|
|
return err
|
|
}
|
|
}
|
|
|
|
err := os.MkdirAll(c.chartDir(), os.ModeDir|os.ModePerm)
|
|
if err != nil {
|
|
klog.Errorf("mkdir %s failed, error: %s", c.chartDir(), err)
|
|
return err
|
|
}
|
|
|
|
if len(c.Kubeconfig) > 0 {
|
|
kubeFile, err := os.OpenFile(c.kubeConfigPath(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = kubeFile.WriteString(c.Kubeconfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kubeFile.Close()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// create chart dir in workspace
|
|
// write values.yaml into workspace
|
|
func (c *helmWrapper) createChart(chartName, chartData, values string) error {
|
|
c.ChartName = chartName
|
|
|
|
// write chart
|
|
f, err := os.Create(c.chartPath())
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = f.Write([]byte(chartData))
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
f.Close()
|
|
|
|
// write values
|
|
f, err = os.Create(filepath.Join(c.Workspace(), "values.yaml"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = f.WriteString(values)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
f.Close()
|
|
return nil
|
|
}
|
|
|
|
// helm uninstall
|
|
func (c *helmWrapper) Uninstall() (err error) {
|
|
start := time.Now()
|
|
defer func() {
|
|
klog.V(2).Infof("run command end, namespace: %s, name: %s elapsed: %v", c.Namespace, c.ReleaseName, time.Now().Sub(start))
|
|
}()
|
|
|
|
if err = c.ensureWorkspace(); err != nil {
|
|
return
|
|
}
|
|
defer c.cleanup()
|
|
|
|
stderr := &bytes.Buffer{}
|
|
stdout := &bytes.Buffer{}
|
|
cmd := exec.Cmd{
|
|
Path: c.cmdPath,
|
|
Dir: c.Workspace(),
|
|
Stdout: stdout,
|
|
Stderr: stderr,
|
|
}
|
|
|
|
cmd.Args = make([]string, 0, 10)
|
|
|
|
// only for mock
|
|
if c.mock {
|
|
cmd.Path = os.Args[0]
|
|
cmd.Args = []string{os.Args[0], "-test.run=TestHelperProcess", "--"}
|
|
cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
|
|
}
|
|
|
|
cmd.Args = append(cmd.Args, c.cmdPath,
|
|
"uninstall",
|
|
c.ReleaseName,
|
|
"--namespace",
|
|
c.Namespace)
|
|
|
|
if c.dryRun {
|
|
cmd.Args = append(cmd.Args, "--dry-run")
|
|
}
|
|
|
|
if c.kubeConfigPath() != "" {
|
|
cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath())
|
|
}
|
|
|
|
klog.V(4).Infof("run command: %s", cmd.String())
|
|
err = cmd.Run()
|
|
|
|
if err != nil {
|
|
eMsg := strings.TrimSpace(stderr.String())
|
|
// release does not exist. It's ok.
|
|
if fmt.Sprintf(UninstallNotFoundFormat, c.ReleaseName) == eMsg {
|
|
return nil
|
|
}
|
|
klog.Errorf("run command failed, stderr: %s, error: %v", eMsg, err)
|
|
return errors.New("%s", eMsg)
|
|
} else {
|
|
klog.V(2).Infof("namespace: %s, name: %s, run command success", c.Namespace, c.ReleaseName)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// helm upgrade
|
|
func (c *helmWrapper) Upgrade(chartName, chartData, values string) (err error) {
|
|
sts, err := c.Status()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if sts.Info.Status == "deployed" {
|
|
return c.install(chartName, chartData, values, true)
|
|
} else {
|
|
err = errors.New("cannot upgrade release %s/%s, current state is %s", c.Namespace, c.ReleaseName, sts.Info.Status)
|
|
return err
|
|
}
|
|
}
|
|
|
|
// helm install
|
|
func (c *helmWrapper) Install(chartName, chartData, values string) (err error) {
|
|
sts, err := c.Status()
|
|
if err == nil {
|
|
// helm release has been installed
|
|
if sts.Info != nil && sts.Info.Status == "deployed" {
|
|
return nil
|
|
}
|
|
return errors.New(releaseExists)
|
|
} else {
|
|
if err.Error() == StatusNotFoundFormat {
|
|
// continue to install
|
|
return c.install(chartName, chartData, values, false)
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
func (c *helmWrapper) install(chartName, chartData, values string, upgrade bool) (err error) {
|
|
if klog.V(2) {
|
|
start := time.Now()
|
|
defer func() {
|
|
klog.V(2).Infof("run command end, namespace: %s, name: %s elapsed: %v", c.Namespace, c.ReleaseName, time.Now().Sub(start))
|
|
}()
|
|
}
|
|
|
|
if err = c.ensureWorkspace(); err != nil {
|
|
return
|
|
}
|
|
defer c.cleanup()
|
|
|
|
err = c.setupPostRenderEnvironment()
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
if err = c.createChart(chartName, chartData, values); err != nil {
|
|
return
|
|
}
|
|
klog.V(8).Infof("namespace: %s, name: %s, chart values: %s", c.Namespace, c.ReleaseName, values)
|
|
|
|
stdout := &bytes.Buffer{}
|
|
stderr := &bytes.Buffer{}
|
|
|
|
cmd := exec.Cmd{
|
|
Path: c.cmdPath,
|
|
Dir: c.Workspace(),
|
|
Stdout: stdout,
|
|
Stderr: stderr,
|
|
}
|
|
|
|
cmd.Args = make([]string, 0, 10)
|
|
|
|
// only for mock
|
|
if c.mock {
|
|
cmd.Path = os.Args[0]
|
|
cmd.Args = []string{os.Args[0], "-test.run=TestHelperProcess", "--"}
|
|
cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"}
|
|
}
|
|
|
|
cmd.Args = append(cmd.Args, c.cmdPath)
|
|
if upgrade {
|
|
cmd.Args = append(cmd.Args, "upgrade")
|
|
} else {
|
|
cmd.Args = append(cmd.Args, "install")
|
|
}
|
|
|
|
cmd.Args = append(cmd.Args, c.ReleaseName, c.chartPath(), "--namespace", c.Namespace)
|
|
|
|
if len(values) > 0 {
|
|
cmd.Args = append(cmd.Args, "--values", filepath.Join(c.Workspace(), "values.yaml"))
|
|
}
|
|
|
|
if c.dryRun {
|
|
cmd.Args = append(cmd.Args, "--dry-run")
|
|
}
|
|
|
|
if c.kubeConfigPath() != "" {
|
|
cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath())
|
|
}
|
|
|
|
// Post render, add annotations or labels to resources
|
|
if len(c.labels) > 0 || len(c.annotations) > 0 {
|
|
cmd.Args = append(cmd.Args, "--post-renderer", filepath.Join(c.Workspace(), postRenderExecFile))
|
|
}
|
|
|
|
if klog.V(8) {
|
|
// output debug info
|
|
cmd.Args = append(cmd.Args, "--debug")
|
|
}
|
|
|
|
klog.V(4).Infof("run command: %s", cmd.String())
|
|
err = cmd.Run()
|
|
|
|
if err != nil {
|
|
klog.Errorf("namespace: %s, name: %s, run command: %s failed, stderr: %s, error: %v", c.Namespace, c.ReleaseName, cmd.String(), stderr, err)
|
|
// return the error of helm install
|
|
return errors.New("%s", stderr.String())
|
|
} else {
|
|
klog.V(2).Infof("namespace: %s, name: %s, run command success", c.Namespace, c.ReleaseName)
|
|
klog.V(8).Infof("namespace: %s, name: %s, run command success, stdout: %s", c.Namespace, c.ReleaseName, stdout)
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
func (c *helmWrapper) Manifest() (manifest string, err error) {
|
|
if err = c.ensureWorkspace(); err != nil {
|
|
return "", err
|
|
}
|
|
defer c.cleanup()
|
|
|
|
stdout := &bytes.Buffer{}
|
|
stderr := &bytes.Buffer{}
|
|
cmd := exec.Cmd{
|
|
Path: c.cmdPath,
|
|
Dir: c.Workspace(),
|
|
Args: []string{
|
|
c.cmdPath,
|
|
"get",
|
|
"manifest",
|
|
c.ReleaseName,
|
|
"--namespace",
|
|
c.Namespace,
|
|
},
|
|
Stderr: stderr,
|
|
Stdout: stdout,
|
|
}
|
|
|
|
if c.kubeConfigPath() != "" {
|
|
cmd.Args = append(cmd.Args, "--kubeconfig", c.kubeConfigPath())
|
|
}
|
|
|
|
if klog.V(8) {
|
|
// output debug info
|
|
cmd.Args = append(cmd.Args, "--debug")
|
|
}
|
|
|
|
klog.V(4).Infof("run command: %s", cmd.String())
|
|
err = cmd.Run()
|
|
|
|
if err != nil {
|
|
klog.Errorf("namespace: %s, name: %s, run command failed, stderr: %s, error: %v", c.Namespace, c.ReleaseName, stderr, err)
|
|
return "", err
|
|
} else {
|
|
klog.V(2).Infof("namespace: %s, name: %s, run command success", c.Namespace, c.ReleaseName)
|
|
klog.V(8).Infof("namespace: %s, name: %s, run command success, stdout: %s", c.Namespace, c.ReleaseName, stdout)
|
|
}
|
|
|
|
return stdout.String(), nil
|
|
}
|