admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update application controllers log level

Browse Source 
* [application] update application controllers log level

* update

---------

Signed-off-by: wenhaozhou <wenhaozhou@yunify.com>
Signed-off-by: hongming <coder.scala@gmail.com>
This commit is contained in:
hongming
2025-03-19 13:37:12 +08:00
committed by ks-ci-bot
parent 522d0b4de5
commit 0e76a4bcc5
5 changed files with 162 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
pkg/controller/application/apprelease_controller.go
View File

@@ -13,40 +13,32 @@ import (
"strings"
"time"
"k8s.io/client-go/rest"
batchv1 "k8s.io/api/batch/v1"
"kubesphere.io/utils/helm"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"kubesphere.io/api/constants"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/handler"
"kubesphere.io/utils/s3"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
"kubesphere.io/kubesphere/pkg/controller"
kscontroller "kubesphere.io/kubesphere/pkg/controller/options"
"github.com/go-logr/logr"
helmrelease "helm.sh/helm/v3/pkg/release"
batchv1 "k8s.io/api/batch/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
appv2 "kubesphere.io/api/application/v2"
"kubesphere.io/kubesphere/pkg/simple/client/application"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
"kubesphere.io/api/constants"
"kubesphere.io/utils/helm"
"kubesphere.io/utils/s3"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"kubesphere.io/kubesphere/pkg/controller"
kscontroller "kubesphere.io/kubesphere/pkg/controller/options"
"kubesphere.io/kubesphere/pkg/simple/client/application"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)
const (
@@ -76,9 +68,10 @@ func (r *AppReleaseReconciler) SetupWithManager(mgr *controller.Manager) error {
r.Client = mgr.GetClient()
clusterClientSet, err := clusterclient.NewClusterClientSet(mgr.GetCache())
if err != nil {
return fmt.Errorf("failed to create cluster client set: %v", err)
return fmt.Errorf("failed to create cluster client set")
}
r.clusterClientSet = clusterClientSet
r.logger = ctrl.Log.WithName("controllers").WithName(helminstallerController)
if r.HelmExecutorOptions == nil || r.HelmExecutorOptions.Image == "" {
return fmt.Errorf("helm executor options is nil or image is empty")
@@ -86,7 +79,7 @@ func (r *AppReleaseReconciler) SetupWithManager(mgr *controller.Manager) error {
r.cmStore, r.ossStore, err = application.InitStore(mgr.Options.S3Options, r.Client)
if err != nil {
klog.Errorf("failed to init store: %v", err)
r.logger.Error(err, "failed to init store")
return err
}
@@ -105,11 +98,11 @@ func (r *AppReleaseReconciler) SetupWithManager(mgr *controller.Manager) error {
func (r *AppReleaseReconciler) mapper(ctx context.Context, o client.Object) (requests []reconcile.Request) {
cluster := o.(*clusterv1alpha1.Cluster)
klog.Infof("cluster %s has been deleted", cluster.Name)
r.logger.V(4).Info("cluster has been deleted", "cluster", cluster)
apprlsList := &appv2.ApplicationReleaseList{}
opts := &client.ListOptions{LabelSelector: labels.SelectorFromSet(labels.Set{constants.ClusterNameLabelKey: cluster.Name})}
if err := r.List(ctx, apprlsList, opts); err != nil {
klog.Errorf("failed to list application releases: %v", err)
r.logger.Error(err, "failed to list application releases")
return requests
}
for _, apprls := range apprlsList.Items {
@@ -124,15 +117,15 @@ type AppReleaseReconciler struct {
HelmExecutorOptions *kscontroller.HelmExecutorOptions
ossStore s3.Interface
cmStore s3.Interface
logger logr.Logger
}
func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
apprls := &appv2.ApplicationRelease{}
if err := r.Client.Get(ctx, req.NamespacedName, apprls); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
logger := r.logger.WithValues("application release", apprls.Name).WithValues("namespace", apprls.Namespace)
timeoutRecheck := apprls.Annotations[appv2.TimeoutRecheck]
var reCheck int
if timeoutRecheck == "" {
@@ -143,12 +136,12 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
dstKubeConfig, runClient, err := r.getClusterInfo(apprls.GetRlsCluster())
if err != nil {
klog.Errorf("failed to get cluster info: %v", err)
logger.Error(err, "failed to get cluster info")
return ctrl.Result{}, err
}
executor, err := r.getExecutor(apprls, dstKubeConfig, runClient)
if err != nil {
klog.Errorf("failed to get executor: %v", err)
logger.Error(err, "failed to get executor")
return ctrl.Result{}, err
}
@@ -160,18 +153,18 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
helmKubeConfig, err := application.GetHelmKubeConfig(ctx, cluster, runClient)
if err != nil {
klog.Errorf("failed to get helm kubeconfig: %v", err)
logger.Error(err, "failed to get helm kubeconfig")
return ctrl.Result{}, err
}
if apierrors.IsNotFound(err) || (err == nil && !cluster.DeletionTimestamp.IsZero()) {
klog.Errorf("cluster not found or deleting %s: %v", apprls.GetRlsCluster(), err)
logger.Error(err, "cluster not found or deleting", "cluster", apprls.GetRlsCluster())
apprls.Status.State = appv2.StatusClusterDeleted
apprls.Status.Message = fmt.Sprintf("cluster %s has been deleted", cluster.Name)
patch, _ := json.Marshal(apprls)
err = r.Status().Patch(ctx, apprls, client.RawPatch(client.Merge.Type(), patch))
if err != nil {
klog.Errorf("failed to update apprelease %s: %v", apprls.Name, err)
logger.Error(err, "failed to update application release")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
@@ -180,7 +173,7 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if !controllerutil.ContainsFinalizer(apprls, HelmReleaseFinalizer) && apprls.ObjectMeta.DeletionTimestamp.IsZero() {
expected := apprls.DeepCopy()
controllerutil.AddFinalizer(expected, HelmReleaseFinalizer)
klog.Infof("add finalizer for apprelease %s", apprls.Name)
logger.V(6).Info("add finalizer for application release")
return ctrl.Result{}, r.Patch(ctx, expected, client.MergeFrom(apprls))
}
@@ -194,14 +187,14 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
wait, err := r.cleanJob(ctx, apprls, runClient)
if err != nil {
klog.Errorf("failed to clean job: %v", err)
logger.Error(err, "failed to clean job")
return ctrl.Result{}, err
}
if wait {
klog.Infof("job wait, job for %s is still active", apprls.Name)
logger.V(6).Info("job wait, job is still active")
return ctrl.Result{RequeueAfter: verificationAgain * time.Second}, nil
}
klog.Infof("job for %s has been cleaned", apprls.Name)
logger.V(4).WithValues().Info("job has been cleaned")
if err = r.Client.Get(ctx, req.NamespacedName, apprls); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
@@ -209,10 +202,10 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
apprls.Finalizers = nil
err = r.Update(ctx, apprls)
if err != nil {
klog.Errorf("failed to remove finalizer for apprelease %s: %v", apprls.Name, err)
logger.Error(err, "failed to remove finalizer for application release")
return ctrl.Result{}, err
}
klog.Infof("remove finalizer for apprelease %s", apprls.Name)
logger.V(6).Info("remove finalizer for application release")
return ctrl.Result{}, nil
}
@@ -262,10 +255,10 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
if apprls.Status.State != appv2.StatusTimeout {
err = r.updateStatus(ctx, apprls, appv2.StatusTimeout, "Installation timeout")
if err != nil {
klog.Errorf("failed to update apprelease %s status : %v", apprls.Name, err)
logger.Error(err, "failed to update application release status")
return ctrl.Result{}, err
}
klog.Infof("Installation timeout, will check status again after %d second", timeoutVerificationAgain)
logger.V(2).Info("installation timeout, will check status again after seconds", "timeout verification again", timeoutVerificationAgain)
return ctrl.Result{RequeueAfter: timeoutVerificationAgain * time.Second}, nil
}
@@ -278,15 +271,15 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
patch, _ := json.Marshal(apprls)
err = r.Patch(ctx, apprls, client.RawPatch(client.Merge.Type(), patch))
if err != nil {
klog.Errorf("failed to update apprelease %s: %v", apprls.Name, err)
logger.Error(err, "failed to update application release")
return ctrl.Result{}, err
}
klog.Infof("update recheck times %s for %s", strconv.Itoa(reCheck+1), apprls.Name)
logger.V(2).Info("update recheck times", "recheck times", strconv.Itoa(reCheck+1))
if deployed {
err = r.updateStatus(ctx, apprls, appv2.StatusActive, "StatusActive")
if err != nil {
klog.Errorf("failed to update apprelease %s %v", apprls.Name, err)
logger.Error(err, "failed to update application release")
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
@@ -299,7 +292,7 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
err = r.updateStatus(ctx, apprls, appv2.StatusActive, release.Info.Description)
return ctrl.Result{}, err
default:
klog.V(5).Infof("helm release %s/%s status %s, check again after %d second", apprls.GetRlsNamespace(), apprls.Name, release.Info.Status, verificationAgain)
r.logger.V(5).Info(fmt.Sprintf("helm release %s/%s status %s, check again after %d seconds", apprls.GetRlsNamespace(), apprls.Name, release.Info.Status, verificationAgain))
return ctrl.Result{RequeueAfter: verificationAgain * time.Second}, nil
}
}
@@ -313,12 +306,13 @@ func (r *AppReleaseReconciler) Reconcile(ctx context.Context, req ctrl.Request)
}
func (r *AppReleaseReconciler) checkJob(ctx context.Context, apprls *appv2.ApplicationRelease, runClient client.Client, release *helmrelease.Release) (ct ctrl.Result, todo bool, err error) {
klog.Infof("helm release %s/%s ready to create or upgrade yet,check job %s", apprls.GetRlsNamespace(), apprls.Name, apprls.Status.InstallJobName)
logger := r.logger.WithValues("application release", apprls).WithValues("namespace", apprls.Namespace)
logger.V(4).Info("helm release %s/%s ready to create or upgrade yet,check job %s", apprls.GetRlsNamespace(), apprls.Name, apprls.Status.InstallJobName)
job := &batchv1.Job{}
if err := runClient.Get(ctx, types.NamespacedName{Namespace: apprls.GetRlsNamespace(), Name: apprls.Status.InstallJobName}, job); err != nil {
if apierrors.IsNotFound(err) {
klog.Errorf("job %s not found", apprls.Status.InstallJobName)
logger.Error(err, "job not found", "install job", apprls.Status.InstallJobName)
msg := "deploy failed, job not found"
return ctrl.Result{}, false, r.updateStatus(ctx, apprls, appv2.StatusDeployFailed, msg)
}
@@ -329,7 +323,7 @@ func (r *AppReleaseReconciler) checkJob(ctx context.Context, apprls *appv2.Appli
return ctrl.Result{}, false, r.updateStatus(ctx, apprls, appv2.StatusActive, "Upgrade succeeful")
}
if job.Status.Failed > 0 {
klog.Infof("install apprls %s job %s , failed times %d/%d", apprls.Name, job.Name, job.Status.Failed, *job.Spec.BackoffLimit+1)
logger.V(2).Info(fmt.Sprintf("install job failed, failed times %d/%d", job.Status.Failed, *job.Spec.BackoffLimit+1), "job", job.Name)
}
if job.Spec.BackoffLimit != nil && job.Status.Failed > *job.Spec.BackoffLimit {
// When in the upgrade state, if job execution fails while the HelmRelease status remains deployed, directly mark the AppRelease as StatusDeployFailed.
@@ -344,45 +338,47 @@ func (r *AppReleaseReconciler) checkJob(ctx context.Context, apprls *appv2.Appli
}
func (r *AppReleaseReconciler) removeAll(ctx context.Context, apprls *appv2.ApplicationRelease, executor helm.Executor, kubeconfig []byte) (ct ctrl.Result, err error) {
logger := r.logger.WithValues("application release", apprls).WithValues("namespace", apprls.Namespace)
err = r.updateStatus(ctx, apprls, appv2.StatusDeleting, "Uninstalling")
if err != nil {
klog.Errorf("failed to update apprelease %s status : %v", apprls.Name, err)
logger.Error(err, "failed to update application release status")
return ctrl.Result{}, err
}
uninstallJobName, err := r.uninstall(ctx, apprls, executor, kubeconfig)
if err != nil {
klog.Errorf("failed to uninstall helm release %s: %v", apprls.Name, err)
logger.Error(err, "failed to uninstall application release")
return ctrl.Result{}, err
}
err = r.cleanStore(ctx, apprls)
if err != nil {
klog.Errorf("failed to clean store: %v", err)
logger.Error(err, "failed to clean store")
return ctrl.Result{}, err
}
klog.Infof("remove apprelease %s success", apprls.Name)
logger.V(4).Info("remove application release success")
if uninstallJobName != "" {
klog.Infof("try to update uninstall apprls job name %s to apprelease %s", uninstallJobName, apprls.Name)
logger.V(4).Info("try to update application release uninstall job", "job", uninstallJobName)
apprls.Status.UninstallJobName = uninstallJobName
apprls.Status.LastUpdate = metav1.Now()
patch, _ := json.Marshal(apprls)
err = r.Status().Patch(ctx, apprls, client.RawPatch(client.Merge.Type(), patch))
if err != nil {
klog.Errorf("failed to update apprelease %s: %v", apprls.Name, err)
logger.Error(err, "failed to update application release")
return ctrl.Result{}, err
}
klog.Infof("update uninstall apprls job name %s to apprelease %s success", uninstallJobName, apprls.Name)
logger.V(4).Info("update application release uninstall job success", "job", uninstallJobName)
}
return ctrl.Result{}, nil
}
func (r *AppReleaseReconciler) getClusterDynamicClient(clusterName string, apprls *appv2.ApplicationRelease) (*dynamic.DynamicClient, error) {
logger := r.logger.WithValues("application release", apprls).WithValues("namespace", apprls.Namespace)
clusterClient, err := r.clusterClientSet.GetClusterClient(clusterName)
if err != nil {
klog.Errorf("failed to get cluster client: %v", err)
logger.Error(err, "failed to get cluster client", "cluster", clusterName)
return nil, err
}
creator := apprls.Annotations[constants.CreatorAnnotationKey]
@@ -392,7 +388,7 @@ func (r *AppReleaseReconciler) getClusterDynamicClient(clusterName string, apprl
UserName: creator,
}
}
klog.Infof("DynamicClient impersonate kubeAsUser: %s", creator)
logger.V(4).Info("DynamicClient impersonate kubeAsUser", "creator", creator)
dynamicClient, err := dynamic.NewForConfig(&conf)
return dynamicClient, err
}