From e389253a4ef0bd120c731f1540196a12a7035b47 Mon Sep 17 00:00:00 2001 From: Xinzhao Xu Date: Thu, 24 Oct 2024 15:12:01 +0800 Subject: [PATCH 1/2] Support for configuring affinity for helm executor (#1989) --- config/ks-core/templates/kubesphere-config.yaml | 3 +++ config/ks-core/values.yaml | 11 +++++++++++ pkg/controller/core/installplan_controller.go | 1 + pkg/controller/options/options.go | 1 + staging/src/kubesphere.io/utils/helm/executor.go | 13 +++++++++++++ 5 files changed, 29 insertions(+) diff --git a/config/ks-core/templates/kubesphere-config.yaml b/config/ks-core/templates/kubesphere-config.yaml index fe83655b2..083b3d6b8 100644 --- a/config/ks-core/templates/kubesphere-config.yaml +++ b/config/ks-core/templates/kubesphere-config.yaml @@ -46,6 +46,9 @@ data: {{- if .Values.helmExecutor.resources }} resources: {{- toYaml .Values.helmExecutor.resources | nindent 8 }} {{- end }} + {{- if .Values.helmExecutor.affinity }} + affinity: {{- toYaml .Values.helmExecutor.affinity | nindent 8 }} + {{- end }} extension: imageRegistry: {{ default .Values.extension.imageRegistry "" | quote }} {{- if .Values.extension.nodeSelector }} diff --git a/config/ks-core/values.yaml b/config/ks-core/values.yaml index 560e6ddec..3c2c131a8 100644 --- a/config/ks-core/values.yaml +++ b/config/ks-core/values.yaml @@ -278,6 +278,17 @@ helmExecutor: requests: cpu: 100m memory: 100Mi + affinity: + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchLabels: + app: ks-controller-manager + topologyKey: "kubernetes.io/hostname" + namespaces: + - kubesphere-system composedApp: # Selector to filter k8s applications to reconcile diff --git a/pkg/controller/core/installplan_controller.go b/pkg/controller/core/installplan_controller.go index 2bc781482..ad860ddf4 100644 --- a/pkg/controller/core/installplan_controller.go +++ b/pkg/controller/core/installplan_controller.go @@ -1556,6 +1556,7 @@ func (r *InstallPlanReconciler) newExecutor(plan *corev1alpha1.InstallPlan) (hel helm.SetExecutorNamespace(plan.Status.TargetNamespace), helm.SetExecutorBackoffLimit(0), helm.SetTTLSecondsAfterFinished(r.HelmExecutorOptions.JobTTLAfterFinished), + helm.SetExecutorAffinity(r.HelmExecutorOptions.Affinity), } if r.HelmExecutorOptions.Resources != nil { executorOptions = append(executorOptions, helm.SetExecutorResources(corev1.ResourceRequirements{ diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index de308a86a..7d12bb3ab 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -38,6 +38,7 @@ type HelmExecutorOptions struct { HistoryMax uint `json:"historyMax,omitempty" yaml:"historyMax,omitempty" mapstructure:"historyMax,omitempty"` JobTTLAfterFinished time.Duration `json:"jobTTLAfterFinished,omitempty" yaml:"jobTTLAfterFinished,omitempty" mapstructure:"jobTTLAfterFinished,omitempty"` Resources *ResourceRequirements `json:"resources,omitempty" yaml:"resources,omitempty" mapstructure:"resources,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty" yaml:"affinity,omitempty" mapstructure:"affinity,omitempty"` } type ResourceRequirements struct { diff --git a/staging/src/kubesphere.io/utils/helm/executor.go b/staging/src/kubesphere.io/utils/helm/executor.go index c478322ac..7163fb68c 100644 --- a/staging/src/kubesphere.io/utils/helm/executor.go +++ b/staging/src/kubesphere.io/utils/helm/executor.go @@ -92,6 +92,7 @@ type executor struct { client kubernetes.Interface helmImage string resources corev1.ResourceRequirements + affinity *corev1.Affinity labels map[string]string owner *metav1.OwnerReference @@ -116,6 +117,12 @@ func SetExecutorResources(resources corev1.ResourceRequirements) ExecutorOption } } +func SetExecutorAffinity(affinity *corev1.Affinity) ExecutorOption { + return func(e *executor) { + e.affinity = affinity + } +} + func SetExecutorLabels(labels map[string]string) ExecutorOption { return func(o *executor) { o.labels = labels @@ -659,6 +666,9 @@ func (e *executor) createInstallJob(ctx context.Context, release, chart string, if e.ttlSecondsAfterFinished > 0 { job.Spec.TTLSecondsAfterFinished = pointer.Int32(e.ttlSecondsAfterFinished) } + if e.affinity != nil { + job.Spec.Template.Spec.Affinity = e.affinity + } if helmOptions.serviceAccount != "" { job.Spec.Template.Spec.ServiceAccountName = helmOptions.serviceAccount } @@ -856,6 +866,9 @@ func (e *executor) Uninstall(ctx context.Context, release string, options ...Hel if e.ttlSecondsAfterFinished > 0 { job.Spec.TTLSecondsAfterFinished = pointer.Int32(e.ttlSecondsAfterFinished) } + if e.affinity != nil { + job.Spec.Template.Spec.Affinity = e.affinity + } if helmOptions.hookImage != "" { job.Spec.Template.Spec.InitContainers = []corev1.Container{ { From 62749787093654c9d0b389ae70d7229ed1c45286 Mon Sep 17 00:00:00 2001 From: KubeSphere CI Bot <47586280+ks-ci-bot@users.noreply.github.com> Date: Tue, 3 Sep 2024 13:36:35 +0800 Subject: [PATCH 2/2] fix helm executor serviceaccount not created (#1948) Co-authored-by: hongming --- pkg/controller/core/installplan_controller.go | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/pkg/controller/core/installplan_controller.go b/pkg/controller/core/installplan_controller.go index ad860ddf4..b846cf8bd 100644 --- a/pkg/controller/core/installplan_controller.go +++ b/pkg/controller/core/installplan_controller.go @@ -711,12 +711,15 @@ func initTargetNamespace(ctx context.Context, client client.Client, namespace, e if err := createNamespaceIfNotExists(ctx, client, namespace, extensionName); err != nil { return fmt.Errorf("failed to create namespace: %v", err) } - sa := rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: fmt.Sprintf("helm-executor.%s", extensionName), - Namespace: namespace, - } return retry.RetryOnConflict(retry.DefaultRetry, func() error { + sa := rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: fmt.Sprintf("helm-executor.%s", extensionName), + Namespace: namespace, + } + if err := createOrUpdateServiceAccount(ctx, client, extensionName, sa); err != nil { + return err + } if err := createOrUpdateRole(ctx, client, namespace, extensionName, role.Rules); err != nil { return err } @@ -733,6 +736,21 @@ func initTargetNamespace(ctx context.Context, client client.Client, namespace, e }) } +func createOrUpdateServiceAccount(ctx context.Context, client client.Client, extensionName string, sa rbacv1.Subject) error { + serviceAccount := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: sa.Name, Namespace: sa.Namespace}} + op, err := controllerutil.CreateOrUpdate(ctx, client, serviceAccount, func() error { + serviceAccount.Labels = map[string]string{corev1alpha1.ExtensionReferenceLabel: extensionName} + return nil + }) + + if err != nil { + return err + } + + klog.V(4).Infof("service account %s in namespace %s %s", serviceAccount.Name, serviceAccount.Namespace, op) + return nil +} + func createOrUpdateClusterRole(ctx context.Context, client client.Client, extensionName string, rules []rbacv1.PolicyRule) error { clusterRoleName := fmt.Sprintf(defaultClusterRoleFormat, extensionName) clusterRole := &rbacv1.ClusterRole{ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName}}