fix: graceful delete ks-core (#6182)

* fix: graceful delete ks-core

Signed-off-by: joyceliu <joyceliu@yunify.com>

* fix: graceful delete ks-core

Signed-off-by: joyceliu <joyceliu@yunify.com>

---------

Signed-off-by: joyceliu <joyceliu@yunify.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

config/ks-core/charts/ks-crds/scripts/post-delete.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# set -x
+
+CRD_NAMES=$1
+MAPPING_CONFIG=$2
+
+for extension in `kubectl get installplan -o json | jq -r '.items[] | select(.status.state == "Installed") | .metadata.name'`
+do
+  namespace=$(kubectl get installplan $extension -o=jsonpath='{.status.targetNamespace}')
+  version=$(kubectl get extension $extension -o=jsonpath='{.status.installedVersion}')
+  extensionversion=$extension-$version
+  echo "Found extension $extensionversion installed"
+  helm status $extension --namespace $namespace
+  if [ $? -eq 0 ]; then
+    helm mapkubeapis $extension --namespace $namespace --mapfile $MAPPING_CONFIG
+  fi
+  helm status $extension-agent --namespace $namespace
+  if [ $? -eq 0 ]; then
+    helm mapkubeapis $extension-agent --namespace $namespace --mapfile $MAPPING_CONFIG
+  fi
+done
+
+
+# remove namespace's finalizers && ownerReferences
+kubectl patch workspaces.tenant.kubesphere.io system-workspace -p '{"metadata":{"finalizers":[]}}' --type=merge
+kubectl patch workspacetemplates.tenant.kubesphere.io system-workspace -p '{"metadata":{"finalizers":[]}}' --type=merge
+for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}' -l 'kubesphere.io/managed=true')
+do
+  kubectl label ns $ns kubesphere.io/workspace- && \
+  kubectl patch ns $ns -p '{"metadata":{"ownerReferences":[]}}' --type=merge && \
+  echo "{\"kind\":\"Namespace\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"$ns\",\"finalizers\":null}}" | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f -
+done
+
+
+# delete crds
+for crd in `kubectl get crds -o jsonpath="{.items[*].metadata.name}"`
+do
+  if [[ ${CRD_NAMES[@]/${crd}/} != ${CRD_NAMES[@]} ]]; then
+     scop=$(eval echo $(kubectl get crd ${crd} -o jsonpath="{.spec.scope}"))
+     if [[ $scop =~ "Namespaced" ]] ; then
+        kubectl get $crd -A --no-headers | awk '{print $1" "$2" ""'$crd'"}' | xargs -n 3 sh -c 'kubectl patch $2 -n $0 $1 -p "{\"metadata\":{\"finalizers\":null}}" --type=merge 2>/dev/null && kubectl delete $2 -n $0 $1 2>/dev/null'
+     else
+        kubectl get $crd -A --no-headers | awk '{print $1" ""'$crd'"}' | xargs -n 2 sh -c 'kubectl patch $1 $0 -p "{\"metadata\":{\"finalizers\":null}}" --type=merge 2>/dev/null && kubectl delete $1 $0 2>/dev/null'
+     fi
+     kubectl delete crd $crd 2>/dev/null;
+  fi
+done

config/ks-core/charts/ks-crds/templates/_images.tpl

@@ -0,0 +1,21 @@
+{{- define "kubectl.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.kubectl.image "global" (default .Values.global (dict "imageRegistry" "docker.io"))) }}
+{{- end -}}
+
+{{- define "common.images.image" -}}
+{{- $registryName := .global.imageRegistry -}}
+{{- $repositoryName := .imageRoot.repository -}}
+{{- $separator := ":" -}}
+{{- $termination := .global.tag | toString -}}
+{{- if .imageRoot.registry }}
+    {{- $registryName = .imageRoot.registry -}}
+{{- end -}}
+{{- if .imageRoot.tag }}
+    {{- $termination = .imageRoot.tag | toString -}}
+{{- end -}}
+{{- if .imageRoot.digest }}
+    {{- $separator = "@" -}}
+    {{- $termination = .imageRoot.digest | toString -}}
+{{- end -}}
+{{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}}
+{{- end -}}

config/ks-core/charts/ks-crds/templates/post-delete-crd-job.yaml

@@ -0,0 +1,89 @@
+{{- $kubeVersion := .Capabilities.KubeVersion }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Release.Name }}-post-delete-crd-scripts"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
+data:
+  map.yaml: |
+    mappings:
+    {{- range $path, $_ := .Files.Glob "crds/**" }}
+    {{- $crd := $.Files.Get $path | fromYaml }}
+    {{- range $_, $version := $crd.spec.versions }}
+      - deprecatedAPI: "apiVersion: {{ $crd.spec.group }}/{{ $version.name }}\nkind: {{ $crd.spec.names.kind }}\n"
+        removedInVersion: "{{ $kubeVersion }}"
+    {{- end }}
+    {{- end }}
+{{ (.Files.Glob "scripts/post-delete.sh").AsConfig | indent 2 }}
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ .Release.Name }}-post-delete-crd"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: "{{ .Release.Name }}-post-delete-crd"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-3"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: "{{ .Release.Name }}-post-delete-crd"
+    namespace: {{ .Release.Namespace }}
+
+---
+
+{{- $crdNameList := list }}
+{{- range $path, $_ := .Files.Glob "crds/**" }}
+{{- $crd := $.Files.Get $path | fromYaml }}
+{{- $crdNameList = append $crdNameList $crd.metadata.name }}
+{{- end }}
+
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ .Release.Name }}-post-delete-crd"
+  annotations:
+    "helm.sh/hook": post-delete
+    "helm.sh/hook-weight": "-2"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+spec:
+  template:
+    spec:
+      restartPolicy: Never
+      serviceAccountName: "{{ .Release.Name }}-post-delete-crd"
+      containers:
+        - name: post-delete-job
+          image: {{ template "kubectl.image" . }}
+          command:
+            - /bin/bash
+            - /scripts/post-delete.sh
+            - '{{ join " " $crdNameList }}'
+            - /scripts/map.yaml
+          volumeMounts:
+            - mountPath: /scripts
+              name: scripts
+          resources: {{- toYaml .Values.kubectl.resources | nindent 12 }}
+      volumes:
+        - name: scripts
+          configMap:
+            name: "{{ .Release.Name }}-post-delete-crd-scripts"
+            defaultMode: 420

config/ks-core/charts/ks-crds/templates/pre-upgrade-crd-job.yaml

@@ -10,6 +10,36 @@ data:
 {{ (.Files.Glob "scripts/install.sh").AsConfig | indent 2 }}
 {{ (.Files.Glob "crds/*").AsConfig | indent 2 }}
 
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "{{ .Release.Name }}-pre-upgrade-crd"
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: "{{ .Release.Name }}-pre-upgrade-crd"
+  annotations:
+    "helm.sh/hook": pre-upgrade
+    "helm.sh/hook-weight": "-1"
+    "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cluster-admin
+subjects:
+  - kind: ServiceAccount
+    name: "{{ .Release.Name }}-pre-upgrade-crd"
+    namespace: {{ .Release.Namespace }}
+
 ---
 apiVersion: batch/v1
 kind: Job
@@ -23,10 +53,10 @@ spec:
   template:
     spec:
       restartPolicy: Never
-      serviceAccountName: {{ include "ks-core.serviceAccountName" . }}
+      serviceAccountName: "{{ .Release.Name }}-pre-upgrade-crd"
       containers:
         - name: crd-install
-          image: {{ template "preUpgrade.image" . }}
+          image: {{ template "kubectl.image" . }}
           command:
             - /bin/bash
             - /scripts/install.sh
@@ -34,7 +64,7 @@ spec:
           volumeMounts:
             - mountPath: /scripts
               name: scripts
-          resources: {{- toYaml .Values.preUpgrade.resources | nindent 12 }}
+          resources: {{- toYaml .Values.kubectl.resources | nindent 12 }}
       volumes:
         - name: scripts
           configMap:

config/ks-core/charts/ks-crds/values.yaml

@@ -1,12 +1,11 @@
 # Default values for ks-crds.
 # This is a YAML-formatted file.
 # Declare variables to be passed into your templates.
-
+kubectl:
-preUpgrade:
   image:
     registry: ""
-    repository: kubesphereio/kubectl
+    repository: kubesphere/kubectl
-    tag: "v1.27.12"
+    tag: "v1.27.16"
     pullPolicy: IfNotPresent
   resources:
     limits:
@@ -14,4 +13,4 @@ preUpgrade:
       memory: 1024Mi
     requests:
       cpu: 20m
-      memory: 100Mi
+      memory: 100Mi

config/ks-core/scripts/post-delete.sh

@@ -2,55 +2,9 @@
 
 # set -x
 
-CRD_NAMES=$1
-MAPPING_CONFIG=$2
-
-for extension in `kubectl get installplan -o json | jq -r '.items[] | select(.status.state == "Installed") | .metadata.name'`
-do
-  namespace=$(kubectl get installplan $extension -o=jsonpath='{.status.targetNamespace}')
-  version=$(kubectl get extension $extension -o=jsonpath='{.status.installedVersion}')
-  extensionversion=$extension-$version
-  echo "Found extension $extensionversion installed"
-  helm status $extension --namespace $namespace
-  if [ $? -eq 0 ]; then
-    helm mapkubeapis $extension --namespace $namespace --mapfile $MAPPING_CONFIG
-  fi
-  helm status $extension-agent --namespace $namespace
-  if [ $? -eq 0 ]; then
-    helm mapkubeapis $extension-agent --namespace $namespace --mapfile $MAPPING_CONFIG
-  fi
-done
-
-
-# remove namespace's finalizers && ownerReferences
-kubectl patch workspaces.tenant.kubesphere.io system-workspace -p '{"metadata":{"finalizers":[]}}' --type=merge
-kubectl patch workspacetemplates.tenant.kubesphere.io system-workspace -p '{"metadata":{"finalizers":[]}}' --type=merge
-for ns in $(kubectl get ns -o jsonpath='{.items[*].metadata.name}' -l 'kubesphere.io/managed=true')
-do
-  kubectl label ns $ns kubesphere.io/workspace- && \
-  kubectl patch ns $ns -p '{"metadata":{"ownerReferences":[]}}' --type=merge && \
-  echo "{\"kind\":\"Namespace\",\"apiVersion\":\"v1\",\"metadata\":{\"name\":\"$ns\",\"finalizers\":null}}" | kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f -
-done
-
-
-# delete crds
-for crd in `kubectl get crds -o jsonpath="{.items[*].metadata.name}"`
-do
-  if [[ ${CRD_NAMES[@]/${crd}/} != ${CRD_NAMES[@]} ]]; then
-     scop=$(eval echo $(kubectl get crd ${crd} -o jsonpath="{.spec.scope}"))
-     if [[ $scop =~ "Namespaced" ]] ; then
-        kubectl get $crd -A --no-headers | awk '{print $1" "$2" ""'$crd'"}' | xargs -n 3 sh -c 'kubectl patch $2 -n $0 $1 -p "{\"metadata\":{\"finalizers\":null}}" --type=merge 2>/dev/null && kubectl delete $2 -n $0 $1 2>/dev/null'
-     else
-        kubectl get $crd -A --no-headers | awk '{print $1" ""'$crd'"}' | xargs -n 2 sh -c 'kubectl patch $1 $0 -p "{\"metadata\":{\"finalizers\":null}}" --type=merge 2>/dev/null && kubectl delete $1 $0 2>/dev/null'
-     fi
-     kubectl delete crd $crd 2>/dev/null;
-  fi
-done
-
-
 EXTENSION_RELATED_RESOURCES='jobs.batch roles.rbac.authorization.k8s.io rolebindings.rbac.authorization.k8s.io clusterroles.rbac.authorization.k8s.io clusterrolebindings.rbac.authorization.k8s.io'
 
 for resource in $EXTENSION_RELATED_RESOURCES;do
   echo "kubectl delete $resource -l kubesphere.io/extension-ref --all-namespaces"
   kubectl delete $resource -l kubesphere.io/managed=true --all-namespaces
-done
+done

config/ks-core/templates/_images.tpl

@@ -33,10 +33,6 @@ Return the proper image name
 {{ include "common.images.image" (dict "imageRoot" .Values.redis.image "global" .Values.global) }}
 {{- end -}}
 
-{{- define "preUpgrade.image" -}}
-{{ include "common.images.image" (dict "imageRoot" .Values.preUpgrade.image "global" .Values.global) }}
-{{- end -}}
-
 {{- define "extensions_museum.image" -}}
 {{ include "common.images.image" (dict "imageRoot" .Values.ksExtensionRepository.image "global" .Values.global) }}
 {{- end -}}

config/ks-core/templates/post-delete-job.yaml

@@ -1,4 +1,3 @@
-{{- $kubeVersion := .Capabilities.KubeVersion }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -8,15 +7,6 @@ metadata:
     "helm.sh/hook-weight": "-1"
     "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded,hook-failed
 data:
-  map.yaml: |
-    mappings:
-    {{- range $path, $_ := .Files.Glob "charts/ks-crds/crds/**" }}
-    {{- $crd := $.Files.Get $path | fromYaml }}
-    {{- range $_, $version := $crd.spec.versions }}
-      - deprecatedAPI: "apiVersion: {{ $crd.spec.group }}/{{ $version.name }}\nkind: {{ $crd.spec.names.kind }}\n"
-        removedInVersion: "{{ $kubeVersion }}"
-    {{- end }}
-    {{- end }}
 {{ (.Files.Glob "scripts/post-delete.sh").AsConfig | indent 2 }}
 
 ---
@@ -51,12 +41,6 @@ subjects:
 
 ---
 
-{{- $crdNameList := list }}
-{{- range $path, $_ := .Files.Glob "charts/ks-crds/crds/**" }}
-{{- $crd := $.Files.Get $path | fromYaml }}
-{{- $crdNameList = append $crdNameList $crd.metadata.name }}
-{{- end }}
-
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -76,8 +60,6 @@ spec:
           command:
             - /bin/bash
             - /scripts/post-delete.sh
-            - '{{ join " " $crdNameList }}'
-            - /scripts/map.yaml
           volumeMounts:
             - mountPath: /scripts
               name: scripts

config/ks-core/values.yaml

@@ -421,7 +421,7 @@ redisHA:
                     - ""
 
 ksCRDs:
-  preUpgrade:
+  kubectl:
     image:
       registry: ""
       repository: kubesphere/kubectl