From 57acaeba4111b1bf96a34b5408cb788180c9002e Mon Sep 17 00:00:00 2001 From: hongming Date: Sat, 23 May 2020 21:26:28 +0800 Subject: [PATCH] use ownerReference control the lifecycle of user's kubeconfig and kubectl pod Signed-off-by: hongming --- cmd/controller-manager/app/controllers.go | 2 +- pkg/constants/constants.go | 2 +- .../certificatesigningrequest_controller.go | 4 +- .../clusterrolebinding_controller.go | 9 +-- pkg/kapis/resources/v1alpha2/handler.go | 3 +- pkg/models/kubeconfig/kubeconfig.go | 19 +---- pkg/models/kubectl/kubectl.go | 69 ++++++++++--------- 7 files changed, 51 insertions(+), 57 deletions(-) diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index 78608a17e..eec2e917a 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -124,7 +124,7 @@ func AddControllers( csrController := certificatesigningrequest.NewController(client.Kubernetes(), kubernetesInformer, client.Config()) - clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(), kubernetesInformer) + clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(), kubernetesInformer, kubesphereInformer) clusterController := cluster.NewClusterController( client.Kubernetes(), diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index 17250127c..95d5088a8 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -38,7 +38,7 @@ const ( DisplayNameAnnotationKey = "kubesphere.io/alias-name" DescriptionAnnotationKey = "kubesphere.io/description" CreatorAnnotationKey = "kubesphere.io/creator" - UsernameAnnotationKey = "kubesphere.io/username" + UsernameLabelKey = "kubesphere.io/username" System = "system" OpenPitrixRuntimeAnnotationKey = "openpitrix_runtime" WorkspaceAdmin = "workspace-admin" diff --git a/pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go b/pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go index 484876c2f..a39e569d5 100644 --- a/pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go +++ b/pkg/controller/certificatesigningrequest/certificatesigningrequest_controller.go @@ -221,7 +221,7 @@ func (c *Controller) reconcile(key string) error { } // csr create by kubesphere auto approve - if username := csr.Annotations[constants.UsernameAnnotationKey]; username != "" { + if username := csr.Labels[constants.UsernameLabelKey]; username != "" { err = c.Approve(csr) if err != nil { klog.Error(err) @@ -280,7 +280,7 @@ func (c *Controller) Approve(csr *certificatesv1beta1.CertificateSigningRequest) } func (c *Controller) UpdateKubeconfig(csr *certificatesv1beta1.CertificateSigningRequest) error { - username := csr.Annotations[constants.UsernameAnnotationKey] + username := csr.Labels[constants.UsernameLabelKey] err := c.kubeconfigOperator.UpdateKubeconfig(username, csr.Status.Certificate) diff --git a/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go b/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go index 7b4ff8592..4b81222d4 100644 --- a/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go +++ b/pkg/controller/clusterrolebinding/clusterrolebinding_controller.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/informers" + k8sinformers "k8s.io/client-go/informers" rbacv1informers "k8s.io/client-go/informers/rbac/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog" iamv1alpha2 "kubesphere.io/kubesphere/pkg/apis/iam/v1alpha2" + ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" "kubesphere.io/kubesphere/pkg/models/kubectl" "time" ) @@ -63,7 +64,7 @@ type Controller struct { kubectlOperator kubectl.Interface } -func NewController(k8sClient kubernetes.Interface, informerFactory informers.SharedInformerFactory) *Controller { +func NewController(k8sClient kubernetes.Interface, k8sInformer k8sinformers.SharedInformerFactory, ksInformer ksinformers.SharedInformerFactory) *Controller { // Create event broadcaster // Add sample-controller types to the default Kubernetes Scheme so Events can be // logged for sample-controller types. @@ -73,13 +74,13 @@ func NewController(k8sClient kubernetes.Interface, informerFactory informers.Sha eventBroadcaster.StartLogging(klog.Infof) eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: k8sClient.CoreV1().Events("")}) recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerName}) - informer := informerFactory.Rbac().V1().ClusterRoleBindings() + informer := k8sInformer.Rbac().V1().ClusterRoleBindings() ctl := &Controller{ k8sClient: k8sClient, informer: informer, lister: informer.Lister(), synced: informer.Informer().HasSynced, - kubectlOperator: kubectl.NewOperator(k8sClient, informerFactory), + kubectlOperator: kubectl.NewOperator(k8sClient, k8sInformer, ksInformer), workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "ClusterRoleBinding"), recorder: recorder, } diff --git a/pkg/kapis/resources/v1alpha2/handler.go b/pkg/kapis/resources/v1alpha2/handler.go index 2ee6b2a82..41fdf1bae 100644 --- a/pkg/kapis/resources/v1alpha2/handler.go +++ b/pkg/kapis/resources/v1alpha2/handler.go @@ -49,7 +49,8 @@ func newResourceHandler(k8sClient kubernetes.Interface, factory informers.Inform gitVerifier: git.NewGitVerifier(factory.KubernetesSharedInformerFactory()), registryGetter: registries.NewRegistryGetter(factory.KubernetesSharedInformerFactory()), kubeconfigOperator: kubeconfig.NewOperator(k8sClient, nil, masterURL), - kubectlOperator: kubectl.NewOperator(k8sClient, factory.KubernetesSharedInformerFactory()), + kubectlOperator: kubectl.NewOperator(k8sClient, factory.KubernetesSharedInformerFactory(), + factory.KubeSphereSharedInformerFactory()), } } diff --git a/pkg/models/kubeconfig/kubeconfig.go b/pkg/models/kubeconfig/kubeconfig.go index 3c84c62f6..de0df83dd 100644 --- a/pkg/models/kubeconfig/kubeconfig.go +++ b/pkg/models/kubeconfig/kubeconfig.go @@ -58,7 +58,6 @@ const ( type Interface interface { GetKubeConfig(username string) (string, error) CreateKubeConfig(user *iamv1alpha2.User) error - DelKubeConfig(username string) error UpdateKubeconfig(username string, certificate []byte) error } @@ -135,7 +134,7 @@ func (o *operator) CreateKubeConfig(user *iamv1alpha2.User) error { } cm := &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: configMapKind, APIVersion: configMapAPIVersion}, - ObjectMeta: metav1.ObjectMeta{Name: configName, Annotations: map[string]string{constants.UsernameAnnotationKey: user.Name}}, + ObjectMeta: metav1.ObjectMeta{Name: configName, Labels: map[string]string{constants.UsernameLabelKey: user.Name}}, Data: map[string]string{kubeconfigFileName: string(kubeconfig)}} err = controllerutil.SetControllerReference(user, cm, scheme.Scheme) @@ -188,18 +187,6 @@ func (o *operator) GetKubeConfig(username string) (string, error) { return string(data), nil } -func (o *operator) DelKubeConfig(username string) error { - configName := fmt.Sprintf(kubeconfigNameFormat, username) - - deletePolicy := metav1.DeletePropagationBackground - err := o.k8sclient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(configName, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) - if err != nil { - klog.Errorln(err) - return err - } - return nil -} - func (o *operator) createCSR(username string) ([]byte, error) { csrConfig := &certutil.Config{ CommonName: username, @@ -247,8 +234,8 @@ func (o *operator) createCSR(username string) ([]byte, error) { APIVersion: "certificates.k8s.io/v1beta1", }, ObjectMeta: metav1.ObjectMeta{ - Name: csrName, - Annotations: map[string]string{constants.UsernameAnnotationKey: username}, + Name: csrName, + Labels: map[string]string{constants.UsernameLabelKey: username}, }, Spec: certificatesv1beta1.CertificateSigningRequestSpec{ Request: csr, diff --git a/pkg/models/kubectl/kubectl.go b/pkg/models/kubectl/kubectl.go index 933af075c..9ff1f0cf0 100644 --- a/pkg/models/kubectl/kubectl.go +++ b/pkg/models/kubectl/kubectl.go @@ -20,18 +20,20 @@ package kubectl import ( "fmt" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/client-go/informers" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" + ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" "kubesphere.io/kubesphere/pkg/models" "math/rand" "os" - - appsv1 "k8s.io/api/apps/v1" - "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "kubesphere.io/kubesphere/pkg/constants" ) @@ -44,16 +46,16 @@ const ( type Interface interface { GetKubectlPod(username string) (models.PodInfo, error) CreateKubectlDeploy(username string) error - DeleteKubectlDeploy(username string) error } type operator struct { - k8sClient kubernetes.Interface - informers informers.SharedInformerFactory + k8sClient kubernetes.Interface + k8sInformer k8sinformers.SharedInformerFactory + ksInformer ksinformers.SharedInformerFactory } -func NewOperator(k8sClient kubernetes.Interface, informers informers.SharedInformerFactory) Interface { - return &operator{k8sClient: k8sClient, informers: informers} +func NewOperator(k8sClient kubernetes.Interface, k8sInformer k8sinformers.SharedInformerFactory, ksInformer ksinformers.SharedInformerFactory) Interface { + return &operator{k8sClient: k8sClient, k8sInformer: k8sInformer, ksInformer: ksInformer} } var DefaultImage = "kubesphere/kubectl:advanced-1.0.0" @@ -66,7 +68,7 @@ func init() { func (o *operator) GetKubectlPod(username string) (models.PodInfo, error) { deployName := fmt.Sprintf(deployNameFormat, username) - deploy, err := o.informers.Apps().V1().Deployments().Lister().Deployments(namespace).Get(deployName) + deploy, err := o.k8sInformer.Apps().V1().Deployments().Lister().Deployments(namespace).Get(deployName) if err != nil { klog.Errorln(err) return models.PodInfo{}, err @@ -74,7 +76,7 @@ func (o *operator) GetKubectlPod(username string) (models.PodInfo, error) { selectors := deploy.Spec.Selector.MatchLabels labelSelector := labels.Set(selectors).AsSelector() - pods, err := o.informers.Core().V1().Pods().Lister().Pods(namespace).List(labelSelector) + pods, err := o.k8sInformer.Core().V1().Pods().Lister().Pods(namespace).List(labelSelector) if err != nil { klog.Errorln(err) return models.PodInfo{}, err @@ -115,9 +117,20 @@ func selectCorrectPod(namespace string, pods []*v1.Pod) (kubectlPod *v1.Pod, err func (o *operator) CreateKubectlDeploy(username string) error { deployName := fmt.Sprintf(deployNameFormat, username) + user, err := o.ksInformer.Iam().V1alpha2().Users().Lister().Get(username) + + if err != nil { + klog.Error(err) + // ignore if user not exist + if errors.IsNotFound(err) { + return nil + } + return err + } + replica := int32(1) - selector := metav1.LabelSelector{MatchLabels: map[string]string{"username": username}} - deployment := appsv1.Deployment{ + selector := metav1.LabelSelector{MatchLabels: map[string]string{constants.UsernameLabelKey: username}} + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: deployName, }, @@ -127,7 +140,7 @@ func (o *operator) CreateKubectlDeploy(username string) error { Template: v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - "username": username, + constants.UsernameLabelKey: username, }, }, Spec: v1.PodSpec{ @@ -142,7 +155,14 @@ func (o *operator) CreateKubectlDeploy(username string) error { }, } - _, err := o.k8sClient.AppsV1().Deployments(namespace).Create(&deployment) + err = controllerutil.SetControllerReference(user, deployment, scheme.Scheme) + + if err != nil { + klog.Errorln(err) + return err + } + + _, err = o.k8sClient.AppsV1().Deployments(namespace).Create(deployment) if err != nil { if errors.IsAlreadyExists(err) { @@ -154,18 +174,3 @@ func (o *operator) CreateKubectlDeploy(username string) error { return nil } - -func (o *operator) DeleteKubectlDeploy(username string) error { - deployName := fmt.Sprintf(deployNameFormat, username) - - err := o.k8sClient.AppsV1().Deployments(namespace).Delete(deployName, metav1.NewDeleteOptions(0)) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - klog.Error(err) - return err - } - - return nil -}