diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index 862956ca9..d23f76e64 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -23,6 +23,8 @@ import ( "k8s.io/client-go/rest" "kubesphere.io/kubesphere/pkg/controller/application" "kubesphere.io/kubesphere/pkg/controller/destinationrule" + "kubesphere.io/kubesphere/pkg/controller/job" + //"kubesphere.io/kubesphere/pkg/controller/job" "kubesphere.io/kubesphere/pkg/controller/virtualservice" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -97,7 +99,7 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{ kubeClient, applicationClient) - //jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient) + jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient) servicemeshInformer.Start(stopCh) istioInformer.Start(stopCh) @@ -108,7 +110,7 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{ "virtualservice-controller": vsController, "destinationrule-controller": drController, "application-controller": apController, - //"job-controller": jobController, + "job-controller": jobController, } for name, ctrl := range controllers { diff --git a/pkg/controller/add_namespace.go b/pkg/controller/add_namespace.go index ca7256fcc..2e1a0b5d0 100644 --- a/pkg/controller/add_namespace.go +++ b/pkg/controller/add_namespace.go @@ -18,7 +18,9 @@ package controller +import "kubesphere.io/kubesphere/pkg/controller/namespace" + func init() { // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. - //AddToManagerFuncs = append(AddToManagerFuncs, namespace.Add) + AddToManagerFuncs = append(AddToManagerFuncs, namespace.Add) } diff --git a/pkg/controller/add_strategy.go b/pkg/controller/add_strategy.go index c45c78749..f4dff2624 100644 --- a/pkg/controller/add_strategy.go +++ b/pkg/controller/add_strategy.go @@ -21,9 +21,6 @@ import ( ) func init() { - // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. - //AddToManagerFuncs = append(AddToManagerFuncs, strategy.Add) - // Add application to manager functions AddToManagerFuncs = append(AddToManagerFuncs, application.Add) diff --git a/pkg/controller/add_workspace.go b/pkg/controller/add_workspace.go index 33d98a8e5..1194ff0a7 100644 --- a/pkg/controller/add_workspace.go +++ b/pkg/controller/add_workspace.go @@ -18,7 +18,9 @@ package controller +import "kubesphere.io/kubesphere/pkg/controller/workspace" + func init() { // AddToManagerFuncs is a list of functions to create controllers and add them to a manager. - //AddToManagerFuncs = append(AddToManagerFuncs, workspace.Add) + AddToManagerFuncs = append(AddToManagerFuncs, workspace.Add) } diff --git a/pkg/models/routers/routers.go b/pkg/models/routers/routers.go index 0b67d7664..48479421f 100644 --- a/pkg/models/routers/routers.go +++ b/pkg/models/routers/routers.go @@ -19,9 +19,11 @@ package routers import ( + "fmt" "github.com/golang/glog" "io/ioutil" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" "kubesphere.io/kubesphere/pkg/simple/client/k8s" "sort" @@ -48,6 +50,37 @@ const ( SIDECAR_INJECT = "sidecar.istio.io/inject" ) +var routerTemplates map[string]runtime.Object + +// Load yamls +func init() { + yamls, err := LoadYamls() + routerTemplates = make(map[string]runtime.Object, 2) + + if err != nil { + glog.Error(err) + return + } + + for _, f := range yamls { + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(f), nil, nil) + + if err != nil { + glog.Error(err) + continue + } + + switch obj.(type) { + case *corev1.Service: + routerTemplates["SERVICE"] = obj + case *extensionsv1beta1.Deployment: + routerTemplates["DEPLOYMENT"] = obj + } + } + +} + // get master node ip, if there are multiple master nodes, // choose first one according by their names alphabetically func getMasterNodeIp() string { @@ -115,6 +148,12 @@ func GetAllRouters() ([]*corev1.Service, error) { // Get router from a namespace func GetRouter(namespace string) (*corev1.Service, error) { + service, err := getRouterService(namespace) + addLoadBalancerIp(service) + return service, err +} + +func getRouterService(namespace string) (*corev1.Service, error) { serviceName := constants.IngressControllerPrefix + namespace serviceLister := informers.SharedInformerFactory().Core().V1().Services().Lister() @@ -127,8 +166,6 @@ func GetRouter(namespace string) (*corev1.Service, error) { glog.Error(err) return nil, err } - - addLoadBalancerIp(service) return service, nil } @@ -163,12 +200,6 @@ func LoadYamls() ([]string, error) { // Create a ingress controller in a namespace func CreateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) { - k8sClient := k8s.Client() - - var router *corev1.Service - - yamls, err := LoadYamls() - injectSidecar := false if enabled, ok := annotations[SERVICEMESH_ENABLED]; ok { if enabled == "true" { @@ -176,74 +207,17 @@ func CreateRouter(namespace string, routerType corev1.ServiceType, annotations m } } + err := createRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, injectSidecar) if err != nil { glog.Error(err) + return nil, err } - for _, f := range yamls { - decode := scheme.Codecs.UniversalDeserializer().Decode - obj, _, err := decode([]byte(f), nil, nil) - - if err != nil { - glog.Error(err) - return router, err - } - - switch obj.(type) { - case *corev1.Service: - service := obj.(*corev1.Service) - - service.SetAnnotations(annotations) - service.Spec.Type = routerType - service.Name = constants.IngressControllerPrefix + namespace - - // Add project selector - service.Labels["project"] = namespace - - service.Spec.Selector["project"] = namespace - - service, err := k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Create(service) - if err != nil { - glog.Error(err) - return nil, err - } - - router = service - - case *extensionsv1beta1.Deployment: - deployment := obj.(*extensionsv1beta1.Deployment) - deployment.Name = constants.IngressControllerPrefix + namespace - - // Add project label - deployment.Spec.Selector.MatchLabels["project"] = namespace - deployment.Spec.Template.Labels["project"] = namespace - - if injectSidecar { - if deployment.Spec.Template.Annotations == nil { - deployment.Spec.Template.Annotations = make(map[string]string, 0) - } - deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "true" - } - - // Isolate namespace - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--watch-namespace="+namespace) - - // Choose self as master - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--election-id="+deployment.Name) - - if routerType == corev1.ServiceTypeLoadBalancer { - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--publish-service="+constants.IngressControllerNamespace+"/"+constants.IngressControllerPrefix+namespace) - } else { - deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--report-node-internal-ip-address") - } - - deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Create(deployment) - if err != nil { - glog.Error(err) - } - default: - //glog.Info("Default resource") - } + router, err := createRouterService(namespace, routerType, annotations) + if err != nil { + glog.Error(err) + _ = deleteRouterWorkload(namespace) + return nil, err } addLoadBalancerIp(router) @@ -253,10 +227,85 @@ func CreateRouter(namespace string, routerType corev1.ServiceType, annotations m // DeleteRouter is used to delete ingress controller related resources in namespace // It will not delete ClusterRole resource cause it maybe used by other controllers func DeleteRouter(namespace string) (*corev1.Service, error) { + err := deleteRouterWorkload(namespace) + if err != nil { + glog.Error(err) + } + + router, err := deleteRouterService(namespace) + + if err != nil { + glog.Error(err) + return router, err + } + return router, nil +} + +func createRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) { + + obj, ok := routerTemplates["SERVICE"] + if !ok { + glog.Error("service template not loaded") + return nil, fmt.Errorf("service template not loaded") + } + k8sClient := k8s.Client() - var err error - var router *corev1.Service + service := obj.(*corev1.Service) + + service.SetAnnotations(annotations) + service.Spec.Type = routerType + service.Name = constants.IngressControllerPrefix + namespace + + // Add project selector + service.Labels["project"] = namespace + + service.Spec.Selector["project"] = namespace + + service, err := k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Create(service) + if err != nil { + glog.Error(err) + return nil, err + } + + return service, nil +} + +func updateRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) { + + k8sClient := k8s.Client() + + service, err := getRouterService(namespace) + if err != nil { + glog.Error(err, "get router failed") + return service, err + } + + service.Spec.Type = routerType + + originalAnnotations := service.GetAnnotations() + + for key, val := range annotations { + originalAnnotations[key] = val + } + + service.SetAnnotations(originalAnnotations) + + service, err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Update(service) + + return service, err +} + +func deleteRouterService(namespace string) (*corev1.Service, error) { + + service, err := getRouterService(namespace) + + if err != nil { + glog.Error(err) + return service, err + } + + k8sClient := k8s.Client() // delete controller service serviceName := constants.IngressControllerPrefix + namespace @@ -265,11 +314,63 @@ func DeleteRouter(namespace string) (*corev1.Service, error) { err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Delete(serviceName, &deleteOptions) if err != nil { glog.Error(err) + return service, err } + return service, nil +} + +func createRouterWorkload(namespace string, publishService bool, servicemeshEnabled bool) error { + obj, ok := routerTemplates["DEPLOYMENT"] + if !ok { + glog.Error("Deployment template file not loaded") + return fmt.Errorf("deployment template file not loaded") + } + + k8sClient := k8s.Client() + + deployment := obj.(*extensionsv1beta1.Deployment) + deployment.Name = constants.IngressControllerPrefix + namespace + + // Add project label + deployment.Spec.Selector.MatchLabels["project"] = namespace + deployment.Spec.Template.Labels["project"] = namespace + + if servicemeshEnabled { + if deployment.Spec.Template.Annotations == nil { + deployment.Spec.Template.Annotations = make(map[string]string, 0) + } + deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "true" + } + + // Isolate namespace + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--watch-namespace="+namespace) + + // Choose self as master + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--election-id="+deployment.Name) + + if publishService { + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--publish-service="+constants.IngressControllerNamespace+"/"+constants.IngressControllerPrefix+namespace) + } else { + deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--report-node-internal-ip-address") + } + + deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Create(deployment) + if err != nil { + glog.Error(err) + return err + } + + return nil +} + +func deleteRouterWorkload(namespace string) error { + k8sClient := k8s.Client() + + deleteOptions := meta_v1.DeleteOptions{} // delete controller deployment deploymentName := constants.IngressControllerPrefix + namespace - err = k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Delete(deploymentName, &deleteOptions) + err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Delete(deploymentName, &deleteOptions) if err != nil { glog.Error(err) } @@ -280,45 +381,60 @@ func DeleteRouter(namespace string) (*corev1.Service, error) { "app": "kubesphere", "component": "ks-router", "tier": "backend", - "project": deploymentName, + "project": namespace, }) replicaSetLister := informers.SharedInformerFactory().Apps().V1().ReplicaSets().Lister() replicaSets, err := replicaSetLister.ReplicaSets(constants.IngressControllerNamespace).List(selector) - if err == nil { + if err != nil { glog.Error(err) } for i := range replicaSets { err = k8sClient.AppsV1().ReplicaSets(constants.IngressControllerNamespace).Delete(replicaSets[i].Name, &deleteOptions) - glog.Error(err) + if err != nil { + glog.Error(err) + } } - return router, nil + return nil } -// Update Ingress Controller Service, change type from NodePort to Loadbalancer or vice versa. +// Update Ingress Controller Service, change type from NodePort to loadbalancer or vice versa. func UpdateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) { var router *corev1.Service - router, err := GetRouter(namespace) + router, err := getRouterService(namespace) if err != nil { glog.Error(err) - return router, nil + return router, err } - router, err = DeleteRouter(namespace) + routerAnnotations := router.GetAnnotations() + + if routerAnnotations[SERVICEMESH_ENABLED] != annotations[SERVICEMESH_ENABLED] || routerType != router.Spec.Type { + err = deleteRouterWorkload(namespace) + if err != nil { + glog.Error(err) + return router, err + } + + enableServicemesh := annotations[SERVICEMESH_ENABLED] == "true" + + err := createRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, enableServicemesh) + if err != nil { + glog.Error(err) + return router, err + } + } + + newRouter, err := updateRouterService(namespace, routerType, annotations) if err != nil { glog.Error(err) + return newRouter, err } - router, err = CreateRouter(namespace, routerType, annotations) - - if err != nil { - glog.Error(err) - } - - return router, nil + return newRouter, nil }