admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix router update bug

Browse Source
This commit is contained in:
Jeff
2019-05-13 13:53:40 +08:00
committed by zryfish
parent 5462f51e65
commit 89f54da60a
5 changed files with 216 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cmd/controller-manager/app/controllers.go
View File

@@ -23,6 +23,8 @@ import (
"k8s.io/client-go/rest"
"kubesphere.io/kubesphere/pkg/controller/application"
"kubesphere.io/kubesphere/pkg/controller/destinationrule"
"kubesphere.io/kubesphere/pkg/controller/job"
//"kubesphere.io/kubesphere/pkg/controller/job"
"kubesphere.io/kubesphere/pkg/controller/virtualservice"
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -97,7 +99,7 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{
kubeClient,
applicationClient)
//jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient)
jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient)
servicemeshInformer.Start(stopCh)
istioInformer.Start(stopCh)
@@ -108,7 +110,7 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{
"virtualservice-controller": vsController,
"destinationrule-controller": drController,
"application-controller": apController,
//"job-controller": jobController,
"job-controller": jobController,
}
for name, ctrl := range controllers {

4
pkg/controller/add_namespace.go
View File

@@ -18,7 +18,9 @@
package controller
import "kubesphere.io/kubesphere/pkg/controller/namespace"
func init() {
// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
//AddToManagerFuncs = append(AddToManagerFuncs, namespace.Add)
AddToManagerFuncs = append(AddToManagerFuncs, namespace.Add)
}

3
pkg/controller/add_strategy.go
View File

@@ -21,9 +21,6 @@ import (
)
func init() {
// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
//AddToManagerFuncs = append(AddToManagerFuncs, strategy.Add)
// Add application to manager functions
AddToManagerFuncs = append(AddToManagerFuncs, application.Add)

4
pkg/controller/add_workspace.go
View File

@@ -18,7 +18,9 @@
package controller
import "kubesphere.io/kubesphere/pkg/controller/workspace"
func init() {
// AddToManagerFuncs is a list of functions to create controllers and add them to a manager.
//AddToManagerFuncs = append(AddToManagerFuncs, workspace.Add)
AddToManagerFuncs = append(AddToManagerFuncs, workspace.Add)
}

296
pkg/models/routers/routers.go
View File

@@ -19,9 +19,11 @@
package routers
import (
"fmt"
"github.com/golang/glog"
"io/ioutil"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"sort"
@@ -48,6 +50,37 @@ const (
SIDECAR_INJECT = "sidecar.istio.io/inject"
)
var routerTemplates map[string]runtime.Object
// Load yamls
func init() {
yamls, err := LoadYamls()
routerTemplates = make(map[string]runtime.Object, 2)
if err != nil {
glog.Error(err)
return
}
for _, f := range yamls {
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, _, err := decode([]byte(f), nil, nil)
if err != nil {
glog.Error(err)
continue
}
switch obj.(type) {
case *corev1.Service:
routerTemplates["SERVICE"] = obj
case *extensionsv1beta1.Deployment:
routerTemplates["DEPLOYMENT"] = obj
}
}
}
// get master node ip, if there are multiple master nodes,
// choose first one according by their names alphabetically
func getMasterNodeIp() string {
@@ -115,6 +148,12 @@ func GetAllRouters() ([]*corev1.Service, error) {
// Get router from a namespace
func GetRouter(namespace string) (*corev1.Service, error) {
service, err := getRouterService(namespace)
addLoadBalancerIp(service)
return service, err
}
func getRouterService(namespace string) (*corev1.Service, error) {
serviceName := constants.IngressControllerPrefix + namespace
serviceLister := informers.SharedInformerFactory().Core().V1().Services().Lister()
@@ -127,8 +166,6 @@ func GetRouter(namespace string) (*corev1.Service, error) {
glog.Error(err)
return nil, err
}
addLoadBalancerIp(service)
return service, nil
}
@@ -163,12 +200,6 @@ func LoadYamls() ([]string, error) {
// Create a ingress controller in a namespace
func CreateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
k8sClient := k8s.Client()
var router *corev1.Service
yamls, err := LoadYamls()
injectSidecar := false
if enabled, ok := annotations[SERVICEMESH_ENABLED]; ok {
if enabled == "true" {
@@ -176,74 +207,17 @@ func CreateRouter(namespace string, routerType corev1.ServiceType, annotations m
}
}
err := createRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, injectSidecar)
if err != nil {
glog.Error(err)
return nil, err
}
for _, f := range yamls {
decode := scheme.Codecs.UniversalDeserializer().Decode
obj, _, err := decode([]byte(f), nil, nil)
if err != nil {
glog.Error(err)
return router, err
}
switch obj.(type) {
case *corev1.Service:
service := obj.(*corev1.Service)
service.SetAnnotations(annotations)
service.Spec.Type = routerType
service.Name = constants.IngressControllerPrefix + namespace
// Add project selector
service.Labels["project"] = namespace
service.Spec.Selector["project"] = namespace
service, err := k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Create(service)
if err != nil {
glog.Error(err)
return nil, err
}
router = service
case *extensionsv1beta1.Deployment:
deployment := obj.(*extensionsv1beta1.Deployment)
deployment.Name = constants.IngressControllerPrefix + namespace
// Add project label
deployment.Spec.Selector.MatchLabels["project"] = namespace
deployment.Spec.Template.Labels["project"] = namespace
if injectSidecar {
if deployment.Spec.Template.Annotations == nil {
deployment.Spec.Template.Annotations = make(map[string]string, 0)
}
deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "true"
}
// Isolate namespace
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--watch-namespace="+namespace)
// Choose self as master
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--election-id="+deployment.Name)
if routerType == corev1.ServiceTypeLoadBalancer {
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--publish-service="+constants.IngressControllerNamespace+"/"+constants.IngressControllerPrefix+namespace)
} else {
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--report-node-internal-ip-address")
}
deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Create(deployment)
if err != nil {
glog.Error(err)
}
default:
//glog.Info("Default resource")
}
router, err := createRouterService(namespace, routerType, annotations)
if err != nil {
glog.Error(err)
_ = deleteRouterWorkload(namespace)
return nil, err
}
addLoadBalancerIp(router)
@@ -253,10 +227,85 @@ func CreateRouter(namespace string, routerType corev1.ServiceType, annotations m
// DeleteRouter is used to delete ingress controller related resources in namespace
// It will not delete ClusterRole resource cause it maybe used by other controllers
func DeleteRouter(namespace string) (*corev1.Service, error) {
err := deleteRouterWorkload(namespace)
if err != nil {
glog.Error(err)
}
router, err := deleteRouterService(namespace)
if err != nil {
glog.Error(err)
return router, err
}
return router, nil
}
func createRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
obj, ok := routerTemplates["SERVICE"]
if !ok {
glog.Error("service template not loaded")
return nil, fmt.Errorf("service template not loaded")
}
k8sClient := k8s.Client()
var err error
var router *corev1.Service
service := obj.(*corev1.Service)
service.SetAnnotations(annotations)
service.Spec.Type = routerType
service.Name = constants.IngressControllerPrefix + namespace
// Add project selector
service.Labels["project"] = namespace
service.Spec.Selector["project"] = namespace
service, err := k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Create(service)
if err != nil {
glog.Error(err)
return nil, err
}
return service, nil
}
func updateRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
k8sClient := k8s.Client()
service, err := getRouterService(namespace)
if err != nil {
glog.Error(err, "get router failed")
return service, err
}
service.Spec.Type = routerType
originalAnnotations := service.GetAnnotations()
for key, val := range annotations {
originalAnnotations[key] = val
}
service.SetAnnotations(originalAnnotations)
service, err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Update(service)
return service, err
}
func deleteRouterService(namespace string) (*corev1.Service, error) {
service, err := getRouterService(namespace)
if err != nil {
glog.Error(err)
return service, err
}
k8sClient := k8s.Client()
// delete controller service
serviceName := constants.IngressControllerPrefix + namespace
@@ -265,11 +314,63 @@ func DeleteRouter(namespace string) (*corev1.Service, error) {
err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Delete(serviceName, &deleteOptions)
if err != nil {
glog.Error(err)
return service, err
}
return service, nil
}
func createRouterWorkload(namespace string, publishService bool, servicemeshEnabled bool) error {
obj, ok := routerTemplates["DEPLOYMENT"]
if !ok {
glog.Error("Deployment template file not loaded")
return fmt.Errorf("deployment template file not loaded")
}
k8sClient := k8s.Client()
deployment := obj.(*extensionsv1beta1.Deployment)
deployment.Name = constants.IngressControllerPrefix + namespace
// Add project label
deployment.Spec.Selector.MatchLabels["project"] = namespace
deployment.Spec.Template.Labels["project"] = namespace
if servicemeshEnabled {
if deployment.Spec.Template.Annotations == nil {
deployment.Spec.Template.Annotations = make(map[string]string, 0)
}
deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "true"
}
// Isolate namespace
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--watch-namespace="+namespace)
// Choose self as master
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--election-id="+deployment.Name)
if publishService {
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--publish-service="+constants.IngressControllerNamespace+"/"+constants.IngressControllerPrefix+namespace)
} else {
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--report-node-internal-ip-address")
}
deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Create(deployment)
if err != nil {
glog.Error(err)
return err
}
return nil
}
func deleteRouterWorkload(namespace string) error {
k8sClient := k8s.Client()
deleteOptions := meta_v1.DeleteOptions{}
// delete controller deployment
deploymentName := constants.IngressControllerPrefix + namespace
err = k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Delete(deploymentName, &deleteOptions)
err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Delete(deploymentName, &deleteOptions)
if err != nil {
glog.Error(err)
}
@@ -280,45 +381,60 @@ func DeleteRouter(namespace string) (*corev1.Service, error) {
"app": "kubesphere",
"component": "ks-router",
"tier": "backend",
"project": deploymentName,
"project": namespace,
})
replicaSetLister := informers.SharedInformerFactory().Apps().V1().ReplicaSets().Lister()
replicaSets, err := replicaSetLister.ReplicaSets(constants.IngressControllerNamespace).List(selector)
if err == nil {
if err != nil {
glog.Error(err)
}
for i := range replicaSets {
err = k8sClient.AppsV1().ReplicaSets(constants.IngressControllerNamespace).Delete(replicaSets[i].Name, &deleteOptions)
glog.Error(err)
if err != nil {
glog.Error(err)
}
}
return router, nil
return nil
}
// Update Ingress Controller Service, change type from NodePort to Loadbalancer or vice versa.
// Update Ingress Controller Service, change type from NodePort to loadbalancer or vice versa.
func UpdateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
var router *corev1.Service
router, err := GetRouter(namespace)
router, err := getRouterService(namespace)
if err != nil {
glog.Error(err)
return router, nil
return router, err
}
router, err = DeleteRouter(namespace)
routerAnnotations := router.GetAnnotations()
if routerAnnotations[SERVICEMESH_ENABLED] != annotations[SERVICEMESH_ENABLED] || routerType != router.Spec.Type {
err = deleteRouterWorkload(namespace)
if err != nil {
glog.Error(err)
return router, err
}
enableServicemesh := annotations[SERVICEMESH_ENABLED] == "true"
err := createRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, enableServicemesh)
if err != nil {
glog.Error(err)
return router, err
}
}
newRouter, err := updateRouterService(namespace, routerType, annotations)
if err != nil {
glog.Error(err)
return newRouter, err
}
router, err = CreateRouter(namespace, routerType, annotations)
if err != nil {
glog.Error(err)
}
return router, nil
return newRouter, nil
}