96d2ac4112
1. refactor kubesphere dependency service client creation, we can disable dependency by config 2. dependencies can be configured by configuration file 3. refactor cmd package using cobra.Command, so we can use hypersphere to invoke command sepearately. Later we only need to build one image to contains all kubesphere core components. One command to rule them all! 4. live reloading configuration currently not implemented
469 lines
12 KiB
Go
469 lines
12 KiB
Go
/*
|
|
|
|
Copyright 2019 The KubeSphere Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
package routers
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/golang/glog"
|
|
"io/ioutil"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"kubesphere.io/kubesphere/pkg/simple/client"
|
|
"sort"
|
|
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"kubesphere.io/kubesphere/pkg/informers"
|
|
|
|
corev1 "k8s.io/api/core/v1"
|
|
extensionsv1beta1 "k8s.io/api/extensions/v1beta1"
|
|
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
|
|
"strings"
|
|
|
|
"kubesphere.io/kubesphere/pkg/constants"
|
|
)
|
|
|
|
// choose router node ip by labels, currently select master node
|
|
var RouterNodeIPLabelSelector = map[string]string{
|
|
"node-role.kubernetes.io/master": "",
|
|
}
|
|
|
|
const (
|
|
SERVICEMESH_ENABLED = "servicemesh.kubesphere.io/enabled"
|
|
SIDECAR_INJECT = "sidecar.istio.io/inject"
|
|
)
|
|
|
|
var routerTemplates map[string]runtime.Object
|
|
|
|
// Load yamls
|
|
func init() {
|
|
yamls, err := LoadYamls()
|
|
routerTemplates = make(map[string]runtime.Object, 2)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return
|
|
}
|
|
|
|
for _, f := range yamls {
|
|
decode := scheme.Codecs.UniversalDeserializer().Decode
|
|
obj, _, err := decode([]byte(f), nil, nil)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
continue
|
|
}
|
|
|
|
switch obj.(type) {
|
|
case *corev1.Service:
|
|
routerTemplates["SERVICE"] = obj
|
|
case *extensionsv1beta1.Deployment:
|
|
routerTemplates["DEPLOYMENT"] = obj
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
// get master node ip, if there are multiple master nodes,
|
|
// choose first one according by their names alphabetically
|
|
func getMasterNodeIp() string {
|
|
|
|
nodeLister := informers.SharedInformerFactory().Core().V1().Nodes().Lister()
|
|
selector := labels.SelectorFromSet(RouterNodeIPLabelSelector)
|
|
|
|
masters, err := nodeLister.List(selector)
|
|
sort.Slice(masters, func(i, j int) bool {
|
|
return strings.Compare(masters[i].Name, masters[j].Name) > 0
|
|
})
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return ""
|
|
}
|
|
|
|
if len(masters) == 0 {
|
|
return ""
|
|
} else {
|
|
for _, address := range masters[0].Status.Addresses {
|
|
if address.Type == corev1.NodeInternalIP {
|
|
return address.Address
|
|
}
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func addLoadBalancerIp(service *corev1.Service) {
|
|
|
|
if service == nil {
|
|
return
|
|
}
|
|
|
|
// append selected node ip as loadbalancer ingress ip
|
|
if service.Spec.Type != corev1.ServiceTypeLoadBalancer && len(service.Status.LoadBalancer.Ingress) == 0 {
|
|
rip := getMasterNodeIp()
|
|
if len(rip) == 0 {
|
|
glog.Info("can not get node ip")
|
|
return
|
|
}
|
|
|
|
gIngress := corev1.LoadBalancerIngress{
|
|
IP: rip,
|
|
}
|
|
|
|
service.Status.LoadBalancer.Ingress = append(service.Status.LoadBalancer.Ingress, gIngress)
|
|
}
|
|
}
|
|
|
|
func GetAllRouters() ([]*corev1.Service, error) {
|
|
|
|
selector := labels.SelectorFromSet(labels.Set{"app": "kubesphere", "component": "ks-router", "tier": "backend"})
|
|
serviceLister := informers.SharedInformerFactory().Core().V1().Services().Lister()
|
|
services, err := serviceLister.Services(constants.IngressControllerNamespace).List(selector)
|
|
|
|
for i := range services {
|
|
addLoadBalancerIp(services[i])
|
|
}
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
return services, nil
|
|
}
|
|
|
|
// Get router from a namespace
|
|
func GetRouter(namespace string) (*corev1.Service, error) {
|
|
service, err := getRouterService(namespace)
|
|
addLoadBalancerIp(service)
|
|
return service, err
|
|
}
|
|
|
|
func getRouterService(namespace string) (*corev1.Service, error) {
|
|
serviceName := constants.IngressControllerPrefix + namespace
|
|
|
|
serviceLister := informers.SharedInformerFactory().Core().V1().Services().Lister()
|
|
service, err := serviceLister.Services(constants.IngressControllerNamespace).Get(serviceName)
|
|
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return nil, errors.NewNotFound(corev1.Resource("service"), serviceName)
|
|
}
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
return service, nil
|
|
}
|
|
|
|
// Load all resource yamls
|
|
func LoadYamls() ([]string, error) {
|
|
|
|
var yamls []string
|
|
|
|
files, err := ioutil.ReadDir(constants.IngressControllerFolder)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
for _, file := range files {
|
|
if file.IsDir() || !strings.HasSuffix(file.Name(), ".yaml") {
|
|
continue
|
|
}
|
|
content, err := ioutil.ReadFile(constants.IngressControllerFolder + "/" + file.Name())
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
} else {
|
|
yamls = append(yamls, string(content))
|
|
}
|
|
}
|
|
|
|
return yamls, nil
|
|
}
|
|
|
|
// Create a ingress controller in a namespace
|
|
func CreateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
|
|
|
|
injectSidecar := false
|
|
if enabled, ok := annotations[SERVICEMESH_ENABLED]; ok {
|
|
if enabled == "true" {
|
|
injectSidecar = true
|
|
}
|
|
}
|
|
|
|
err := createOrUpdateRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, injectSidecar)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
router, err := createRouterService(namespace, routerType, annotations)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
_ = deleteRouterWorkload(namespace)
|
|
return nil, err
|
|
}
|
|
|
|
addLoadBalancerIp(router)
|
|
return router, nil
|
|
}
|
|
|
|
// DeleteRouter is used to delete ingress controller related resources in namespace
|
|
// It will not delete ClusterRole resource cause it maybe used by other controllers
|
|
func DeleteRouter(namespace string) (*corev1.Service, error) {
|
|
err := deleteRouterWorkload(namespace)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
}
|
|
|
|
router, err := deleteRouterService(namespace)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return router, err
|
|
}
|
|
return router, nil
|
|
}
|
|
|
|
func createRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
|
|
|
|
obj, ok := routerTemplates["SERVICE"]
|
|
if !ok {
|
|
glog.Error("service template not loaded")
|
|
return nil, fmt.Errorf("service template not loaded")
|
|
}
|
|
|
|
k8sClient := client.ClientSets().K8s().Kubernetes()
|
|
|
|
service := obj.(*corev1.Service)
|
|
|
|
service.SetAnnotations(annotations)
|
|
service.Spec.Type = routerType
|
|
service.Name = constants.IngressControllerPrefix + namespace
|
|
|
|
// Add project selector
|
|
service.Labels["project"] = namespace
|
|
|
|
service.Spec.Selector["project"] = namespace
|
|
|
|
service, err := k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Create(service)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
return service, nil
|
|
}
|
|
|
|
func updateRouterService(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
|
|
|
|
k8sClient := client.ClientSets().K8s().Kubernetes()
|
|
|
|
service, err := getRouterService(namespace)
|
|
if err != nil {
|
|
glog.Error(err, "get router failed")
|
|
return service, err
|
|
}
|
|
|
|
service.Spec.Type = routerType
|
|
|
|
originalAnnotations := service.GetAnnotations()
|
|
|
|
for key, val := range annotations {
|
|
originalAnnotations[key] = val
|
|
}
|
|
|
|
service.SetAnnotations(originalAnnotations)
|
|
|
|
service, err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Update(service)
|
|
|
|
return service, err
|
|
}
|
|
|
|
func deleteRouterService(namespace string) (*corev1.Service, error) {
|
|
|
|
service, err := getRouterService(namespace)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return service, err
|
|
}
|
|
|
|
k8sClient := client.ClientSets().K8s().Kubernetes()
|
|
|
|
// delete controller service
|
|
serviceName := constants.IngressControllerPrefix + namespace
|
|
deleteOptions := meta_v1.DeleteOptions{}
|
|
|
|
err = k8sClient.CoreV1().Services(constants.IngressControllerNamespace).Delete(serviceName, &deleteOptions)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return service, err
|
|
}
|
|
|
|
return service, nil
|
|
}
|
|
|
|
func createOrUpdateRouterWorkload(namespace string, publishService bool, servicemeshEnabled bool) error {
|
|
obj, ok := routerTemplates["DEPLOYMENT"]
|
|
if !ok {
|
|
glog.Error("Deployment template file not loaded")
|
|
return fmt.Errorf("deployment template file not loaded")
|
|
}
|
|
|
|
deployName := constants.IngressControllerPrefix + namespace
|
|
|
|
k8sClient := client.ClientSets().K8s().Kubernetes()
|
|
deployment, err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Get(deployName, meta_v1.GetOptions{})
|
|
|
|
createDeployment := true
|
|
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
deployment = obj.(*extensionsv1beta1.Deployment)
|
|
|
|
deployment.Name = constants.IngressControllerPrefix + namespace
|
|
|
|
// Add project label
|
|
deployment.Spec.Selector.MatchLabels["project"] = namespace
|
|
deployment.Spec.Template.Labels["project"] = namespace
|
|
|
|
// Isolate namespace
|
|
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--watch-namespace="+namespace)
|
|
|
|
// Choose self as master
|
|
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--election-id="+deployment.Name)
|
|
|
|
}
|
|
} else {
|
|
createDeployment = false
|
|
|
|
for i := range deployment.Spec.Template.Spec.Containers {
|
|
if deployment.Spec.Template.Spec.Containers[i].Name == "nginx-ingress-controller" {
|
|
var args []string
|
|
for j := range deployment.Spec.Template.Spec.Containers[i].Args {
|
|
if strings.HasPrefix("--publish-service", deployment.Spec.Template.Spec.Containers[i].Args[j]) ||
|
|
strings.HasPrefix("--report-node-internal-ip-address", deployment.Spec.Template.Spec.Containers[i].Args[j]) {
|
|
continue
|
|
}
|
|
args = append(args, deployment.Spec.Template.Spec.Containers[i].Args[j])
|
|
}
|
|
deployment.Spec.Template.Spec.Containers[i].Args = args
|
|
}
|
|
}
|
|
}
|
|
|
|
if deployment.Spec.Template.Annotations == nil {
|
|
deployment.Spec.Template.Annotations = make(map[string]string, 0)
|
|
}
|
|
if servicemeshEnabled {
|
|
deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "true"
|
|
} else {
|
|
deployment.Spec.Template.Annotations[SIDECAR_INJECT] = "false"
|
|
}
|
|
|
|
if publishService {
|
|
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--publish-service="+constants.IngressControllerNamespace+"/"+constants.IngressControllerPrefix+namespace)
|
|
} else {
|
|
deployment.Spec.Template.Spec.Containers[0].Args = append(deployment.Spec.Template.Spec.Containers[0].Args, "--report-node-internal-ip-address")
|
|
}
|
|
|
|
if createDeployment {
|
|
deployment, err = k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Create(deployment)
|
|
} else {
|
|
deployment, err = k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Update(deployment)
|
|
}
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func deleteRouterWorkload(namespace string) error {
|
|
k8sClient := client.ClientSets().K8s().Kubernetes()
|
|
|
|
deleteOptions := meta_v1.DeleteOptions{}
|
|
// delete controller deployment
|
|
deploymentName := constants.IngressControllerPrefix + namespace
|
|
err := k8sClient.ExtensionsV1beta1().Deployments(constants.IngressControllerNamespace).Delete(deploymentName, &deleteOptions)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
}
|
|
|
|
// delete replicaset if there are any
|
|
selector := labels.SelectorFromSet(
|
|
labels.Set{
|
|
"app": "kubesphere",
|
|
"component": "ks-router",
|
|
"tier": "backend",
|
|
"project": namespace,
|
|
})
|
|
replicaSetLister := informers.SharedInformerFactory().Apps().V1().ReplicaSets().Lister()
|
|
replicaSets, err := replicaSetLister.ReplicaSets(constants.IngressControllerNamespace).List(selector)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
}
|
|
|
|
for i := range replicaSets {
|
|
err = k8sClient.AppsV1().ReplicaSets(constants.IngressControllerNamespace).Delete(replicaSets[i].Name, &deleteOptions)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Update Ingress Controller Service, change type from NodePort to loadbalancer or vice versa.
|
|
func UpdateRouter(namespace string, routerType corev1.ServiceType, annotations map[string]string) (*corev1.Service, error) {
|
|
var router *corev1.Service
|
|
|
|
router, err := getRouterService(namespace)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return router, err
|
|
}
|
|
|
|
enableServicemesh := annotations[SERVICEMESH_ENABLED] == "true"
|
|
|
|
err = createOrUpdateRouterWorkload(namespace, routerType == corev1.ServiceTypeLoadBalancer, enableServicemesh)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return router, err
|
|
}
|
|
|
|
newRouter, err := updateRouterService(namespace, routerType, annotations)
|
|
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return newRouter, err
|
|
}
|
|
|
|
return newRouter, nil
|
|
}
|