diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index 0c36bc033..1f87906c3 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -492,7 +492,9 @@ func addAllControllers(mgr manager.Manager, client k8s.Client, informerFactory i kubesphereInformer.Cluster().V1alpha1().Clusters(), client.KubeSphere().ClusterV1alpha1().Clusters(), cmOptions.MultiClusterOptions.ClusterControllerResyncPeriod, - cmOptions.MultiClusterOptions.HostClusterName) + cmOptions.MultiClusterOptions.HostClusterName, + kubernetesInformer.Core().V1().ConfigMaps(), + ) addController(mgr, "cluster", clusterController) } } diff --git a/pkg/apiserver/config/config.go b/pkg/apiserver/config/config.go index 780f153c9..fbc878fd9 100644 --- a/pkg/apiserver/config/config.go +++ b/pkg/apiserver/config/config.go @@ -22,17 +22,17 @@ import ( "strings" "sync" - "k8s.io/klog" - "github.com/fsnotify/fsnotify" - - "kubesphere.io/kubesphere/pkg/apiserver/authentication" - "kubesphere.io/kubesphere/pkg/apiserver/authorization" - "github.com/spf13/viper" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/klog" networkv1alpha1 "kubesphere.io/api/network/v1alpha1" + "kubesphere.io/kubesphere/pkg/apiserver/authentication" + "kubesphere.io/kubesphere/pkg/apiserver/authorization" + "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/models/terminal" "kubesphere.io/kubesphere/pkg/simple/client/alerting" "kubesphere.io/kubesphere/pkg/simple/client/auditing" @@ -363,3 +363,17 @@ func (conf *Config) stripEmptyOptions() { conf.GPUOptions = nil } } + +// GetFromConfigMap returns KubeSphere ruuning config by the given ConfigMap. +func GetFromConfigMap(cm *corev1.ConfigMap) (*Config, error) { + c := &Config{} + value, ok := cm.Data[constants.KubeSphereConfigMapDataKey] + if !ok { + return nil, fmt.Errorf("failed to get configmap kubesphere.yaml value") + } + + if err := yaml.Unmarshal([]byte(value), c); err != nil { + return nil, fmt.Errorf("failed to unmarshal value from configmap. err: %s", err) + } + return c, nil +} diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index f5433ebfd..7ca15b2d1 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -31,6 +31,8 @@ const ( IngressControllerNamespace = KubeSphereControlNamespace AdminUserName = "admin" IngressControllerPrefix = "kubesphere-router-" + KubeSphereConfigName = "kubesphere-config" + KubeSphereConfigMapDataKey = "kubesphere.yaml" ClusterNameLabelKey = "kubesphere.io/cluster" NameLabelKey = "kubesphere.io/name" diff --git a/pkg/controller/cluster/cluster_controller.go b/pkg/controller/cluster/cluster_controller.go index c85da289e..2b97602ed 100644 --- a/pkg/controller/cluster/cluster_controller.go +++ b/pkg/controller/cluster/cluster_controller.go @@ -28,6 +28,7 @@ import ( "sync" "time" + "gopkg.in/yaml.v2" v1 "k8s.io/api/core/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -37,6 +38,7 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" + coreinformers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -51,9 +53,12 @@ import ( clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1" + "kubesphere.io/kubesphere/pkg/apiserver/config" clusterclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned/typed/cluster/v1alpha1" clusterinformer "kubesphere.io/kubesphere/pkg/client/informers/externalversions/cluster/v1alpha1" clusterlister "kubesphere.io/kubesphere/pkg/client/listers/cluster/v1alpha1" + "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/simple/client/multicluster" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/version" ) @@ -169,6 +174,7 @@ func NewClusterController( clusterClient clusterclient.ClusterInterface, resyncPeriod time.Duration, hostClusterName string, + configmapInformer coreinformers.ConfigMapInformer, ) *clusterController { broadcaster := record.NewBroadcaster() @@ -201,6 +207,18 @@ func NewClusterController( DeleteFunc: c.addCluster, }, resyncPeriod) + configmapInformer.Informer().AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(oldObj, newObj interface{}) { + oldCM := oldObj.(*v1.ConfigMap) + newCM := newObj.(*v1.ConfigMap) + if oldCM.ResourceVersion == newCM.ResourceVersion { + return + } + // Update the clusterName field when the kubesphere-config configmap is updated. + c.syncClusterNameInConfigMap() + }, + }, resyncPeriod) + return c } @@ -635,9 +653,63 @@ func (c *clusterController) syncCluster(key string) error { } } + if err = c.setClusterNameInConfigMap(clusterDt.client, cluster.Name); err != nil { + return err + } + return nil } +func (c *clusterController) setClusterNameInConfigMap(client kubernetes.Interface, name string) error { + cm, err := client.CoreV1().ConfigMaps(constants.KubeSphereNamespace).Get(context.TODO(), constants.KubeSphereConfigName, metav1.GetOptions{}) + if err != nil { + return err + } + + configData, err := config.GetFromConfigMap(cm) + if err != nil { + return err + } + if configData.MultiClusterOptions == nil { + configData.MultiClusterOptions = &multicluster.Options{} + } + if configData.MultiClusterOptions.ClusterName == name { + return nil + } + + configData.MultiClusterOptions.ClusterName = name + newConfigData, err := yaml.Marshal(configData) + if err != nil { + return err + } + cm.Data[constants.KubeSphereConfigMapDataKey] = string(newConfigData) + if _, err = client.CoreV1().ConfigMaps(constants.KubeSphereNamespace).Update(context.TODO(), cm, metav1.UpdateOptions{}); err != nil { + return err + } + return nil +} + +func (c *clusterController) syncClusterNameInConfigMap() { + clusters, err := c.clusterLister.List(labels.Everything()) + if err != nil { + klog.Errorf("list clusters failed: %v", err) + return + } + + for _, cluster := range clusters { + clusterDt, ok := c.clusterMap[cluster.Name] + if !ok { + continue + } + if err = retry.RetryOnConflict(retry.DefaultRetry, func() (err error) { + return c.setClusterNameInConfigMap(clusterDt.client, cluster.Name) + }); err != nil { + klog.Errorf("update configmap %s failed: %v", constants.KubeSphereConfigName, err) + continue + } + } +} + func (c *clusterController) checkIfClusterIsHostCluster(memberClusterNodes *v1.NodeList) bool { hostNodes, err := c.client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{}) if err != nil { diff --git a/pkg/kapis/cluster/v1alpha1/handler.go b/pkg/kapis/cluster/v1alpha1/handler.go index 015817d83..3bbeedf59 100644 --- a/pkg/kapis/cluster/v1alpha1/handler.go +++ b/pkg/kapis/cluster/v1alpha1/handler.go @@ -29,7 +29,6 @@ import ( "time" "github.com/emicklei/go-restful" - "gopkg.in/yaml.v2" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -41,6 +40,7 @@ import ( "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/rest" + "kubesphere.io/api/cluster/v1alpha1" "kubesphere.io/kubesphere/pkg/api" @@ -49,16 +49,15 @@ import ( kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned" "kubesphere.io/kubesphere/pkg/client/informers/externalversions" clusterlister "kubesphere.io/kubesphere/pkg/client/listers/cluster/v1alpha1" + "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/version" ) const ( - defaultAgentImage = "kubesphere/tower:v1.0" - defaultTimeout = 10 * time.Second - KubesphereNamespace = "kubesphere-system" - KubeSphereConfigName = "kubesphere-config" - KubeSphereApiServer = "ks-apiserver" + defaultAgentImage = "kubesphere/tower:v1.0" + defaultTimeout = 10 * time.Second + KubeSphereApiServer = "ks-apiserver" ) var errClusterConnectionIsNotProxy = fmt.Errorf("cluster is not using proxy connection") @@ -404,7 +403,7 @@ func validateKubeSphereAPIServer(config *rest.Config) (*version.Info, error) { Transport: transport, } - response, err := client.Get(fmt.Sprintf("%s/api/v1/namespaces/%s/services/:%s:/proxy/kapis/version", config.Host, KubesphereNamespace, KubeSphereApiServer)) + response, err := client.Get(fmt.Sprintf("%s/api/v1/namespaces/%s/services/:%s:/proxy/kapis/version", config.Host, constants.KubeSphereNamespace, KubeSphereApiServer)) if err != nil { return nil, err } @@ -415,13 +414,13 @@ func validateKubeSphereAPIServer(config *rest.Config) (*version.Info, error) { response.Body = ioutil.NopCloser(bytes.NewBuffer(responseBytes)) if response.StatusCode != http.StatusOK { - return nil, fmt.Errorf("invalid response: %s , please make sure %s.%s.svc of member cluster is up and running", KubeSphereApiServer, KubesphereNamespace, responseBody) + return nil, fmt.Errorf("invalid response: %s , please make sure %s.%s.svc of member cluster is up and running", KubeSphereApiServer, constants.KubeSphereNamespace, responseBody) } ver := version.Info{} err = json.NewDecoder(response.Body).Decode(&ver) if err != nil { - return nil, fmt.Errorf("invalid response: %s , please make sure %s.%s.svc of member cluster is up and running", KubeSphereApiServer, KubesphereNamespace, responseBody) + return nil, fmt.Errorf("invalid response: %s , please make sure %s.%s.svc of member cluster is up and running", KubeSphereApiServer, constants.KubeSphereNamespace, responseBody) } return &ver, nil @@ -449,37 +448,20 @@ func (h *handler) validateMemberClusterConfiguration(clientSet kubernetes.Interf // getMemberClusterConfig returns KubeSphere running config by the given member cluster kubeconfig func (h *handler) getMemberClusterConfig(clientSet kubernetes.Interface) (*config.Config, error) { - memberCm, err := clientSet.CoreV1().ConfigMaps(KubesphereNamespace).Get(context.Background(), KubeSphereConfigName, metav1.GetOptions{}) + memberCm, err := clientSet.CoreV1().ConfigMaps(constants.KubeSphereNamespace).Get(context.Background(), constants.KubeSphereConfigName, metav1.GetOptions{}) if err != nil { return nil, err } - return getConfigFromCm(memberCm) + return config.GetFromConfigMap(memberCm) } // getHostClusterConfig returns KubeSphere running config from host cluster ConfigMap func (h *handler) getHostClusterConfig() (*config.Config, error) { - hostCm, err := h.configMapLister.ConfigMaps(KubesphereNamespace).Get(KubeSphereConfigName) + hostCm, err := h.configMapLister.ConfigMaps(constants.KubeSphereNamespace).Get(constants.KubeSphereConfigName) if err != nil { - return nil, fmt.Errorf("failed to get host cluster %s/configmap/%s, err: %s", KubesphereNamespace, KubeSphereConfigName, err) + return nil, fmt.Errorf("failed to get host cluster %s/configmap/%s, err: %s", constants.KubeSphereNamespace, constants.KubeSphereConfigName, err) } - return getConfigFromCm(hostCm) -} - -// getConfigFromCm returns KubeSphere ruuning config by the given ConfigMap. -func getConfigFromCm(cm *corev1.ConfigMap) (*config.Config, error) { - Config := config.New() - - value, ok := cm.Data["kubesphere.yaml"] - if !ok { - return nil, fmt.Errorf("failed to get %s/configmap/%s kubesphere.yaml value", KubesphereNamespace, KubeSphereConfigName) - } - - err := yaml.Unmarshal([]byte(value), Config) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal value from %s/configmap/%s. err: %s", KubesphereNamespace, KubeSphereConfigName, err) - } - - return Config, nil + return config.GetFromConfigMap(hostCm) } diff --git a/pkg/kapis/cluster/v1alpha1/handler_test.go b/pkg/kapis/cluster/v1alpha1/handler_test.go index 33597f919..85d159d2b 100644 --- a/pkg/kapis/cluster/v1alpha1/handler_test.go +++ b/pkg/kapis/cluster/v1alpha1/handler_test.go @@ -39,6 +39,7 @@ import ( "kubesphere.io/api/cluster/v1alpha1" "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake" + "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/utils/k8sutil" ) @@ -112,16 +113,16 @@ authentication: var hostCm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Namespace: KubesphereNamespace, - Name: KubeSphereConfigName, + Namespace: constants.KubeSphereNamespace, + Name: constants.KubeSphereConfigName, }, Data: hostMap, } var memberCm = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Namespace: KubesphereNamespace, - Name: KubeSphereConfigName, + Namespace: constants.KubeSphereNamespace, + Name: constants.KubeSphereConfigName, }, Data: memberMap, } @@ -465,14 +466,14 @@ func addMemberClusterResource(targetCm *corev1.ConfigMap, t *testing.T) { t.Fatal(err) } - _, err = c.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: KubesphereNamespace}}, metav1.CreateOptions{}) + _, err = c.CoreV1().Namespaces().Create(context.Background(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: constants.KubeSphereNamespace}}, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { t.Fatal(err) } - _, err = c.CoreV1().ConfigMaps(KubesphereNamespace).Create(context.Background(), targetCm, metav1.CreateOptions{}) + _, err = c.CoreV1().ConfigMaps(constants.KubeSphereNamespace).Create(context.Background(), targetCm, metav1.CreateOptions{}) if err != nil && errors.IsAlreadyExists(err) { - _, err = c.CoreV1().ConfigMaps(KubesphereNamespace).Update(context.Background(), targetCm, metav1.UpdateOptions{}) + _, err = c.CoreV1().ConfigMaps(constants.KubeSphereNamespace).Update(context.Background(), targetCm, metav1.UpdateOptions{}) if err != nil { t.Fatal(err) } @@ -486,7 +487,7 @@ func addMemberClusterResource(targetCm *corev1.ConfigMap, t *testing.T) { t.Fatal(err) } - _, err = c.AppsV1().Deployments(KubesphereNamespace).Create(context.Background(), &deploy, metav1.CreateOptions{}) + _, err = c.AppsV1().Deployments(constants.KubeSphereNamespace).Create(context.Background(), &deploy, metav1.CreateOptions{}) if err != nil && !errors.IsAlreadyExists(err) { t.Fatal(err) } diff --git a/pkg/simple/client/multicluster/options.go b/pkg/simple/client/multicluster/options.go index 7f9ede5fe..dfc5b9acc 100644 --- a/pkg/simple/client/multicluster/options.go +++ b/pkg/simple/client/multicluster/options.go @@ -53,6 +53,10 @@ type Options struct { // HostClusterName is the name of the control plane cluster, default set to host. HostClusterName string `json:"hostClusterName,omitempty" yaml:"hostClusterName,omitempty"` + + // ClusterName is the name of the current cluster, + // this value will be set by the cluster-controller and stored in the kubesphere-config configmap. + ClusterName string `json:"clusterName,omitempty" yaml:"clusterName,omitempty"` } // NewOptions returns a default nil options