create openpitrix runtime when new cluster added (#2204)

cmd/controller-manager/app/controllers.go

@@ -48,18 +48,21 @@ import (
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/kubefed/pkg/controller/util"
 )
 
-func AddControllers(
+func addControllers(
 	mgr manager.Manager,
 	client k8s.Client,
 	informerFactory informers.InformerFactory,
 	devopsClient devops.Interface,
 	s3Client s3.Interface,
+	openpitrixClient openpitrix.Client,
 	multiClusterEnabled bool,
+	networkPolicyEnabled bool,
 	stopCh <-chan struct{}) error {
 
 	kubernetesInformer := informerFactory.KubernetesSharedInformerFactory()
@@ -94,33 +97,37 @@ func AddControllers(
 
 	jobController := job.NewJobController(kubernetesInformer.Batch().V1().Jobs(), client.Kubernetes())
 
-	s2iBinaryController := s2ibinary.NewController(client.Kubernetes(),
-		client.KubeSphere(),
-		kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
-		s3Client,
-	)
+	var s2iBinaryController, s2iRunController, devopsProjectController, devopsPipelineController, devopsCredentialController manager.Runnable
+	if devopsClient != nil {
+		s2iBinaryController = s2ibinary.NewController(client.Kubernetes(),
+			client.KubeSphere(),
+			kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
+			s3Client,
+		)
 
-	s2iRunController := s2irun.NewS2iRunController(client.Kubernetes(),
-		client.KubeSphere(),
-		kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
-		kubesphereInformer.Devops().V1alpha1().S2iRuns())
+		s2iRunController = s2irun.NewS2iRunController(client.Kubernetes(),
+			client.KubeSphere(),
+			kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
+			kubesphereInformer.Devops().V1alpha1().S2iRuns())
 
-	devopsProjectController := devopsproject.NewController(client.Kubernetes(),
-		client.KubeSphere(), devopsClient,
-		informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
-		informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().DevOpsProjects(),
-	)
+		devopsProjectController = devopsproject.NewController(client.Kubernetes(),
+			client.KubeSphere(), devopsClient,
+			informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
+			informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().DevOpsProjects(),
+		)
 
-	devopsPipelineController := pipeline.NewController(client.Kubernetes(),
-		client.KubeSphere(),
-		devopsClient,
-		informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
-		informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().Pipelines())
+		devopsPipelineController = pipeline.NewController(client.Kubernetes(),
+			client.KubeSphere(),
+			devopsClient,
+			informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
+			informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().Pipelines())
 
-	devopsCredentialController := devopscredential.NewController(client.Kubernetes(),
-		devopsClient,
-		informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
-		informerFactory.KubernetesSharedInformerFactory().Core().V1().Secrets())
+		devopsCredentialController = devopscredential.NewController(client.Kubernetes(),
+			devopsClient,
+			informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
+			informerFactory.KubernetesSharedInformerFactory().Core().V1().Secrets())
+
+	}
 
 	storageCapabilityController := capability.NewController(
 		client.Kubernetes(),
@@ -223,22 +230,29 @@ func AddControllers(
 		kubesphereInformer.Iam().V1alpha2().RoleBases(), kubesphereInformer.Iam().V1alpha2().WorkspaceRoles(),
 		fedWorkspaceCache, fedWorkspaceCacheController, multiClusterEnabled)
 
-	clusterController := cluster.NewClusterController(
-		client.Kubernetes(),
-		client.Config(),
-		kubesphereInformer.Cluster().V1alpha1().Clusters(),
-		client.KubeSphere().ClusterV1alpha1().Clusters())
-
-	nsnpProvider, err := provider.NewNsNetworkPolicyProvider(client.Kubernetes(),
-		kubernetesInformer.Networking().V1().NetworkPolicies())
-	if err != nil {
-		return err
+	var clusterController manager.Runnable
+	if multiClusterEnabled {
+		clusterController = cluster.NewClusterController(
+			client.Kubernetes(),
+			client.Config(),
+			kubesphereInformer.Cluster().V1alpha1().Clusters(),
+			client.KubeSphere().ClusterV1alpha1().Clusters(),
+			openpitrixClient)
+	}
+
+	var nsnpController manager.Runnable
+	if networkPolicyEnabled {
+		nsnpProvider, err := provider.NewNsNetworkPolicyProvider(client.Kubernetes(), kubernetesInformer.Networking().V1().NetworkPolicies())
+		if err != nil {
+			return err
+		}
+
+		nsnpController = nsnetworkpolicy.NewNSNetworkPolicyController(client.Kubernetes(),
+			client.KubeSphere().NetworkV1alpha1(), kubesphereInformer.Network().V1alpha1().NamespaceNetworkPolicies(),
+			kubernetesInformer.Core().V1().Services(), kubernetesInformer.Core().V1().Nodes(),
+			kubesphereInformer.Tenant().V1alpha1().Workspaces(),
+			kubernetesInformer.Core().V1().Namespaces(), nsnpProvider)
 	}
-	nsnpController := nsnetworkpolicy.NewNSNetworkPolicyController(client.Kubernetes(),
-		client.KubeSphere().NetworkV1alpha1(), kubesphereInformer.Network().V1alpha1().NamespaceNetworkPolicies(),
-		kubernetesInformer.Core().V1().Services(), kubernetesInformer.Core().V1().Nodes(),
-		kubesphereInformer.Tenant().V1alpha1().Workspaces(),
-		kubernetesInformer.Core().V1().Namespaces(), nsnpProvider)
 
 	controllers := map[string]manager.Runnable{
 		"virtualservice-controller":     vsController,
@@ -274,6 +288,11 @@ func AddControllers(
 	}
 
 	for name, ctrl := range controllers {
+		if ctrl == nil {
+			klog.V(4).Infof("%s is not going to run due to dependent component disabled.", name)
+			continue
+		}
+
 		if err := mgr.Add(ctrl); err != nil {
 			klog.Error(err, "add controller to manager failed", "name", name)
 			return err

cmd/controller-manager/app/options/options.go

@@ -10,6 +10,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
 	"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
+	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"strings"
@@ -21,9 +22,11 @@ type KubeSphereControllerManagerOptions struct {
 	DevopsOptions       *jenkins.Options
 	S3Options           *s3.Options
 	OpenPitrixOptions   *openpitrix.Options
+	NetworkOptions      *network.Options
 	MultiClusterOptions *multicluster.Options
 	LeaderElect         bool
 	LeaderElection      *leaderelection.LeaderElectionConfig
+	WebhookCertDir      string
 }
 
 func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions {
@@ -32,13 +35,15 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions
 		DevopsOptions:       jenkins.NewDevopsOptions(),
 		S3Options:           s3.NewS3Options(),
 		OpenPitrixOptions:   openpitrix.NewOptions(),
+		NetworkOptions:      network.NewNetworkOptions(),
 		MultiClusterOptions: multicluster.NewOptions(),
 		LeaderElection: &leaderelection.LeaderElectionConfig{
 			LeaseDuration: 30 * time.Second,
 			RenewDeadline: 15 * time.Second,
 			RetryPeriod:   5 * time.Second,
 		},
-		LeaderElect: false,
+		LeaderElect:    false,
+		WebhookCertDir: "",
 	}
 
 	return s
@@ -58,6 +63,7 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
 	s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
 	s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
 	s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
+	s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
 	s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
 
 	fs := fss.FlagSet("leaderelection")
@@ -67,6 +73,11 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
 		"Whether to enable leader election. This field should be enabled when controller manager"+
 		"deployed with multiple replicas.")
 
+	fs.StringVar(&s.WebhookCertDir, "webhook-cert-dir", s.WebhookCertDir, ""+
+		"Certificate directory used to setup webhooks, need tls.crt and tls.key placed inside."+
+		"if not set, webhook server would look up the server key and certificate in"+
+		"{TempDir}/k8s-webhook-server/serving-certs")
+
 	kfs := fss.FlagSet("klog")
 	local := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(local)
@@ -84,6 +95,7 @@ func (s *KubeSphereControllerManagerOptions) Validate() []error {
 	errs = append(errs, s.KubernetesOptions.Validate()...)
 	errs = append(errs, s.S3Options.Validate()...)
 	errs = append(errs, s.OpenPitrixOptions.Validate()...)
+	errs = append(errs, s.NetworkOptions.Validate()...)
 	return errs
 }
 

cmd/controller-manager/app/server.go

@@ -40,6 +40,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/devops"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/utils/term"
 	"os"
@@ -58,6 +59,7 @@ func NewControllerManagerCommand() *cobra.Command {
 			DevopsOptions:       conf.DevopsOptions,
 			S3Options:           conf.S3Options,
 			OpenPitrixOptions:   conf.OpenPitrixOptions,
+			NetworkOptions:      conf.NetworkOptions,
 			MultiClusterOptions: conf.MultiClusterOptions,
 			LeaderElection:      s.LeaderElection,
 			LeaderElect:         s.LeaderElect,
@@ -111,6 +113,15 @@ func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 		}
 	}
 
+	var openpitrixClient openpitrix.Client
+	if s.OpenPitrixOptions != nil && !s.OpenPitrixOptions.IsEmpty() {
+		openpitrixClient, err = openpitrix.NewClient(s.OpenPitrixOptions)
+		if err != nil {
+			klog.Errorf("Failed to create openpitrix client %v", err)
+			return err
+		}
+	}
+
 	var s3Client s3.Interface
 	if s.S3Options != nil && len(s.S3Options.Endpoint) != 0 {
 		s3Client, err = s3.NewS3Client(s.S3Options)
@@ -125,7 +136,7 @@ func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 
 	run := func(ctx context.Context) {
 		klog.V(0).Info("setting up manager")
-		mgr, err := manager.New(kubernetesClient.Config(), manager.Options{})
+		mgr, err := manager.New(kubernetesClient.Config(), manager.Options{CertDir: s.WebhookCertDir})
 		if err != nil {
 			klog.Fatalf("unable to set up overall controller manager: %v", err)
 		}
@@ -146,7 +157,7 @@ func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 			klog.Fatal("Unable to create namespace controller")
 		}
 
-		if err := AddControllers(mgr, kubernetesClient, informerFactory, devopsClient, s3Client, s.MultiClusterOptions.Enable, stopCh); err != nil {
+		if err := addControllers(mgr, kubernetesClient, informerFactory, devopsClient, s3Client, openpitrixClient, s.MultiClusterOptions.Enable, s.NetworkOptions.EnableNetworkPolicy, stopCh); err != nil {
 			klog.Fatalf("unable to register controllers to the manager: %v", err)
 		}
 
@@ -190,9 +201,6 @@ func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{})
 	// add a uniquifier so that two processes on the same host don't accidentally both become active
 	id = id + "_" + string(uuid.NewUUID())
 
-	// TODO: change lockType to lease
-	// once we finished moving to Kubernetes v1.16+, we
-	// change lockType to lease
 	lock, err := resourcelock.New(resourcelock.LeasesResourceLock,
 		"kubesphere-system",
 		"ks-controller-manager",