admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

583
cmd/controller-manager/app/controllers.go
View File

@@ -1,583 +0,0 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"fmt"
"time"
"github.com/kubesphere/pvc-autoresizer/runners"
"github.com/prometheus/common/config"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/kubefed/pkg/controller/util"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
"kubesphere.io/kubesphere/pkg/controller/alerting"
"kubesphere.io/kubesphere/pkg/controller/application"
"kubesphere.io/kubesphere/pkg/controller/certificatesigningrequest"
"kubesphere.io/kubesphere/pkg/controller/cluster"
"kubesphere.io/kubesphere/pkg/controller/clusterrolebinding"
"kubesphere.io/kubesphere/pkg/controller/destinationrule"
"kubesphere.io/kubesphere/pkg/controller/globalrole"
"kubesphere.io/kubesphere/pkg/controller/globalrolebinding"
"kubesphere.io/kubesphere/pkg/controller/group"
"kubesphere.io/kubesphere/pkg/controller/groupbinding"
"kubesphere.io/kubesphere/pkg/controller/helm"
"kubesphere.io/kubesphere/pkg/controller/job"
"kubesphere.io/kubesphere/pkg/controller/loginrecord"
"kubesphere.io/kubesphere/pkg/controller/namespace"
"kubesphere.io/kubesphere/pkg/controller/network/ippool"
"kubesphere.io/kubesphere/pkg/controller/network/nsnetworkpolicy"
"kubesphere.io/kubesphere/pkg/controller/network/nsnetworkpolicy/provider"
"kubesphere.io/kubesphere/pkg/controller/notification"
"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmapplication"
"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmcategory"
"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrelease"
"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrepo"
"kubesphere.io/kubesphere/pkg/controller/quota"
"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
"kubesphere.io/kubesphere/pkg/controller/storage/capability"
"kubesphere.io/kubesphere/pkg/controller/user"
"kubesphere.io/kubesphere/pkg/controller/virtualservice"
"kubesphere.io/kubesphere/pkg/controller/workspace"
"kubesphere.io/kubesphere/pkg/controller/workspacerole"
"kubesphere.io/kubesphere/pkg/controller/workspacerolebinding"
"kubesphere.io/kubesphere/pkg/controller/workspacetemplate"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/kubeconfig"
"kubesphere.io/kubesphere/pkg/simple/client/devops"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
ippoolclient "kubesphere.io/kubesphere/pkg/simple/client/network/ippool"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
)
var allControllers = []string{
"user",
"workspacetemplate",
"workspace",
"workspacerole",
"workspacerolebinding",
"namespace",
"helmrepo",
"helmcategory",
"helmapplication",
"helmapplicationversion",
"helmrelease",
"helm",
"application",
"serviceaccount",
"resourcequota",
"virtualservice",
"destinationrule",
"job",
"storagecapability",
"pvcautoresizer",
"workloadrestart",
"loginrecord",
"cluster",
"nsnp",
"ippool",
"csr",
"clusterrolebinding",
"fedglobalrolecache",
"globalrole",
"fedglobalrolebindingcache",
"globalrolebinding",
"groupbinding",
"group",
"notification",
"pvcworkloadrestarter",
"rulegroup",
"clusterrulegroup",
"globalrulegroup",
}
// setup all available controllers one by one
func addAllControllers(mgr manager.Manager, client k8s.Client, informerFactory informers.InformerFactory,
cmOptions *options.KubeSphereControllerManagerOptions,
stopCh <-chan struct{}) error {
var err error
////////////////////////////////////
// begin init necessary informers
////////////////////////////////////
kubernetesInformer := informerFactory.KubernetesSharedInformerFactory()
istioInformer := informerFactory.IstioSharedInformerFactory()
kubesphereInformer := informerFactory.KubeSphereSharedInformerFactory()
////////////////////////////////////
// end informers
////////////////////////////////////
////////////////////////////////////
// begin init necessary clients
////////////////////////////////////
kubeconfigClient := kubeconfig.NewOperator(client.Kubernetes(),
informerFactory.KubernetesSharedInformerFactory().Core().V1().ConfigMaps().Lister(),
client.Config())
var devopsClient devops.Interface
if cmOptions.DevopsOptions != nil && len(cmOptions.DevopsOptions.Host) != 0 {
devopsClient, err = jenkins.NewDevopsClient(cmOptions.DevopsOptions)
if err != nil {
return fmt.Errorf("failed to connect jenkins, please check jenkins status, error: %v", err)
}
}
var ldapClient ldapclient.Interface
// when there is no ldapOption, we set ldapClient as nil, which means we don't need to sync user info into ldap.
if cmOptions.LdapOptions != nil && len(cmOptions.LdapOptions.Host) != 0 {
if cmOptions.LdapOptions.Host == ldapclient.FAKE_HOST { // for debug only
ldapClient = ldapclient.NewSimpleLdap()
} else {
ldapClient, err = ldapclient.NewLdapClient(cmOptions.LdapOptions, stopCh)
if err != nil {
return fmt.Errorf("failed to connect to ldap service, please check ldap status, error: %v", err)
}
}
} else {
klog.Warning("ks-controller-manager starts without ldap provided, it will not sync user into ldap")
}
////////////////////////////////////
// end init clients
////////////////////////////////////
////////////////////////////////////////////////////////
// begin init controller and add to manager one by one
////////////////////////////////////////////////////////
// "user" controller
if cmOptions.IsControllerEnabled("user") {
userController := &user.Reconciler{
MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable,
MaxConcurrentReconciles: 4,
LdapClient: ldapClient,
DevopsClient: devopsClient,
KubeconfigClient: kubeconfigClient,
AuthenticationOptions: cmOptions.AuthenticationOptions,
}
addControllerWithSetup(mgr, "user", userController)
}
// "workspacetemplate" controller
if cmOptions.IsControllerEnabled("workspacetemplate") {
workspaceTemplateReconciler := &workspacetemplate.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
addControllerWithSetup(mgr, "workspacetemplate", workspaceTemplateReconciler)
}
// "workspace" controller
if cmOptions.IsControllerEnabled("workspace") {
workspaceReconciler := &workspace.Reconciler{}
addControllerWithSetup(mgr, "workspace", workspaceReconciler)
}
// "workspacerole" controller
if cmOptions.IsControllerEnabled("workspacerole") {
workspaceRoleReconciler := &workspacerole.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
addControllerWithSetup(mgr, "workspacerole", workspaceRoleReconciler)
}
// "workspacerolebinding" controller
if cmOptions.IsControllerEnabled("workspacerolebinding") {
workspaceRoleBindingReconciler := &workspacerolebinding.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
addControllerWithSetup(mgr, "workspacerolebinding", workspaceRoleBindingReconciler)
}
// "namespace" controller
if cmOptions.IsControllerEnabled("namespace") {
namespaceReconciler := &namespace.Reconciler{GatewayOptions: cmOptions.GatewayOptions}
addControllerWithSetup(mgr, "namespace", namespaceReconciler)
}
// "helmrepo" controller
if cmOptions.IsControllerEnabled("helmrepo") {
helmRepoReconciler := &helmrepo.ReconcileHelmRepo{}
addControllerWithSetup(mgr, "helmrepo", helmRepoReconciler)
}
// "helmcategory" controller
if cmOptions.IsControllerEnabled("helmcategory") {
helmCategoryReconciler := &helmcategory.ReconcileHelmCategory{}
addControllerWithSetup(mgr, "helmcategory", helmCategoryReconciler)
}
var opS3Client s3.Interface
if !cmOptions.OpenPitrixOptions.AppStoreConfIsEmpty() {
opS3Client, err = s3.NewS3Client(cmOptions.OpenPitrixOptions.S3Options)
if err != nil {
klog.Fatalf("failed to connect to s3, please check openpitrix s3 service status, error: %v", err)
}
// "helmapplication" controller
if cmOptions.IsControllerEnabled("helmapplication") {
reconcileHelmApp := &helmapplication.ReconcileHelmApplication{}
addControllerWithSetup(mgr, "helmapplication", reconcileHelmApp)
}
// "helmapplicationversion" controller
if cmOptions.IsControllerEnabled("helmapplicationversion") {
reconcileHelmAppVersion := &helmapplication.ReconcileHelmApplicationVersion{}
addControllerWithSetup(mgr, "helmapplicationversion", reconcileHelmAppVersion)
}
}
// "helmrelease" controller
if cmOptions.IsControllerEnabled("helmrelease") {
reconcileHelmRelease := &helmrelease.ReconcileHelmRelease{
// nil interface is valid value.
StorageClient: opS3Client,
KsFactory: informerFactory.KubeSphereSharedInformerFactory(),
MultiClusterEnable: cmOptions.MultiClusterOptions.Enable,
WaitTime: cmOptions.OpenPitrixOptions.ReleaseControllerOptions.WaitTime,
MaxConcurrent: cmOptions.OpenPitrixOptions.ReleaseControllerOptions.MaxConcurrent,
StopChan: stopCh,
}
addControllerWithSetup(mgr, "helmrelease", reconcileHelmRelease)
}
// "helm" controller
if cmOptions.IsControllerEnabled("helm") {
if !cmOptions.GatewayOptions.IsEmpty() {
helmReconciler := &helm.Reconciler{GatewayOptions: cmOptions.GatewayOptions}
addControllerWithSetup(mgr, "helm", helmReconciler)
}
}
// "application" controller
if cmOptions.IsControllerEnabled("application") {
selector, _ := labels.Parse(cmOptions.ApplicationSelector)
applicationReconciler := &application.ApplicationReconciler{
Scheme: mgr.GetScheme(),
Client: mgr.GetClient(),
Mapper: mgr.GetRESTMapper(),
ApplicationSelector: selector,
}
addControllerWithSetup(mgr, "application", applicationReconciler)
}
// "serviceaccount" controller
if cmOptions.IsControllerEnabled("serviceaccount") {
saReconciler := &serviceaccount.Reconciler{}
addControllerWithSetup(mgr, "serviceaccount", saReconciler)
}
// "resourcequota" controller
if cmOptions.IsControllerEnabled("resourcequota") {
resourceQuotaReconciler := &quota.Reconciler{
MaxConcurrentReconciles: quota.DefaultMaxConcurrentReconciles,
ResyncPeriod: quota.DefaultResyncPeriod,
InformerFactory: informerFactory.KubernetesSharedInformerFactory(),
}
addControllerWithSetup(mgr, "resourcequota", resourceQuotaReconciler)
}
serviceMeshEnabled := cmOptions.ServiceMeshOptions != nil && len(cmOptions.ServiceMeshOptions.IstioPilotHost) != 0
if serviceMeshEnabled {
// "virtualservice" controller
if cmOptions.IsControllerEnabled("virtualservice") {
vsController := virtualservice.NewVirtualServiceController(kubernetesInformer.Core().V1().Services(),
istioInformer.Networking().V1alpha3().VirtualServices(),
istioInformer.Networking().V1alpha3().DestinationRules(),
kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
client.Kubernetes(),
client.Istio(),
client.KubeSphere())
addController(mgr, "virtualservice", vsController)
}
// "destinationrule" controller
if cmOptions.IsControllerEnabled("destinationrule") {
drController := destinationrule.NewDestinationRuleController(kubernetesInformer.Apps().V1().Deployments(),
istioInformer.Networking().V1alpha3().DestinationRules(),
kubernetesInformer.Core().V1().Services(),
kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
client.Kubernetes(),
client.Istio(),
client.KubeSphere())
addController(mgr, "destinationrule", drController)
}
}
// "job" controller
if cmOptions.IsControllerEnabled("job") {
jobController := job.NewJobController(kubernetesInformer.Batch().V1().Jobs(), client.Kubernetes())
addController(mgr, "job", jobController)
}
// "storagecapability" controller
if cmOptions.IsControllerEnabled("storagecapability") {
storageCapabilityController := capability.NewController(
client.Kubernetes().StorageV1().StorageClasses(),
kubernetesInformer.Storage().V1().StorageClasses(),
kubernetesInformer.Storage().V1().CSIDrivers(),
)
addController(mgr, "storagecapability", storageCapabilityController)
}
// "pvcautoresizer" controller
monitoringOptionsEnable := cmOptions.MonitoringOptions != nil && len(cmOptions.MonitoringOptions.Endpoint) != 0
if monitoringOptionsEnable {
if cmOptions.IsControllerEnabled("pvcautoresizer") {
if err := runners.SetupIndexer(mgr, false); err != nil {
return err
}
promClient, err := runners.NewPrometheusClient(cmOptions.MonitoringOptions.Endpoint, &config.HTTPClientConfig{})
if err != nil {
return err
}
pvcAutoResizerController := runners.NewPVCAutoresizer(
promClient,
mgr.GetClient(),
ctrl.Log.WithName("pvcautoresizer"),
1*time.Minute,
mgr.GetEventRecorderFor("pvcautoresizer"),
)
addController(mgr, "pvcautoresizer", pvcAutoResizerController)
}
}
// "pvcworkloadrestarter" controller
if cmOptions.IsControllerEnabled("pvcworkloadrestarter") {
restarter := runners.NewRestarter(
mgr.GetClient(),
ctrl.Log.WithName("pvcworkloadrestarter"),
1*time.Minute,
mgr.GetEventRecorderFor("pvcworkloadrestarter"),
)
addController(mgr, "pvcworkloadrestarter", restarter)
}
// "loginrecord" controller
if cmOptions.IsControllerEnabled("loginrecord") {
loginRecordController := loginrecord.NewLoginRecordController(
client.Kubernetes(),
client.KubeSphere(),
kubesphereInformer.Iam().V1alpha2().LoginRecords(),
kubesphereInformer.Iam().V1alpha2().Users(),
cmOptions.AuthenticationOptions.LoginHistoryRetentionPeriod,
cmOptions.AuthenticationOptions.LoginHistoryMaximumEntries)
addController(mgr, "loginrecord", loginRecordController)
}
// "csr" controller
if cmOptions.IsControllerEnabled("csr") {
csrController := certificatesigningrequest.NewController(client.Kubernetes(),
kubernetesInformer.Certificates().V1().CertificateSigningRequests(),
kubernetesInformer.Core().V1().ConfigMaps(), client.Config())
addController(mgr, "csr", csrController)
}
// "clusterrolebinding" controller
if cmOptions.IsControllerEnabled("clusterrolebinding") {
clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(),
kubernetesInformer.Rbac().V1().ClusterRoleBindings(),
kubernetesInformer.Apps().V1().Deployments(),
kubernetesInformer.Core().V1().Pods(),
kubesphereInformer.Iam().V1alpha2().Users(),
cmOptions.AuthenticationOptions.KubectlImage)
addController(mgr, "clusterrolebinding", clusterRoleBindingController)
}
// "fedglobalrolecache" controller
var fedGlobalRoleCache cache.Store
var fedGlobalRoleCacheController cache.Controller
if cmOptions.IsControllerEnabled("fedglobalrolecache") {
if cmOptions.MultiClusterOptions.Enable {
fedGlobalRoleClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleResource)
if err != nil {
klog.Fatalf("Unable to create FedGlobalRole controller: %v", err)
}
fedGlobalRoleCache, fedGlobalRoleCacheController = util.NewResourceInformer(fedGlobalRoleClient, "",
&iamv1alpha2.FedGlobalRoleResource, func(object runtimeclient.Object) {})
go fedGlobalRoleCacheController.Run(stopCh)
addSuccessfullyControllers.Insert("fedglobalrolecache")
}
}
// "globalrole" controller
if cmOptions.IsControllerEnabled("globalrole") {
if cmOptions.MultiClusterOptions.Enable {
globalRoleController := globalrole.NewController(client.Kubernetes(), client.KubeSphere(),
kubesphereInformer.Iam().V1alpha2().GlobalRoles(), fedGlobalRoleCache, fedGlobalRoleCacheController)
addController(mgr, "globalrole", globalRoleController)
}
}
// "fedglobalrolebindingcache" controller
var fedGlobalRoleBindingCache cache.Store
var fedGlobalRoleBindingCacheController cache.Controller
if cmOptions.IsControllerEnabled("fedglobalrolebindingcache") {
if cmOptions.MultiClusterOptions.Enable {
fedGlobalRoleBindingClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleBindingResource)
if err != nil {
klog.Fatalf("Unable to create FedGlobalRoleBinding controller: %v", err)
}
fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController = util.NewResourceInformer(fedGlobalRoleBindingClient, "",
&iamv1alpha2.FedGlobalRoleBindingResource, func(object runtimeclient.Object) {})
go fedGlobalRoleBindingCacheController.Run(stopCh)
addSuccessfullyControllers.Insert("fedglobalrolebindingcache")
}
}
// "globalrolebinding" controller
if cmOptions.IsControllerEnabled("globalrolebinding") {
globalRoleBindingController := globalrolebinding.NewController(client.Kubernetes(), client.KubeSphere(),
kubesphereInformer.Iam().V1alpha2().GlobalRoleBindings(),
fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController,
cmOptions.MultiClusterOptions.Enable)
addController(mgr, "globalrolebinding", globalRoleBindingController)
}
// "groupbinding" controller
if cmOptions.IsControllerEnabled("groupbinding") {
groupBindingController := groupbinding.NewController(client.Kubernetes(), client.KubeSphere(),
kubesphereInformer.Iam().V1alpha2().GroupBindings(),
kubesphereInformer.Types().V1beta1().FederatedGroupBindings(),
cmOptions.MultiClusterOptions.Enable)
addController(mgr, "groupbinding", groupBindingController)
}
// "group" controller
if cmOptions.IsControllerEnabled("group") {
groupController := group.NewController(client.Kubernetes(), client.KubeSphere(),
kubesphereInformer.Iam().V1alpha2().Groups(),
kubesphereInformer.Types().V1beta1().FederatedGroups(),
cmOptions.MultiClusterOptions.Enable)
addController(mgr, "group", groupController)
}
// "cluster" controller
if cmOptions.IsControllerEnabled("cluster") {
if cmOptions.MultiClusterOptions.Enable {
clusterController := cluster.NewClusterController(
client.Kubernetes(),
client.KubeSphere(),
client.Config(),
kubesphereInformer.Cluster().V1alpha1().Clusters(),
kubesphereInformer.Iam().V1alpha2().Users().Lister(),
cmOptions.MultiClusterOptions.ClusterControllerResyncPeriod,
cmOptions.MultiClusterOptions.HostClusterName,
)
addController(mgr, "cluster", clusterController)
}
}
// "nsnp" controller
if cmOptions.IsControllerEnabled("nsnp") {
if cmOptions.NetworkOptions.EnableNetworkPolicy {
nsnpProvider, err := provider.NewNsNetworkPolicyProvider(client.Kubernetes(), kubernetesInformer.Networking().V1().NetworkPolicies())
if err != nil {
klog.Fatalf("Unable to create NSNetworkPolicy controller: %v", err)
}
nsnpController := nsnetworkpolicy.NewNSNetworkPolicyController(client.Kubernetes(),
client.KubeSphere().NetworkV1alpha1(),
kubesphereInformer.Network().V1alpha1().NamespaceNetworkPolicies(),
kubernetesInformer.Core().V1().Services(),
kubernetesInformer.Core().V1().Nodes(),
kubesphereInformer.Tenant().V1alpha1().Workspaces(),
kubernetesInformer.Core().V1().Namespaces(), nsnpProvider, cmOptions.NetworkOptions.NSNPOptions)
addController(mgr, "nsnp", nsnpController)
}
}
// "ippool" controller
if cmOptions.IsControllerEnabled("ippool") {
ippoolProvider := ippoolclient.NewProvider(kubernetesInformer, client.KubeSphere(), client.Kubernetes(),
cmOptions.NetworkOptions.IPPoolType, cmOptions.KubernetesOptions)
if ippoolProvider != nil {
ippoolController := ippool.NewIPPoolController(kubesphereInformer, kubernetesInformer, client.Kubernetes(),
client.KubeSphere(), ippoolProvider)
addController(mgr, "ippool", ippoolController)
}
}
// "notification" controller
if cmOptions.IsControllerEnabled("notification") {
if cmOptions.MultiClusterOptions.Enable {
notificationController, err := notification.NewController(client.Kubernetes(), mgr.GetClient(), mgr.GetCache())
if err != nil {
klog.Fatalf("Unable to create Notification controller: %v", err)
}
addController(mgr, "notification", notificationController)
}
}
// controllers for alerting
alertingOptionsEnable := cmOptions.AlertingOptions != nil && (cmOptions.AlertingOptions.PrometheusEndpoint != "" || cmOptions.AlertingOptions.ThanosRulerEndpoint != "")
if alertingOptionsEnable {
// "rulegroup" controller
if cmOptions.IsControllerEnabled("rulegroup") {
rulegroupReconciler := &alerting.RuleGroupReconciler{}
addControllerWithSetup(mgr, "rulegroup", rulegroupReconciler)
}
// "clusterrulegroup" controller
if cmOptions.IsControllerEnabled("clusterrulegroup") {
clusterrulegroupReconciler := &alerting.ClusterRuleGroupReconciler{}
addControllerWithSetup(mgr, "clusterrulegroup", clusterrulegroupReconciler)
}
// "globalrulegroup" controller
if cmOptions.IsControllerEnabled("globalrulegroup") {
globalrulegroupReconciler := &alerting.GlobalRuleGroupReconciler{}
addControllerWithSetup(mgr, "globalrulegroup", globalrulegroupReconciler)
}
}
// log all controllers process result
for _, name := range allControllers {
if cmOptions.IsControllerEnabled(name) {
if addSuccessfullyControllers.Has(name) {
klog.Infof("%s controller is enabled and added successfully.", name)
} else {
klog.Infof("%s controller is enabled but is not going to run due to its dependent component being disabled.", name)
}
} else {
klog.Infof("%s controller is disabled by controller selectors.", name)
}
}
return nil
}
var addSuccessfullyControllers = sets.New[string]()
type setupableController interface {
SetupWithManager(mgr ctrl.Manager) error
}
func addControllerWithSetup(mgr manager.Manager, name string, controller setupableController) {
if err := controller.SetupWithManager(mgr); err != nil {
klog.Fatalf("Unable to create %v controller: %v", name, err)
}
addSuccessfullyControllers.Insert(name)
}
func addController(mgr manager.Manager, name string, controller manager.Runnable) {
if err := mgr.Add(controller); err != nil {
klog.Fatalf("Unable to create %v controller: %v", name, err)
}
addSuccessfullyControllers.Insert(name)
}

56
cmd/controller-manager/app/helper.go
View File

@@ -1,56 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package app
import (
"context"
"fmt"
"net/http"
"time"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/klog/v2"
)
// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" before timeout.
func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error {
var lastErr error
err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
healthStatus := 0
result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do(context.Background()).StatusCode(&healthStatus)
if result.Error() != nil {
lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error())
return false, nil
}
if healthStatus != http.StatusOK {
content, _ := result.Raw()
lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content))
klog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content))
return false, nil
}
return true, nil
})
if err != nil {
return fmt.Errorf("%v: %v", err, lastErr)
}
return nil
}

276
cmd/controller-manager/app/options/options.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
@@ -22,52 +11,29 @@ import (
"strings"
"time"
"github.com/Masterminds/semver/v3"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/leaderelection"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
"kubesphere.io/kubesphere/pkg/simple/client/gateway"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/controller"
"kubesphere.io/kubesphere/pkg/controller/options"
"kubesphere.io/kubesphere/pkg/scheme"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
"kubesphere.io/kubesphere/pkg/simple/client/network"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)
type KubeSphereControllerManagerOptions struct {
KubernetesOptions *k8s.KubernetesOptions
DevopsOptions *jenkins.Options
AuthenticationOptions *authentication.Options
LdapOptions *ldapclient.Options
OpenPitrixOptions *openpitrix.Options
NetworkOptions *network.Options
MultiClusterOptions *multicluster.Options
ServiceMeshOptions *servicemesh.Options
GatewayOptions *gateway.Options
MonitoringOptions *prometheus.Options
AlertingOptions *alerting.Options
LeaderElect bool
LeaderElection *leaderelection.LeaderElectionConfig
WebhookCertDir string
// KubeSphere is using sigs.k8s.io/application as fundamental object to implement Application Management.
// There are other projects also built on sigs.k8s.io/application, when KubeSphere installed along side
// them, conflicts happen. So we leave an option to only reconcile applications matched with the given
// selector. Default will reconcile all applications.
// For example
// "kubesphere.io/creator=" means reconcile applications with this label key
// "!kubesphere.io/creator" means exclude applications with this key
ApplicationSelector string
type ControllerManagerOptions struct {
options.Options
LeaderElect bool
LeaderElection *leaderelection.LeaderElectionConfig
WebhookCertDir string
// ControllerGates is the list of controller gates to enable or disable controller.
// '*' means "all enabled by default controllers"
// 'foo' means "enable 'foo'"
@@ -78,49 +44,28 @@ type KubeSphereControllerManagerOptions struct {
// e.g. *,-foo, means "disable 'foo'"
ControllerGates []string
// Enable gops or not.
GOPSEnabled bool
DebugMode bool
}
func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions {
s := &KubeSphereControllerManagerOptions{
KubernetesOptions: k8s.NewKubernetesOptions(),
DevopsOptions: jenkins.NewDevopsOptions(),
LdapOptions: ldapclient.NewOptions(),
OpenPitrixOptions: openpitrix.NewOptions(),
NetworkOptions: network.NewNetworkOptions(),
MultiClusterOptions: multicluster.NewOptions(),
ServiceMeshOptions: servicemesh.NewServiceMeshOptions(),
AuthenticationOptions: authentication.NewOptions(),
GatewayOptions: gateway.NewGatewayOptions(),
AlertingOptions: alerting.NewAlertingOptions(),
func NewControllerManagerOptions() *ControllerManagerOptions {
return &ControllerManagerOptions{
LeaderElection: &leaderelection.LeaderElectionConfig{
LeaseDuration: 30 * time.Second,
RenewDeadline: 15 * time.Second,
RetryPeriod: 5 * time.Second,
},
LeaderElect: false,
WebhookCertDir: "",
ApplicationSelector: "",
ControllerGates: []string{"*"},
LeaderElect: false,
WebhookCertDir: "",
ControllerGates: []string{"*"},
}
return s
}
func (s *KubeSphereControllerManagerOptions) Flags(allControllerNameSelectors []string) cliflag.NamedFlagSets {
func (s *ControllerManagerOptions) Flags() cliflag.NamedFlagSets {
fss := cliflag.NamedFlagSets{}
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
s.AuthenticationOptions.AddFlags(fss.FlagSet("authentication"), s.AuthenticationOptions)
s.LdapOptions.AddFlags(fss.FlagSet("ldap"), s.LdapOptions)
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
s.GatewayOptions.AddFlags(fss.FlagSet("gateway"), s.GatewayOptions)
s.AlertingOptions.AddFlags(fss.FlagSet("alerting"), s.AlertingOptions)
fs := fss.FlagSet("leaderelection")
s.bindLeaderElectionFlags(s.LeaderElection, fs)
@@ -134,16 +79,12 @@ func (s *KubeSphereControllerManagerOptions) Flags(allControllerNameSelectors []
"{TempDir}/k8s-webhook-server/serving-certs")
gfs := fss.FlagSet("generic")
gfs.StringVar(&s.ApplicationSelector, "application-selector", s.ApplicationSelector, ""+
"Only reconcile application(sigs.k8s.io/application) objects match given selector, this could avoid conflicts with "+
"other projects built on top of sig-application. Default behavior is to reconcile all of application objects.")
gfs.StringSliceVar(&s.ControllerGates, "controllers", []string{"*"}, fmt.Sprintf(""+
"A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+
"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s",
strings.Join(allControllerNameSelectors, ", ")))
strings.Join(controller.Controllers.Keys(), ", ")))
gfs.BoolVar(&s.GOPSEnabled, "gops", s.GOPSEnabled, "Whether to enable gops or not. When enabled this option, "+
"controller-manager will listen on a random port on 127.0.0.1, then you can use the gops tool to list and diagnose the controller-manager currently running.")
gfs.BoolVar(&s.DebugMode, "debug", false, "Don't enable this if you don't know what it means.")
kfs := fss.FlagSet("klog")
local := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -157,26 +98,14 @@ func (s *KubeSphereControllerManagerOptions) Flags(allControllerNameSelectors []
}
// Validate Options and Genetic Options
func (s *KubeSphereControllerManagerOptions) Validate(allControllerNameSelectors []string) []error {
func (s *ControllerManagerOptions) Validate() []error {
var errs []error
errs = append(errs, s.DevopsOptions.Validate()...)
errs = append(errs, s.KubernetesOptions.Validate()...)
errs = append(errs, s.OpenPitrixOptions.Validate()...)
errs = append(errs, s.NetworkOptions.Validate()...)
errs = append(errs, s.LdapOptions.Validate()...)
errs = append(errs, s.MultiClusterOptions.Validate()...)
errs = append(errs, s.AlertingOptions.Validate()...)
// genetic option: application-selector
if len(s.ApplicationSelector) != 0 {
_, err := labels.Parse(s.ApplicationSelector)
if err != nil {
errs = append(errs, err)
}
}
errs = append(errs, s.ComposedAppOptions.Validate()...)
// genetic option: controllers, check all selectors are valid
allControllersNameSet := sets.New(allControllerNameSelectors...)
allControllersNameSet := sets.KeySet(controller.Controllers)
for _, selector := range s.ControllerGates {
if selector == "*" {
continue
@@ -186,29 +115,10 @@ func (s *KubeSphereControllerManagerOptions) Validate(allControllerNameSelectors
errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", selector))
}
}
return errs
}
// IsControllerEnabled check if a specified controller enabled or not.
func (s *KubeSphereControllerManagerOptions) IsControllerEnabled(name string) bool {
hasStar := false
for _, ctrl := range s.ControllerGates {
if ctrl == name {
return true
}
if ctrl == "-"+name {
return false
}
if ctrl == "*" {
hasStar = true
}
}
return hasStar
}
func (s *KubeSphereControllerManagerOptions) bindLeaderElectionFlags(l *leaderelection.LeaderElectionConfig, fs *pflag.FlagSet) {
func (s *ControllerManagerOptions) bindLeaderElectionFlags(l *leaderelection.LeaderElectionConfig, fs *pflag.FlagSet) {
fs.DurationVar(&l.LeaseDuration, "leader-elect-lease-duration", l.LeaseDuration, ""+
"The duration that non-leader candidates will wait after observing a leadership "+
"renewal until attempting to acquire leadership of a led but unrenewed leader "+
@@ -224,18 +134,118 @@ func (s *KubeSphereControllerManagerOptions) bindLeaderElectionFlags(l *leaderel
"of a leadership. This is only applicable if leader election is enabled.")
}
// MergeConfig merge new config without validation
// Merge new config without validation
// When misconfigured, the app should just crash directly
func (s *KubeSphereControllerManagerOptions) MergeConfig(cfg *controllerconfig.Config) {
s.KubernetesOptions = cfg.KubernetesOptions
s.DevopsOptions = cfg.DevopsOptions
s.AuthenticationOptions = cfg.AuthenticationOptions
s.LdapOptions = cfg.LdapOptions
s.OpenPitrixOptions = cfg.OpenPitrixOptions
s.NetworkOptions = cfg.NetworkOptions
s.MultiClusterOptions = cfg.MultiClusterOptions
s.ServiceMeshOptions = cfg.ServiceMeshOptions
s.GatewayOptions = cfg.GatewayOptions
s.MonitoringOptions = cfg.MonitoringOptions
s.AlertingOptions = cfg.AlertingOptions
func (s *ControllerManagerOptions) Merge(conf *config.Config) {
if conf == nil {
return
}
if conf.KubernetesOptions != nil {
s.KubernetesOptions = conf.KubernetesOptions
}
if conf.AuthenticationOptions != nil {
s.AuthenticationOptions = conf.AuthenticationOptions
}
if conf.MultiClusterOptions != nil {
s.MultiClusterOptions = conf.MultiClusterOptions
}
if conf.TerminalOptions != nil {
s.TerminalOptions = conf.TerminalOptions
}
if conf.TelemetryOptions != nil {
s.TelemetryOptions = conf.TelemetryOptions
}
if conf.HelmExecutorOptions != nil {
s.HelmExecutorOptions = conf.HelmExecutorOptions
}
if conf.ExtensionOptions != nil {
s.ExtensionOptions = conf.ExtensionOptions
}
if conf.KubeSphereOptions != nil {
s.KubeSphereOptions = conf.KubeSphereOptions
}
if conf.ComposedAppOptions != nil {
s.ComposedAppOptions = conf.ComposedAppOptions
}
if conf.S3Options != nil {
s.S3Options = conf.S3Options
}
}
func (s *ControllerManagerOptions) NewControllerManager() (*controller.Manager, error) {
cm := &controller.Manager{}
webhookServer := webhook.NewServer(webhook.Options{
CertDir: s.WebhookCertDir,
Port: 8443,
})
cmOptions := manager.Options{
Scheme: scheme.Scheme,
WebhookServer: webhookServer,
}
if s.LeaderElect {
cmOptions = manager.Options{
Scheme: scheme.Scheme,
WebhookServer: webhookServer,
LeaderElection: s.LeaderElect,
LeaderElectionNamespace: "kubesphere-system",
LeaderElectionID: "ks-controller-manager-leader-election",
LeaseDuration: &s.LeaderElection.LeaseDuration,
RetryPeriod: &s.LeaderElection.RetryPeriod,
RenewDeadline: &s.LeaderElection.RenewDeadline,
}
}
k8sClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
if err != nil {
return nil, fmt.Errorf("unable to create kubernetes client: %v", err)
}
k8sVersionInfo, err := k8sClient.Discovery().ServerVersion()
if err != nil {
return nil, fmt.Errorf("unable to fetch k8s version info: %v", err)
}
k8sVersion, err := semver.NewVersion(k8sVersionInfo.GitVersion)
if err != nil {
return nil, err
}
klog.V(0).Info("setting up manager")
ctrl.SetLogger(klog.NewKlogr())
// Use 8443 instead of 443 because we need root permission to bind port 443
mgr, err := manager.New(k8sClient.Config(), cmOptions)
if err != nil {
klog.Fatalf("unable to set up overall controller manager: %v", err)
}
clusterClient, err := clusterclient.NewClusterClientSet(mgr.GetCache())
if err != nil {
return nil, fmt.Errorf("unable to create cluster client: %v", err)
}
cm.K8sClient = k8sClient
cm.ClusterClient = clusterClient
cm.Options = s.Options
cm.IsControllerEnabled = s.IsControllerEnabled
cm.Manager = mgr
cm.K8sVersion = k8sVersion
return cm, nil
}
// IsControllerEnabled check if a specified controller enabled or not.
func (s *ControllerManagerOptions) IsControllerEnabled(name string) bool {
allowedAll := false
for _, controllerGate := range s.ControllerGates {
if controllerGate == name {
return true
}
if controllerGate == "-"+name {
return false
}
if controllerGate == "*" {
allowedAll = true
}
}
return allowedAll
}

20
cmd/controller-manager/app/options/options_test.go
View File

@@ -1,16 +1,8 @@
// Copyright 2022 The KubeSphere Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
import (
@@ -72,7 +64,7 @@ func TestIsControllerEnabled(t *testing.T) {
}
for _, tc := range testcases {
option := NewKubeSphereControllerManagerOptions()
option := NewControllerManagerOptions()
option.ControllerGates = tc.controllerFlags
actual := option.IsControllerEnabled(tc.controllerName)
assert.Equal(t, tc.expected, actual, "%v: expected %v, got %v", tc.name, tc.expected, actual)

321
cmd/controller-manager/app/server.go
View File

@@ -1,76 +1,127 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package app
import (
"context"
"fmt"
"os"
"github.com/google/gops/agent"
"github.com/spf13/cobra"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/runtime"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
"sigs.k8s.io/controller-runtime/pkg/webhook"
"sigs.k8s.io/controller-runtime/pkg/webhook/conversion"
alertingv2beta1 "kubesphere.io/api/alerting/v2beta1"
"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
"kubesphere.io/kubesphere/pkg/apis"
controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/controller"
"kubesphere.io/kubesphere/pkg/controller/application"
"kubesphere.io/kubesphere/pkg/controller/certificatesigningrequest"
"kubesphere.io/kubesphere/pkg/controller/cluster"
"kubesphere.io/kubesphere/pkg/controller/network/webhooks"
"kubesphere.io/kubesphere/pkg/controller/clusterlabel"
"kubesphere.io/kubesphere/pkg/controller/clusterrole"
"kubesphere.io/kubesphere/pkg/controller/clusterrolebinding"
ksconfig "kubesphere.io/kubesphere/pkg/controller/config"
"kubesphere.io/kubesphere/pkg/controller/conversion"
"kubesphere.io/kubesphere/pkg/controller/core"
"kubesphere.io/kubesphere/pkg/controller/extension"
"kubesphere.io/kubesphere/pkg/controller/globalrole"
"kubesphere.io/kubesphere/pkg/controller/globalrolebinding"
"kubesphere.io/kubesphere/pkg/controller/job"
"kubesphere.io/kubesphere/pkg/controller/k8sapplication"
"kubesphere.io/kubesphere/pkg/controller/ksserviceaccount"
"kubesphere.io/kubesphere/pkg/controller/kubeconfig"
"kubesphere.io/kubesphere/pkg/controller/kubectl"
"kubesphere.io/kubesphere/pkg/controller/loginrecord"
"kubesphere.io/kubesphere/pkg/controller/namespace"
"kubesphere.io/kubesphere/pkg/controller/quota"
storagewebhooks "kubesphere.io/kubesphere/pkg/controller/storage/webhooks"
"kubesphere.io/kubesphere/pkg/controller/role"
"kubesphere.io/kubesphere/pkg/controller/rolebinding"
"kubesphere.io/kubesphere/pkg/controller/roletemplate"
"kubesphere.io/kubesphere/pkg/controller/secret"
"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
"kubesphere.io/kubesphere/pkg/controller/storageclass"
"kubesphere.io/kubesphere/pkg/controller/telemetry"
"kubesphere.io/kubesphere/pkg/controller/user"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/utils/metrics"
"kubesphere.io/kubesphere/pkg/controller/workspace"
"kubesphere.io/kubesphere/pkg/controller/workspacerole"
"kubesphere.io/kubesphere/pkg/controller/workspacerolebinding"
"kubesphere.io/kubesphere/pkg/controller/workspacetemplate"
"kubesphere.io/kubesphere/pkg/utils/term"
"kubesphere.io/kubesphere/pkg/version"
)
func init() {
// core
runtime.Must(controller.Register(&core.ExtensionReconciler{}))
runtime.Must(controller.Register(&core.CategoryReconciler{}))
runtime.Must(controller.Register(&core.RepositoryReconciler{}))
runtime.Must(controller.Register(&core.InstallPlanReconciler{}))
runtime.Must(controller.Register(&core.InstallPlanWebhook{}))
// extension
runtime.Must(controller.Register(&extension.JSBundleWebhook{}))
runtime.Must(controller.Register(&extension.APIServiceWebhook{}))
runtime.Must(controller.Register(&extension.ReverseProxyWebhook{}))
runtime.Must(controller.Register(&extension.ExtensionEntryWebhook{}))
// rbac
runtime.Must(controller.Register(&globalrole.Reconciler{}))
runtime.Must(controller.Register(&globalrolebinding.Reconciler{}))
runtime.Must(controller.Register(&workspacerole.Reconciler{}))
runtime.Must(controller.Register(&workspacerolebinding.Reconciler{}))
runtime.Must(controller.Register(&clusterrole.Reconciler{}))
runtime.Must(controller.Register(&clusterrolebinding.Reconciler{}))
runtime.Must(controller.Register(&role.Reconciler{}))
runtime.Must(controller.Register(&rolebinding.Reconciler{}))
runtime.Must(controller.Register(&roletemplate.Reconciler{}))
runtime.Must(controller.Register(&namespace.Reconciler{}))
// user management
runtime.Must(controller.Register(&user.Reconciler{}))
runtime.Must(controller.Register(&user.Webhook{}))
runtime.Must(controller.Register(&loginrecord.Reconciler{}))
// multi cluster
runtime.Must(controller.Register(&cluster.Reconciler{}))
runtime.Must(controller.Register(&cluster.Webhook{}))
runtime.Must(controller.Register(&clusterlabel.Reconciler{}))
// multi tenancy
runtime.Must(controller.Register(&workspace.Reconciler{}))
runtime.Must(controller.Register(&workspacetemplate.Reconciler{}))
// kubesphere service account
runtime.Must(controller.Register(&ksserviceaccount.Reconciler{}))
runtime.Must(controller.Register(&ksserviceaccount.Webhook{}))
runtime.Must(controller.Register(&secret.ServiceAccountSecretReconciler{}))
// additional capabilities
runtime.Must(controller.Register(&serviceaccount.Reconciler{}))
runtime.Must(controller.Register(&job.Reconciler{}))
runtime.Must(controller.Register(&storageclass.Reconciler{}))
runtime.Must(controller.Register(&telemetry.Runnable{}))
runtime.Must(controller.Register(&ksconfig.Webhook{}))
runtime.Must(controller.Register(&conversion.Webhook{}))
// kubeconfig
runtime.Must(controller.Register(&kubeconfig.Reconciler{}))
runtime.Must(controller.Register(&certificatesigningrequest.Reconciler{}))
// resource quota
runtime.Must(controller.Register(&quota.Reconciler{}))
runtime.Must(controller.Register(&quota.Webhook{}))
// app store
runtime.Must(controller.Register(&application.AppReleaseReconciler{}))
runtime.Must(controller.Register(&application.RepoReconciler{}))
runtime.Must(controller.Register(&application.AppCategoryReconciler{}))
runtime.Must(controller.Register(&application.AppVersionReconciler{}))
// k8s application
runtime.Must(controller.Register(&k8sapplication.Reconciler{}))
// kubectl
runtime.Must(controller.Register(&kubectl.Reconciler{}))
}
func NewControllerManagerCommand() *cobra.Command {
s := options.NewKubeSphereControllerManagerOptions()
conf, err := controllerconfig.TryLoadFromDisk()
if err == nil {
// make sure LeaderElection is not nil
s = &options.KubeSphereControllerManagerOptions{
KubernetesOptions: conf.KubernetesOptions,
DevopsOptions: conf.DevopsOptions,
AuthenticationOptions: conf.AuthenticationOptions,
LdapOptions: conf.LdapOptions,
OpenPitrixOptions: conf.OpenPitrixOptions,
NetworkOptions: conf.NetworkOptions,
MultiClusterOptions: conf.MultiClusterOptions,
ServiceMeshOptions: conf.ServiceMeshOptions,
GatewayOptions: conf.GatewayOptions,
MonitoringOptions: conf.MonitoringOptions,
AlertingOptions: conf.AlertingOptions,
LeaderElection: s.LeaderElection,
LeaderElect: s.LeaderElect,
WebhookCertDir: s.WebhookCertDir,
}
s := options.NewControllerManagerOptions()
if conf, err := config.TryLoadFromDisk(); err == nil {
s.Merge(conf)
} else {
klog.Fatalf("Failed to load configuration from disk: %v", err)
}
@@ -78,33 +129,26 @@ func NewControllerManagerCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "controller-manager",
Long: `KubeSphere controller manager is a daemon that embeds the control loops shipped with KubeSphere.`,
Run: func(cmd *cobra.Command, args []string) {
if errs := s.Validate(allControllers); len(errs) != 0 {
klog.Error(utilerrors.NewAggregate(errs))
os.Exit(1)
RunE: func(cmd *cobra.Command, args []string) error {
if errs := s.Validate(); len(errs) != 0 {
return utilerrors.NewAggregate(errs)
}
if s.GOPSEnabled {
if s.DebugMode {
// Add agent to report additional information such as the current stack trace, Go version, memory stats, etc.
// Bind to a random port on address 127.0.0.1
if err := agent.Listen(agent.Options{}); err != nil {
klog.Fatal(err)
klog.Fatalln(err)
}
}
if err = Run(s, controllerconfig.WatchConfigChange(), signals.SetupSignalHandler()); err != nil {
klog.Error(err)
os.Exit(1)
}
return Run(signals.SetupSignalHandler(), s)
},
SilenceUsage: true,
}
fs := cmd.Flags()
namedFlagSets := s.Flags(allControllers)
namedFlagSets := s.Flags()
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
cmd.Flags().AddFlagSet(f)
}
usageFmt := "Usage:\n %s\n"
@@ -123,155 +167,16 @@ func NewControllerManagerCommand() *cobra.Command {
}
cmd.AddCommand(versionCmd)
return cmd
}
func Run(s *options.KubeSphereControllerManagerOptions, configCh <-chan controllerconfig.Config, ctx context.Context) error {
ictx, cancelFunc := context.WithCancel(context.TODO())
errCh := make(chan error)
defer close(errCh)
go func() {
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
// The ctx (signals.SetupSignalHandler()) is to control the entire program life cycle,
// The ictx(internal context) is created here to control the life cycle of the controller-manager(all controllers, sharedInformer, webhook etc.)
// when config changed, stop server and renew context, start new server
for {
select {
case <-ctx.Done():
cancelFunc()
return nil
case cfg := <-configCh:
cancelFunc()
s.MergeConfig(&cfg)
ictx, cancelFunc = context.WithCancel(context.TODO())
go func() {
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
case err := <-errCh:
cancelFunc()
return err
}
}
}
func run(s *options.KubeSphereControllerManagerOptions, ctx context.Context) error {
kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
func Run(ctx context.Context, s *options.ControllerManagerOptions) error {
cm, err := s.NewControllerManager()
if err != nil {
klog.Errorf("Failed to create kubernetes clientset %v", err)
return err
return fmt.Errorf("failed to create controller manager: %v", err)
}
informerFactory := informers.NewInformerFactories(
kubernetesClient.Kubernetes(),
kubernetesClient.KubeSphere(),
kubernetesClient.Istio(),
kubernetesClient.Snapshot(),
kubernetesClient.ApiExtensions(),
kubernetesClient.Prometheus())
mgrOptions := manager.Options{
CertDir: s.WebhookCertDir,
Port: 8443,
if err := cm.Run(ctx, controller.Controllers); err != nil {
return fmt.Errorf("failed to run controller manager: %v", err)
}
if s.LeaderElect {
mgrOptions = manager.Options{
CertDir: s.WebhookCertDir,
Port: 8443,
LeaderElection: s.LeaderElect,
LeaderElectionNamespace: "kubesphere-system",
LeaderElectionID: "ks-controller-manager-leader-election",
LeaseDuration: &s.LeaderElection.LeaseDuration,
RetryPeriod: &s.LeaderElection.RetryPeriod,
RenewDeadline: &s.LeaderElection.RenewDeadline,
}
}
klog.V(0).Info("setting up manager")
ctrl.SetLogger(klog.NewKlogr())
// Use 8443 instead of 443 cause we need root permission to bind port 443
mgr, err := manager.New(kubernetesClient.Config(), mgrOptions)
if err != nil {
klog.Fatalf("unable to set up overall controller manager: %v", err)
}
if err = apis.AddToScheme(mgr.GetScheme()); err != nil {
klog.Fatalf("unable add APIs to scheme: %v", err)
}
// register common meta types into schemas.
metav1.AddToGroupVersion(mgr.GetScheme(), metav1.SchemeGroupVersion)
// TODO(jeff): refactor config with CRD
// install all controllers
if err = addAllControllers(mgr,
kubernetesClient,
informerFactory,
s,
ctx.Done()); err != nil {
klog.Fatalf("unable to register controllers to the manager: %v", err)
}
// Start cache data after all informer is registered
klog.V(0).Info("Starting cache resource from apiserver...")
informerFactory.Start(ctx.Done())
// Setup webhooks
klog.V(2).Info("setting up webhook server")
hookServer := mgr.GetWebhookServer()
klog.V(2).Info("registering webhooks to the webhook server")
if s.IsControllerEnabled("cluster") && s.MultiClusterOptions.Enable {
hookServer.Register("/validate-cluster-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &cluster.ValidatingHandler{Client: mgr.GetClient()}})
}
hookServer.Register("/validate-email-iam-kubesphere-io-v1alpha2", &webhook.Admission{Handler: &user.EmailValidator{Client: mgr.GetClient()}})
hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}})
hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}})
pvcAdmission, err := storagewebhooks.NewAccessorHandler()
if err != nil {
klog.Fatalf("unable to create pvc admission: %v", err)
}
hookServer.Register("/persistentvolumeclaims", &webhook.Admission{Handler: pvcAdmission})
resourceQuotaAdmission, err := quota.NewResourceQuotaAdmission(mgr.GetClient(), mgr.GetScheme())
if err != nil {
klog.Fatalf("unable to create resource quota admission: %v", err)
}
hookServer.Register("/validate-quota-kubesphere-io-v1alpha2", &webhook.Admission{Handler: resourceQuotaAdmission})
hookServer.Register("/convert", &conversion.Webhook{})
rulegroup := alertingv2beta1.RuleGroup{}
if err := rulegroup.SetupWebhookWithManager(mgr); err != nil {
klog.Fatalf("Unable to setup RuleGroup webhook: %v", err)
}
clusterrulegroup := alertingv2beta1.ClusterRuleGroup{}
if err := clusterrulegroup.SetupWebhookWithManager(mgr); err != nil {
klog.Fatalf("Unable to setup ClusterRuleGroup webhook: %v", err)
}
globalrulegroup := alertingv2beta1.GlobalRuleGroup{}
if err := globalrulegroup.SetupWebhookWithManager(mgr); err != nil {
klog.Fatalf("Unable to setup GlobalRuleGroup webhook: %v", err)
}
klog.V(2).Info("registering metrics to the webhook server")
// Add an extra metric endpoint, so we can use the the same metric definition with ks-apiserver
// /kapis/metrics is independent of controller-manager's built-in /metrics
mgr.AddMetricsExtraHandler("/kapis/metrics", metrics.Handler())
klog.V(0).Info("Starting the controllers.")
if err = mgr.Start(ctx); err != nil {
klog.Fatalf("unable to run the manager: %v", err)
}
return nil
}

27
cmd/controller-manager/controller-manager.go
View File

@@ -1,31 +1,20 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package main
import (
"os"
"k8s.io/component-base/cli"
"kubesphere.io/kubesphere/cmd/controller-manager/app"
)
func main() {
command := app.NewControllerManagerCommand()
if err := command.Execute(); err != nil {
os.Exit(1)
}
cmd := app.NewControllerManagerCommand()
code := cli.Run(cmd)
os.Exit(code)
}

17
cmd/ks-apiserver/apiserver.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package main

244
cmd/ks-apiserver/app/options/options.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
@@ -22,76 +11,51 @@ import (
"fmt"
"net/http"
"strings"
"sync"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
"k8s.io/client-go/kubernetes/scheme"
"github.com/Masterminds/semver/v3"
"golang.org/x/net/context"
corev1 "k8s.io/api/core/v1"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"kubesphere.io/kubesphere/pkg/apis"
"kubesphere.io/kubesphere/pkg/apiserver"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/options"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/models/auth"
resourcev1beta1 "kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
"kubesphere.io/kubesphere/pkg/scheme"
genericoptions "kubesphere.io/kubesphere/pkg/server/options"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/metricsserver"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)
type ServerRunOptions struct {
ConfigFile string
type APIServerOptions struct {
options.Options
GenericServerRunOptions *genericoptions.ServerRunOptions
*apiserverconfig.Config
schemeOnce sync.Once
DebugMode bool
ConfigFile string
DebugMode bool
}
func NewServerRunOptions() *ServerRunOptions {
s := &ServerRunOptions{
func NewAPIServerOptions() *APIServerOptions {
return &APIServerOptions{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Config: apiserverconfig.New(),
schemeOnce: sync.Once{},
}
return s
}
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
func (s *APIServerOptions) Flags() (fss cliflag.NamedFlagSets) {
fs := fss.FlagSet("generic")
fs.BoolVar(&s.DebugMode, "debug", false, "Don't enable this if you don't know what it means.")
s.GenericServerRunOptions.AddFlags(fs, s.GenericServerRunOptions)
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
s.AuthenticationOptions.AddFlags(fss.FlagSet("authentication"), s.AuthenticationOptions)
s.AuthorizationOptions.AddFlags(fss.FlagSet("authorization"), s.AuthorizationOptions)
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), s.SonarQubeOptions)
s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), s.MonitoringOptions)
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), s.LoggingOptions)
s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
s.EventsOptions.AddFlags(fss.FlagSet("events"), s.EventsOptions)
s.AuditingOptions.AddFlags(fss.FlagSet("auditing"), s.AuditingOptions)
s.AlertingOptions.AddFlags(fss.FlagSet("alerting"), s.AlertingOptions)
fs = fss.FlagSet("klog")
local := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -105,78 +69,67 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
}
// NewAPIServer creates an APIServer instance using given options
func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIServer, error) {
func (s *APIServerOptions) NewAPIServer(ctx context.Context) (*apiserver.APIServer, error) {
apiServer := &apiserver.APIServer{
Config: s.Config,
Options: s.Options,
}
kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
if err != nil {
return nil, err
}
apiServer.KubernetesClient = kubernetesClient
ctrl.SetLogger(klog.NewKlogr())
informerFactory := informers.NewInformerFactories(kubernetesClient.Kubernetes(), kubernetesClient.KubeSphere(),
kubernetesClient.Istio(), kubernetesClient.Snapshot(), kubernetesClient.ApiExtensions(), kubernetesClient.Prometheus())
apiServer.InformerFactory = informerFactory
if s.MonitoringOptions == nil || len(s.MonitoringOptions.Endpoint) == 0 {
return nil, fmt.Errorf("moinitoring service address in configuration MUST not be empty, please check configmap/kubesphere-config in kubesphere-system namespace")
} else {
if apiServer.MonitoringClient, err = prometheus.NewPrometheus(s.MonitoringOptions); err != nil {
return nil, fmt.Errorf("failed to connect to prometheus, please check prometheus status, error: %v", err)
}
var err error
if apiServer.K8sClient, err = k8s.NewKubernetesClient(s.KubernetesOptions); err != nil {
return nil, fmt.Errorf("failed to create kubernetes client, error: %v", err)
}
apiServer.MetricsClient = metricsserver.NewMetricsClient(kubernetesClient.Kubernetes(), s.KubernetesOptions)
if s.LoggingOptions.Host != "" {
if apiServer.LoggingClient, err = esclient.NewClient(s.LoggingOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
}
}
if s.DevopsOptions.Host != "" {
if apiServer.DevopsClient, err = jenkins.NewDevopsClient(s.DevopsOptions); err != nil {
return nil, fmt.Errorf("failed to connect to jenkins, please check jenkins status, error: %v", err)
}
}
if s.SonarQubeOptions.Host != "" {
sonarClient, err := sonarqube.NewSonarQubeClient(s.SonarQubeOptions)
if err != nil {
return nil, fmt.Errorf("failed to connecto to sonarqube, please check sonarqube status, error: %v", err)
}
apiServer.SonarClient = sonarqube.NewSonar(sonarClient.SonarQube())
}
if apiServer.CacheClient, err = cache.New(s.CacheOptions, stopCh); err != nil {
if apiServer.CacheClient, err = cache.New(s.CacheOptions, ctx.Done()); err != nil {
return nil, fmt.Errorf("failed to create cache, error: %v", err)
}
if s.EventsOptions.Host != "" {
if apiServer.EventsClient, err = eventsclient.NewClient(s.EventsOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
if c, err := cluster.New(apiServer.K8sClient.Config(), func(options *cluster.Options) {
options.Scheme = scheme.Scheme
}); err != nil {
return nil, fmt.Errorf("unable to create controller runtime cluster: %v", err)
} else {
apiServer.RuntimeCache = c.GetCache()
key := "involvedObject.name"
indexerFunc := func(obj client.Object) []string {
e := obj.(*corev1.Event)
return []string{e.InvolvedObject.Name}
}
}
if s.AuditingOptions.Host != "" {
if apiServer.AuditingClient, err = auditingclient.NewClient(s.AuditingOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
if err = apiServer.RuntimeCache.IndexField(ctx, &corev1.Event{}, key, indexerFunc); err != nil {
klog.Fatalf("unable to create index field: %v", err)
}
apiServer.RuntimeClient = c.GetClient()
}
if s.AlertingOptions != nil && (s.AlertingOptions.PrometheusEndpoint != "" || s.AlertingOptions.ThanosRulerEndpoint != "") {
if apiServer.AlertingClient, err = alerting.NewRuleClient(s.AlertingOptions); err != nil {
return nil, fmt.Errorf("failed to init alerting client: %v", err)
}
apiServer.ResourceManager, err = resourcev1beta1.New(ctx, apiServer.RuntimeClient, apiServer.RuntimeCache)
if err != nil {
return nil, fmt.Errorf("unable to create resource manager: %v", err)
}
if s.Config.MultiClusterOptions.Enable {
apiServer.ClusterClient = clusterclient.NewClusterClient(informerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters())
if err := identityprovider.SharedIdentityProviderController.WatchConfigurationChanges(ctx, apiServer.RuntimeCache); err != nil {
return nil, fmt.Errorf("unable to setup identity provider: %v", err)
}
apiServer.OpenpitrixClient = openpitrixv1.NewOpenpitrixClient(informerFactory, apiServer.KubernetesClient.KubeSphere(), s.OpenPitrixOptions, apiServer.ClusterClient)
if apiServer.ClusterClient, err = clusterclient.NewClusterClientSet(apiServer.RuntimeCache); err != nil {
return nil, fmt.Errorf("unable to create cluster client: %v", err)
}
if apiServer.TokenOperator, err = auth.NewTokenOperator(apiServer.CacheClient, s.Options.AuthenticationOptions); err != nil {
return nil, fmt.Errorf("unable to create issuer: %v", err)
}
k8sVersionInfo, err := apiServer.K8sClient.Discovery().ServerVersion()
if err != nil {
return nil, fmt.Errorf("unable to fetch k8s version info: %v", err)
}
k8sVersion, err := semver.NewVersion(k8sVersionInfo.GitVersion)
if err != nil {
return nil, err
}
apiServer.K8sVersionInfo = k8sVersionInfo
apiServer.K8sVersion = k8sVersion
server := &http.Server{
Addr: fmt.Sprintf(":%d", s.GenericServerRunOptions.InsecurePort),
@@ -187,41 +140,46 @@ func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIS
if err != nil {
return nil, err
}
server.TLSConfig = &tls.Config{
Certificates: []tls.Certificate{certificate},
}
server.Addr = fmt.Sprintf(":%d", s.GenericServerRunOptions.SecurePort)
}
sch := scheme.Scheme
s.schemeOnce.Do(func() {
if err := apis.AddToScheme(sch); err != nil {
klog.Fatalf("unable add APIs to scheme: %v", err)
}
})
mapper, err := apiutil.NewDynamicRESTMapper(apiServer.KubernetesClient.Config())
if err != nil {
klog.Fatalf("unable create dynamic RESTMapper: %v", err)
}
apiServer.RuntimeCache, err = runtimecache.New(apiServer.KubernetesClient.Config(), runtimecache.Options{Scheme: sch, Mapper: mapper})
if err != nil {
klog.Fatalf("unable to create controller runtime cache: %v", err)
}
apiServer.RuntimeClient, err = runtimeclient.New(apiServer.KubernetesClient.Config(), runtimeclient.Options{Scheme: sch})
if err != nil {
klog.Fatalf("unable to create controller runtime client: %v", err)
}
apiServer.Issuer, err = token.NewIssuer(s.AuthenticationOptions)
if err != nil {
klog.Fatalf("unable to create issuer: %v", err)
}
apiServer.Server = server
return apiServer, nil
}
func (s *APIServerOptions) Merge(conf *config.Config) {
if conf == nil {
return
}
if conf.KubernetesOptions != nil {
s.KubernetesOptions = conf.KubernetesOptions
}
if conf.CacheOptions != nil {
s.CacheOptions = conf.CacheOptions
}
if conf.AuthenticationOptions != nil {
s.AuthenticationOptions = conf.AuthenticationOptions
}
if conf.AuthorizationOptions != nil {
s.AuthorizationOptions = conf.AuthorizationOptions
}
if conf.MultiClusterOptions != nil {
s.MultiClusterOptions = conf.MultiClusterOptions
}
if conf.AuditingOptions != nil {
s.AuditingOptions = conf.AuditingOptions
}
if conf.TerminalOptions != nil {
s.TerminalOptions = conf.TerminalOptions
}
if conf.S3Options != nil {
s.S3Options = conf.S3Options
}
if conf.ExperimentalOptions != nil {
s.ExperimentalOptions = conf.ExperimentalOptions
}
}

31
cmd/ks-apiserver/app/options/validation.go
View File

@@ -1,41 +1,18 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
// Validate validates server run options, to find
// options' misconfiguration
func (s *ServerRunOptions) Validate() []error {
func (s *APIServerOptions) Validate() []error {
var errors []error
errors = append(errors, s.GenericServerRunOptions.Validate()...)
errors = append(errors, s.DevopsOptions.Validate()...)
errors = append(errors, s.KubernetesOptions.Validate()...)
errors = append(errors, s.ServiceMeshOptions.Validate()...)
errors = append(errors, s.MonitoringOptions.Validate()...)
errors = append(errors, s.SonarQubeOptions.Validate()...)
errors = append(errors, s.S3Options.Validate()...)
errors = append(errors, s.OpenPitrixOptions.Validate()...)
errors = append(errors, s.NetworkOptions.Validate()...)
errors = append(errors, s.LoggingOptions.Validate()...)
errors = append(errors, s.AuthenticationOptions.Validate()...)
errors = append(errors, s.AuthorizationOptions.Validate()...)
errors = append(errors, s.EventsOptions.Validate()...)
errors = append(errors, s.AuditingOptions.Validate()...)
errors = append(errors, s.AlertingOptions.Validate()...)
return errors
}

89
cmd/ks-apiserver/app/server.go
View File

@@ -1,23 +1,13 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package app
import (
"context"
"errors"
"fmt"
"net/http"
@@ -26,28 +16,25 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
"kubesphere.io/kubesphere/cmd/ks-apiserver/app/options"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/utils/term"
"kubesphere.io/kubesphere/pkg/version"
)
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
// Load configuration from file
conf, err := apiserverconfig.TryLoadFromDisk()
if err == nil {
s.Config = conf
s := options.NewAPIServerOptions()
if conf, err := config.TryLoadFromDisk(); err == nil {
s.Merge(conf)
} else {
klog.Fatalf("Failed to load configuration from disk: %v", err)
}
cmd := &cobra.Command{
Use: "ks-apiserver",
Use: constants.KubeSphereAPIServerName,
Long: `The KubeSphere API server validates and configures data for the API objects.
The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
@@ -60,11 +47,11 @@ cluster's shared state through which all other components interact.`,
// Add agent to report additional information such as the current stack trace, Go version, memory stats, etc.
// Bind to a random port on address 127.0.0.1.
if err := agent.Listen(agent.Options{}); err != nil {
klog.Fatal(err)
klog.Fatalln(err)
}
}
return Run(s, apiserverconfig.WatchConfigChange(), signals.SetupSignalHandler())
return Run(signals.SetupSignalHandler(), s)
},
SilenceUsage: true,
}
@@ -78,7 +65,7 @@ cluster's shared state through which all other components interact.`,
usageFmt := "Usage:\n %s\n"
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
})
@@ -91,62 +78,20 @@ cluster's shared state through which all other components interact.`,
}
cmd.AddCommand(versionCmd)
return cmd
}
func Run(s *options.ServerRunOptions, configCh <-chan apiserverconfig.Config, ctx context.Context) error {
ictx, cancelFunc := context.WithCancel(context.TODO())
errCh := make(chan error)
defer close(errCh)
go func() {
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
// The ctx (signals.SetupSignalHandler()) is to control the entire program life cycle,
// The ictx(internal context) is created here to control the life cycle of the ks-apiserver(http server, sharedInformer etc.)
// when config change, stop server and renew context, start new server
for {
select {
case <-ctx.Done():
cancelFunc()
return nil
case cfg := <-configCh:
cancelFunc()
s.Config = &cfg
ictx, cancelFunc = context.WithCancel(context.TODO())
go func() {
if errs := s.Validate(); len(errs) != 0 {
for _, err := range errs {
errCh <- err
}
}
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
case err := <-errCh:
cancelFunc()
return err
}
}
}
func run(s *options.ServerRunOptions, ctx context.Context) error {
apiserver, err := s.NewAPIServer(ctx.Done())
func Run(ctx context.Context, s *options.APIServerOptions) error {
apiServer, err := s.NewAPIServer(ctx)
if err != nil {
return err
}
err = apiserver.PrepareRun(ctx.Done())
if err != nil {
if err = apiServer.PrepareRun(ctx.Done()); err != nil {
return err
}
err = apiserver.Run(ctx)
if err == http.ErrServerClosed {
if errors.Is(apiServer.Run(ctx), http.ErrServerClosed) {
return nil
}
return err