This is a huge commit, it does following things: (#1942)
1. Remove ks-iam standalone binary, move it to ks-apiserver 2. Generate all devops apis inside kubesphere repository, no need to import s2ioperator. 3. Reorganize ldap code, make it more flexible to use.
This commit is contained in:
@@ -1,12 +1,18 @@
|
||||
package options
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"flag"
|
||||
"fmt"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/klog"
|
||||
"kubesphere.io/kubesphere/pkg/api/iam"
|
||||
"kubesphere.io/kubesphere/pkg/apiserver"
|
||||
genericoptions "kubesphere.io/kubesphere/pkg/server/options"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/cache"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/ldap"
|
||||
esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/mysql"
|
||||
@@ -14,6 +20,7 @@ import (
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/s3"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -29,6 +36,9 @@ type ServerRunOptions struct {
|
||||
S3Options *s3.Options
|
||||
OpenPitrixOptions *openpitrix.Options
|
||||
LoggingOptions *esclient.Options
|
||||
LdapOptions *ldap.Options
|
||||
CacheOptions *cache.Options
|
||||
AuthenticateOptions *iam.AuthenticationOptions
|
||||
}
|
||||
|
||||
func NewServerRunOptions() *ServerRunOptions {
|
||||
@@ -44,22 +54,28 @@ func NewServerRunOptions() *ServerRunOptions {
|
||||
S3Options: s3.NewS3Options(),
|
||||
OpenPitrixOptions: openpitrix.NewOptions(),
|
||||
LoggingOptions: esclient.NewElasticSearchOptions(),
|
||||
LdapOptions: ldap.NewOptions(),
|
||||
CacheOptions: cache.NewRedisOptions(),
|
||||
AuthenticateOptions: iam.NewAuthenticateOptions(),
|
||||
}
|
||||
|
||||
return &s
|
||||
}
|
||||
|
||||
func (s *ServerRunOptions) Flags(c *ServerRunOptions) (fss cliflag.NamedFlagSets) {
|
||||
s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic"), c.GenericServerRunOptions)
|
||||
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), c.KubernetesOptions)
|
||||
s.MySQLOptions.AddFlags(fss.FlagSet("mysql"), c.MySQLOptions)
|
||||
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), c.DevopsOptions)
|
||||
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), c.SonarQubeOptions)
|
||||
s.S3Options.AddFlags(fss.FlagSet("s3"), c.S3Options)
|
||||
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), c.OpenPitrixOptions)
|
||||
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), c.ServiceMeshOptions)
|
||||
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), c.MonitoringOptions)
|
||||
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), c.LoggingOptions)
|
||||
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
|
||||
s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic"), s.GenericServerRunOptions)
|
||||
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
|
||||
s.AuthenticateOptions.AddFlags(fss.FlagSet("authenticate"), s.AuthenticateOptions)
|
||||
s.MySQLOptions.AddFlags(fss.FlagSet("mysql"), s.MySQLOptions)
|
||||
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
|
||||
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), s.SonarQubeOptions)
|
||||
s.LdapOptions.AddFlags(fss.FlagSet("ldap"), s.LdapOptions)
|
||||
s.CacheOptions.AddFlags(fss.FlagSet("cache"), s.CacheOptions)
|
||||
s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
|
||||
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
|
||||
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
|
||||
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), s.MonitoringOptions)
|
||||
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), s.LoggingOptions)
|
||||
|
||||
fs := fss.FlagSet("klog")
|
||||
local := flag.NewFlagSet("klog", flag.ExitOnError)
|
||||
@@ -71,3 +87,62 @@ func (s *ServerRunOptions) Flags(c *ServerRunOptions) (fss cliflag.NamedFlagSets
|
||||
|
||||
return fss
|
||||
}
|
||||
|
||||
func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIServer, error) {
|
||||
kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
monitoringClient := prometheus.NewPrometheus(s.MonitoringOptions)
|
||||
|
||||
loggingClient, err := esclient.NewElasticsearch(s.LoggingOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s3Client, err := s3.NewS3Client(s.S3Options)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
devopsClient, err := jenkins.NewDevopsClient(s.DevopsOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ldapClient, err := ldap.NewLdapClient(s.LdapOptions, stopCh)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cacheClient, err := cache.NewRedisClient(s.CacheOptions, stopCh)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: fmt.Sprintf(":%d", s.GenericServerRunOptions.InsecurePort),
|
||||
}
|
||||
|
||||
if s.GenericServerRunOptions.SecurePort != 0 {
|
||||
certificate, err := tls.LoadX509KeyPair(s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
server.TLSConfig.Certificates = []tls.Certificate{certificate}
|
||||
}
|
||||
|
||||
apiServer := &apiserver.APIServer{
|
||||
Server: server,
|
||||
KubernetesClient: kubernetesClient,
|
||||
MonitoringClient: monitoringClient,
|
||||
LoggingClient: loggingClient,
|
||||
S3Client: s3Client,
|
||||
DevopsClient: devopsClient,
|
||||
LdapClient: ldapClient,
|
||||
CacheClient: cacheClient,
|
||||
}
|
||||
|
||||
return apiServer, nil
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ package options
|
||||
func (s *ServerRunOptions) Validate() []error {
|
||||
var errors []error
|
||||
|
||||
errors = append(errors, s.GenericServerRunOptions.Validate()...)
|
||||
errors = append(errors, s.DevopsOptions.Validate()...)
|
||||
errors = append(errors, s.KubernetesOptions.Validate()...)
|
||||
errors = append(errors, s.MySQLOptions.Validate()...)
|
||||
|
||||
@@ -21,31 +21,35 @@ import (
|
||||
"fmt"
|
||||
kconfig "github.com/kiali/kiali/config"
|
||||
"github.com/spf13/cobra"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
utilerrors "k8s.io/apimachinery/pkg/util/errors"
|
||||
cliflag "k8s.io/component-base/cli/flag"
|
||||
"k8s.io/klog"
|
||||
"kubesphere.io/kubesphere/cmd/ks-apiserver/app/options"
|
||||
"kubesphere.io/kubesphere/pkg/apiserver"
|
||||
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
|
||||
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
|
||||
"kubesphere.io/kubesphere/pkg/apiserver/servicemesh/tracing"
|
||||
kinformers "kubesphere.io/kubesphere/pkg/informers"
|
||||
"kubesphere.io/kubesphere/pkg/kapis"
|
||||
"kubesphere.io/kubesphere/pkg/server"
|
||||
apiserverconfig "kubesphere.io/kubesphere/pkg/server/config"
|
||||
"kubesphere.io/kubesphere/pkg/server/filter"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/s3"
|
||||
"kubesphere.io/kubesphere/pkg/utils/signals"
|
||||
"kubesphere.io/kubesphere/pkg/utils/term"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func NewAPIServerCommand() *cobra.Command {
|
||||
s := options.NewServerRunOptions()
|
||||
|
||||
// Load configuration from file
|
||||
conf, err := apiserverconfig.TryLoadFromDisk()
|
||||
if err == nil {
|
||||
s = &options.ServerRunOptions{
|
||||
KubernetesOptions: conf.KubernetesOptions,
|
||||
DevopsOptions: conf.DevopsOptions,
|
||||
SonarQubeOptions: conf.SonarQubeOptions,
|
||||
ServiceMeshOptions: conf.ServiceMeshOptions,
|
||||
MySQLOptions: conf.MySQLOptions,
|
||||
MonitoringOptions: conf.MonitoringOptions,
|
||||
S3Options: conf.S3Options,
|
||||
OpenPitrixOptions: conf.OpenPitrixOptions,
|
||||
LoggingOptions: conf.LoggingOptions,
|
||||
AuthenticateOptions: conf.AuthenticateOptions,
|
||||
}
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
Use: "ks-apiserver",
|
||||
Long: `The KubeSphere API server validates and configures data for the api objects.
|
||||
@@ -60,11 +64,8 @@ cluster's shared state through which all other components interact.`,
|
||||
},
|
||||
}
|
||||
|
||||
configOptions := load()
|
||||
|
||||
fs := cmd.Flags()
|
||||
namedFlagSets := s.Flags(configOptions)
|
||||
|
||||
namedFlagSets := s.Flags()
|
||||
for _, f := range namedFlagSets.FlagSets {
|
||||
fs.AddFlagSet(f)
|
||||
}
|
||||
@@ -80,24 +81,19 @@ cluster's shared state through which all other components interact.`,
|
||||
|
||||
func Run(s *options.ServerRunOptions, stopCh <-chan struct{}) error {
|
||||
|
||||
err := CreateClientSet(apiserverconfig.Get(), stopCh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = WaitForResourceSync(stopCh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
initializeServicemeshConfig(s)
|
||||
|
||||
err = CreateAPIServer(s)
|
||||
apiserver, err := s.NewAPIServer(stopCh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
err = apiserver.PrepareRun()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return apiserver.Run(stopCh)
|
||||
}
|
||||
|
||||
func initializeServicemeshConfig(s *options.ServerRunOptions) {
|
||||
@@ -119,234 +115,3 @@ func initializeServicemeshConfig(s *options.ServerRunOptions) {
|
||||
|
||||
kconfig.Set(config)
|
||||
}
|
||||
|
||||
//
|
||||
func CreateAPIServer(s *options.ServerRunOptions, stopCh <-chan struct{}) error {
|
||||
var err error
|
||||
|
||||
deps := createDeps(s, stopCh)
|
||||
|
||||
apiserver := apiserver.New(deps)
|
||||
|
||||
container := runtime.Container
|
||||
container.DoNotRecover(false)
|
||||
container.Filter(filter.Logging)
|
||||
container.RecoverHandler(server.LogStackOnRecover)
|
||||
|
||||
kapis.InstallAPIs(container)
|
||||
|
||||
// install config api
|
||||
apiserverconfig.InstallAPI(container)
|
||||
|
||||
if s.GenericServerRunOptions.InsecurePort != 0 {
|
||||
err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container)
|
||||
if err == nil {
|
||||
klog.V(0).Infof("Server listening on insecure port %d.", s.GenericServerRunOptions.InsecurePort)
|
||||
}
|
||||
}
|
||||
|
||||
if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 {
|
||||
err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container)
|
||||
if err == nil {
|
||||
klog.V(0).Infof("Server listening on secure port %d.", s.GenericServerRunOptions.SecurePort)
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func createDeps(s *options.ServerRunOptions, stopCh <-chan struct{}) *apiserver.Dependencies {
|
||||
deps := &apiserver.Dependencies{}
|
||||
|
||||
if s.KubernetesOptions == nil || s.KubernetesOptions.KubeConfig == "" {
|
||||
klog.Warning("kubeconfig not provided, will use in-cluster config")
|
||||
}
|
||||
|
||||
var err error
|
||||
deps.KubeClient, err = k8s.NewKubernetesClient(s.KubernetesOptions)
|
||||
if err != nil {
|
||||
klog.Fatalf("error happened when initializing kubernetes client, %v", err)
|
||||
}
|
||||
|
||||
if s.S3Options != nil && s.S3Options.Endpoint != "" {
|
||||
deps.S3, err = s3.NewS3Client(s.S3Options)
|
||||
if err != nil {
|
||||
klog.Fatalf("error initializing s3 client, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s.OpenPitrixOptions != nil && !s.OpenPitrixOptions.IsEmpty() {
|
||||
deps.OpenPitrix, err = openpitrix.NewClient(s.OpenPitrixOptions)
|
||||
if err != nil {
|
||||
klog.Fatalf("error happened when initializing openpitrix client, %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return deps
|
||||
|
||||
}
|
||||
|
||||
func WaitForResourceSync(stopCh <-chan struct{}) error {
|
||||
klog.V(0).Info("Start cache objects")
|
||||
|
||||
discoveryClient := client.ClientSets().K8s().Discovery()
|
||||
apiResourcesList, err := discoveryClient.ServerResources()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
isResourceExists := func(resource schema.GroupVersionResource) bool {
|
||||
for _, apiResource := range apiResourcesList {
|
||||
if apiResource.GroupVersion == resource.GroupVersion().String() {
|
||||
for _, rsc := range apiResource.APIResources {
|
||||
if rsc.Name == resource.Resource {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
informerFactory := kinformers.NewInformerFactories(client.ClientSets().K8s().Kubernetes(), client.ClientSets().K8s().KubeSphere(), client.ClientSets().K8s().S2i(),
|
||||
client.ClientSets().K8s().Application())
|
||||
|
||||
// resources we have to create informer first
|
||||
k8sGVRs := []schema.GroupVersionResource{
|
||||
{Group: "", Version: "v1", Resource: "namespaces"},
|
||||
{Group: "", Version: "v1", Resource: "nodes"},
|
||||
{Group: "", Version: "v1", Resource: "resourcequotas"},
|
||||
{Group: "", Version: "v1", Resource: "pods"},
|
||||
{Group: "", Version: "v1", Resource: "services"},
|
||||
{Group: "", Version: "v1", Resource: "persistentvolumeclaims"},
|
||||
{Group: "", Version: "v1", Resource: "secrets"},
|
||||
{Group: "", Version: "v1", Resource: "configmaps"},
|
||||
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"},
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"},
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"},
|
||||
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"},
|
||||
|
||||
{Group: "apps", Version: "v1", Resource: "deployments"},
|
||||
{Group: "apps", Version: "v1", Resource: "daemonsets"},
|
||||
{Group: "apps", Version: "v1", Resource: "replicasets"},
|
||||
{Group: "apps", Version: "v1", Resource: "statefulsets"},
|
||||
{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
|
||||
|
||||
{Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"},
|
||||
|
||||
{Group: "batch", Version: "v1", Resource: "jobs"},
|
||||
{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
|
||||
|
||||
{Group: "extensions", Version: "v1beta1", Resource: "ingresses"},
|
||||
|
||||
{Group: "autoscaling", Version: "v2beta2", Resource: "horizontalpodautoscalers"},
|
||||
}
|
||||
|
||||
for _, gvr := range k8sGVRs {
|
||||
if !isResourceExists(gvr) {
|
||||
klog.Warningf("resource %s not exists in the cluster", gvr)
|
||||
} else {
|
||||
_, err := informerFactory.KubernetesSharedInformerFactory().ForResource(gvr)
|
||||
if err != nil {
|
||||
klog.Errorf("cannot create informer for %s", gvr)
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
informerFactory.KubernetesSharedInformerFactory().Start(stopCh)
|
||||
informerFactory.KubernetesSharedInformerFactory().WaitForCacheSync(stopCh)
|
||||
|
||||
s2iInformerFactory := informerFactory.S2iSharedInformerFactory()
|
||||
|
||||
s2iGVRs := []schema.GroupVersionResource{
|
||||
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuildertemplates"},
|
||||
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2iruns"},
|
||||
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuilders"},
|
||||
}
|
||||
|
||||
for _, gvr := range s2iGVRs {
|
||||
if !isResourceExists(gvr) {
|
||||
klog.Warningf("resource %s not exists in the cluster", gvr)
|
||||
} else {
|
||||
_, err := s2iInformerFactory.ForResource(gvr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
s2iInformerFactory.Start(stopCh)
|
||||
s2iInformerFactory.WaitForCacheSync(stopCh)
|
||||
|
||||
ksInformerFactory := informerFactory.KubeSphereSharedInformerFactory()
|
||||
|
||||
ksGVRs := []schema.GroupVersionResource{
|
||||
{Group: "tenant.kubesphere.io", Version: "v1alpha1", Resource: "workspaces"},
|
||||
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibinaries"},
|
||||
|
||||
{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "strategies"},
|
||||
{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "servicepolicies"},
|
||||
}
|
||||
|
||||
for _, gvr := range ksGVRs {
|
||||
if !isResourceExists(gvr) {
|
||||
klog.Warningf("resource %s not exists in the cluster", gvr)
|
||||
} else {
|
||||
_, err := ksInformerFactory.ForResource(gvr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ksInformerFactory.Start(stopCh)
|
||||
ksInformerFactory.WaitForCacheSync(stopCh)
|
||||
|
||||
appInformerFactory := informerFactory.ApplicationSharedInformerFactory()
|
||||
|
||||
appGVRs := []schema.GroupVersionResource{
|
||||
{Group: "app.k8s.io", Version: "v1beta1", Resource: "applications"},
|
||||
}
|
||||
|
||||
for _, gvr := range appGVRs {
|
||||
if !isResourceExists(gvr) {
|
||||
klog.Warningf("resource %s not exists in the cluster", gvr)
|
||||
} else {
|
||||
_, err := appInformerFactory.ForResource(gvr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
appInformerFactory.Start(stopCh)
|
||||
appInformerFactory.WaitForCacheSync(stopCh)
|
||||
|
||||
klog.V(0).Info("Finished caching objects")
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// load options from config file
|
||||
func load() *options.ServerRunOptions {
|
||||
conf := apiserverconfig.Get()
|
||||
|
||||
return &options.ServerRunOptions{
|
||||
KubernetesOptions: conf.KubernetesOptions,
|
||||
DevopsOptions: conf.DevopsOptions,
|
||||
SonarQubeOptions: conf.SonarQubeOptions,
|
||||
ServiceMeshOptions: conf.ServiceMeshOptions,
|
||||
MySQLOptions: conf.MySQLOptions,
|
||||
MonitoringOptions: conf.MonitoringOptions,
|
||||
S3Options: conf.S3Options,
|
||||
OpenPitrixOptions: conf.OpenPitrixOptions,
|
||||
LoggingOptions: conf.LoggingOptions,
|
||||
}
|
||||
}
|
||||
|
||||
func initConfigz() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user