This is a huge commit, it does following things: (#1942)

1. Remove ks-iam standalone binary, move it to ks-apiserver
2. Generate all devops apis inside kubesphere repository, no need to
import s2ioperator.
3. Reorganize ldap code, make it more flexible to use.
This commit is contained in:
zryfish
2020-03-10 13:50:17 +08:00
committed by GitHub
parent 7270307b66
commit 641615b299
235 changed files with 5538 additions and 38064 deletions

View File

@@ -1,12 +1,18 @@
package options
import (
"crypto/tls"
"flag"
"fmt"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/api/iam"
"kubesphere.io/kubesphere/pkg/apiserver"
genericoptions "kubesphere.io/kubesphere/pkg/server/options"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/ldap"
esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/mysql"
@@ -14,6 +20,7 @@ import (
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
"net/http"
"strings"
)
@@ -29,6 +36,9 @@ type ServerRunOptions struct {
S3Options *s3.Options
OpenPitrixOptions *openpitrix.Options
LoggingOptions *esclient.Options
LdapOptions *ldap.Options
CacheOptions *cache.Options
AuthenticateOptions *iam.AuthenticationOptions
}
func NewServerRunOptions() *ServerRunOptions {
@@ -44,22 +54,28 @@ func NewServerRunOptions() *ServerRunOptions {
S3Options: s3.NewS3Options(),
OpenPitrixOptions: openpitrix.NewOptions(),
LoggingOptions: esclient.NewElasticSearchOptions(),
LdapOptions: ldap.NewOptions(),
CacheOptions: cache.NewRedisOptions(),
AuthenticateOptions: iam.NewAuthenticateOptions(),
}
return &s
}
func (s *ServerRunOptions) Flags(c *ServerRunOptions) (fss cliflag.NamedFlagSets) {
s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic"), c.GenericServerRunOptions)
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), c.KubernetesOptions)
s.MySQLOptions.AddFlags(fss.FlagSet("mysql"), c.MySQLOptions)
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), c.DevopsOptions)
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), c.SonarQubeOptions)
s.S3Options.AddFlags(fss.FlagSet("s3"), c.S3Options)
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), c.OpenPitrixOptions)
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), c.ServiceMeshOptions)
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), c.MonitoringOptions)
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), c.LoggingOptions)
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic"), s.GenericServerRunOptions)
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
s.AuthenticateOptions.AddFlags(fss.FlagSet("authenticate"), s.AuthenticateOptions)
s.MySQLOptions.AddFlags(fss.FlagSet("mysql"), s.MySQLOptions)
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), s.SonarQubeOptions)
s.LdapOptions.AddFlags(fss.FlagSet("ldap"), s.LdapOptions)
s.CacheOptions.AddFlags(fss.FlagSet("cache"), s.CacheOptions)
s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), s.MonitoringOptions)
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), s.LoggingOptions)
fs := fss.FlagSet("klog")
local := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -71,3 +87,62 @@ func (s *ServerRunOptions) Flags(c *ServerRunOptions) (fss cliflag.NamedFlagSets
return fss
}
func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIServer, error) {
kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
if err != nil {
return nil, err
}
monitoringClient := prometheus.NewPrometheus(s.MonitoringOptions)
loggingClient, err := esclient.NewElasticsearch(s.LoggingOptions)
if err != nil {
return nil, err
}
s3Client, err := s3.NewS3Client(s.S3Options)
if err != nil {
return nil, err
}
devopsClient, err := jenkins.NewDevopsClient(s.DevopsOptions)
if err != nil {
return nil, err
}
ldapClient, err := ldap.NewLdapClient(s.LdapOptions, stopCh)
if err != nil {
return nil, err
}
cacheClient, err := cache.NewRedisClient(s.CacheOptions, stopCh)
if err != nil {
return nil, err
}
server := &http.Server{
Addr: fmt.Sprintf(":%d", s.GenericServerRunOptions.InsecurePort),
}
if s.GenericServerRunOptions.SecurePort != 0 {
certificate, err := tls.LoadX509KeyPair(s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey)
if err != nil {
return nil, err
}
server.TLSConfig.Certificates = []tls.Certificate{certificate}
}
apiServer := &apiserver.APIServer{
Server: server,
KubernetesClient: kubernetesClient,
MonitoringClient: monitoringClient,
LoggingClient: loggingClient,
S3Client: s3Client,
DevopsClient: devopsClient,
LdapClient: ldapClient,
CacheClient: cacheClient,
}
return apiServer, nil
}

View File

@@ -5,6 +5,7 @@ package options
func (s *ServerRunOptions) Validate() []error {
var errors []error
errors = append(errors, s.GenericServerRunOptions.Validate()...)
errors = append(errors, s.DevopsOptions.Validate()...)
errors = append(errors, s.KubernetesOptions.Validate()...)
errors = append(errors, s.MySQLOptions.Validate()...)

View File

@@ -21,31 +21,35 @@ import (
"fmt"
kconfig "github.com/kiali/kiali/config"
"github.com/spf13/cobra"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog"
"kubesphere.io/kubesphere/cmd/ks-apiserver/app/options"
"kubesphere.io/kubesphere/pkg/apiserver"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/apiserver/servicemesh/tracing"
kinformers "kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/kapis"
"kubesphere.io/kubesphere/pkg/server"
apiserverconfig "kubesphere.io/kubesphere/pkg/server/config"
"kubesphere.io/kubesphere/pkg/server/filter"
"kubesphere.io/kubesphere/pkg/simple/client"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/utils/signals"
"kubesphere.io/kubesphere/pkg/utils/term"
"net/http"
)
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
// Load configuration from file
conf, err := apiserverconfig.TryLoadFromDisk()
if err == nil {
s = &options.ServerRunOptions{
KubernetesOptions: conf.KubernetesOptions,
DevopsOptions: conf.DevopsOptions,
SonarQubeOptions: conf.SonarQubeOptions,
ServiceMeshOptions: conf.ServiceMeshOptions,
MySQLOptions: conf.MySQLOptions,
MonitoringOptions: conf.MonitoringOptions,
S3Options: conf.S3Options,
OpenPitrixOptions: conf.OpenPitrixOptions,
LoggingOptions: conf.LoggingOptions,
AuthenticateOptions: conf.AuthenticateOptions,
}
}
cmd := &cobra.Command{
Use: "ks-apiserver",
Long: `The KubeSphere API server validates and configures data for the api objects.
@@ -60,11 +64,8 @@ cluster's shared state through which all other components interact.`,
},
}
configOptions := load()
fs := cmd.Flags()
namedFlagSets := s.Flags(configOptions)
namedFlagSets := s.Flags()
for _, f := range namedFlagSets.FlagSets {
fs.AddFlagSet(f)
}
@@ -80,24 +81,19 @@ cluster's shared state through which all other components interact.`,
func Run(s *options.ServerRunOptions, stopCh <-chan struct{}) error {
err := CreateClientSet(apiserverconfig.Get(), stopCh)
if err != nil {
return err
}
err = WaitForResourceSync(stopCh)
if err != nil {
return err
}
initializeServicemeshConfig(s)
err = CreateAPIServer(s)
apiserver, err := s.NewAPIServer(stopCh)
if err != nil {
return err
}
return nil
err = apiserver.PrepareRun()
if err != nil {
return nil
}
return apiserver.Run(stopCh)
}
func initializeServicemeshConfig(s *options.ServerRunOptions) {
@@ -119,234 +115,3 @@ func initializeServicemeshConfig(s *options.ServerRunOptions) {
kconfig.Set(config)
}
//
func CreateAPIServer(s *options.ServerRunOptions, stopCh <-chan struct{}) error {
var err error
deps := createDeps(s, stopCh)
apiserver := apiserver.New(deps)
container := runtime.Container
container.DoNotRecover(false)
container.Filter(filter.Logging)
container.RecoverHandler(server.LogStackOnRecover)
kapis.InstallAPIs(container)
// install config api
apiserverconfig.InstallAPI(container)
if s.GenericServerRunOptions.InsecurePort != 0 {
err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container)
if err == nil {
klog.V(0).Infof("Server listening on insecure port %d.", s.GenericServerRunOptions.InsecurePort)
}
}
if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 {
err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container)
if err == nil {
klog.V(0).Infof("Server listening on secure port %d.", s.GenericServerRunOptions.SecurePort)
}
}
return err
}
func createDeps(s *options.ServerRunOptions, stopCh <-chan struct{}) *apiserver.Dependencies {
deps := &apiserver.Dependencies{}
if s.KubernetesOptions == nil || s.KubernetesOptions.KubeConfig == "" {
klog.Warning("kubeconfig not provided, will use in-cluster config")
}
var err error
deps.KubeClient, err = k8s.NewKubernetesClient(s.KubernetesOptions)
if err != nil {
klog.Fatalf("error happened when initializing kubernetes client, %v", err)
}
if s.S3Options != nil && s.S3Options.Endpoint != "" {
deps.S3, err = s3.NewS3Client(s.S3Options)
if err != nil {
klog.Fatalf("error initializing s3 client, %v", err)
}
}
if s.OpenPitrixOptions != nil && !s.OpenPitrixOptions.IsEmpty() {
deps.OpenPitrix, err = openpitrix.NewClient(s.OpenPitrixOptions)
if err != nil {
klog.Fatalf("error happened when initializing openpitrix client, %v", err)
}
}
return deps
}
func WaitForResourceSync(stopCh <-chan struct{}) error {
klog.V(0).Info("Start cache objects")
discoveryClient := client.ClientSets().K8s().Discovery()
apiResourcesList, err := discoveryClient.ServerResources()
if err != nil {
return err
}
isResourceExists := func(resource schema.GroupVersionResource) bool {
for _, apiResource := range apiResourcesList {
if apiResource.GroupVersion == resource.GroupVersion().String() {
for _, rsc := range apiResource.APIResources {
if rsc.Name == resource.Resource {
return true
}
}
}
}
return false
}
informerFactory := kinformers.NewInformerFactories(client.ClientSets().K8s().Kubernetes(), client.ClientSets().K8s().KubeSphere(), client.ClientSets().K8s().S2i(),
client.ClientSets().K8s().Application())
// resources we have to create informer first
k8sGVRs := []schema.GroupVersionResource{
{Group: "", Version: "v1", Resource: "namespaces"},
{Group: "", Version: "v1", Resource: "nodes"},
{Group: "", Version: "v1", Resource: "resourcequotas"},
{Group: "", Version: "v1", Resource: "pods"},
{Group: "", Version: "v1", Resource: "services"},
{Group: "", Version: "v1", Resource: "persistentvolumeclaims"},
{Group: "", Version: "v1", Resource: "secrets"},
{Group: "", Version: "v1", Resource: "configmaps"},
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"},
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"},
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"},
{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"},
{Group: "apps", Version: "v1", Resource: "deployments"},
{Group: "apps", Version: "v1", Resource: "daemonsets"},
{Group: "apps", Version: "v1", Resource: "replicasets"},
{Group: "apps", Version: "v1", Resource: "statefulsets"},
{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
{Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"},
{Group: "batch", Version: "v1", Resource: "jobs"},
{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
{Group: "extensions", Version: "v1beta1", Resource: "ingresses"},
{Group: "autoscaling", Version: "v2beta2", Resource: "horizontalpodautoscalers"},
}
for _, gvr := range k8sGVRs {
if !isResourceExists(gvr) {
klog.Warningf("resource %s not exists in the cluster", gvr)
} else {
_, err := informerFactory.KubernetesSharedInformerFactory().ForResource(gvr)
if err != nil {
klog.Errorf("cannot create informer for %s", gvr)
return err
}
}
}
informerFactory.KubernetesSharedInformerFactory().Start(stopCh)
informerFactory.KubernetesSharedInformerFactory().WaitForCacheSync(stopCh)
s2iInformerFactory := informerFactory.S2iSharedInformerFactory()
s2iGVRs := []schema.GroupVersionResource{
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuildertemplates"},
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2iruns"},
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuilders"},
}
for _, gvr := range s2iGVRs {
if !isResourceExists(gvr) {
klog.Warningf("resource %s not exists in the cluster", gvr)
} else {
_, err := s2iInformerFactory.ForResource(gvr)
if err != nil {
return err
}
}
}
s2iInformerFactory.Start(stopCh)
s2iInformerFactory.WaitForCacheSync(stopCh)
ksInformerFactory := informerFactory.KubeSphereSharedInformerFactory()
ksGVRs := []schema.GroupVersionResource{
{Group: "tenant.kubesphere.io", Version: "v1alpha1", Resource: "workspaces"},
{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibinaries"},
{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "strategies"},
{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "servicepolicies"},
}
for _, gvr := range ksGVRs {
if !isResourceExists(gvr) {
klog.Warningf("resource %s not exists in the cluster", gvr)
} else {
_, err := ksInformerFactory.ForResource(gvr)
if err != nil {
return err
}
}
}
ksInformerFactory.Start(stopCh)
ksInformerFactory.WaitForCacheSync(stopCh)
appInformerFactory := informerFactory.ApplicationSharedInformerFactory()
appGVRs := []schema.GroupVersionResource{
{Group: "app.k8s.io", Version: "v1beta1", Resource: "applications"},
}
for _, gvr := range appGVRs {
if !isResourceExists(gvr) {
klog.Warningf("resource %s not exists in the cluster", gvr)
} else {
_, err := appInformerFactory.ForResource(gvr)
if err != nil {
return err
}
}
}
appInformerFactory.Start(stopCh)
appInformerFactory.WaitForCacheSync(stopCh)
klog.V(0).Info("Finished caching objects")
return nil
}
// load options from config file
func load() *options.ServerRunOptions {
conf := apiserverconfig.Get()
return &options.ServerRunOptions{
KubernetesOptions: conf.KubernetesOptions,
DevopsOptions: conf.DevopsOptions,
SonarQubeOptions: conf.SonarQubeOptions,
ServiceMeshOptions: conf.ServiceMeshOptions,
MySQLOptions: conf.MySQLOptions,
MonitoringOptions: conf.MonitoringOptions,
S3Options: conf.S3Options,
OpenPitrixOptions: conf.OpenPitrixOptions,
LoggingOptions: conf.LoggingOptions,
}
}
func initConfigz() error {
return nil
}