feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions

View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
@@ -22,76 +11,51 @@ import (
"fmt"
"net/http"
"strings"
"sync"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
"k8s.io/client-go/kubernetes/scheme"
"github.com/Masterminds/semver/v3"
"golang.org/x/net/context"
corev1 "k8s.io/api/core/v1"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/cluster"
"kubesphere.io/kubesphere/pkg/apis"
"kubesphere.io/kubesphere/pkg/apiserver"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/options"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/models/auth"
resourcev1beta1 "kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
"kubesphere.io/kubesphere/pkg/scheme"
genericoptions "kubesphere.io/kubesphere/pkg/server/options"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
esclient "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/metricsserver"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)
type ServerRunOptions struct {
ConfigFile string
type APIServerOptions struct {
options.Options
GenericServerRunOptions *genericoptions.ServerRunOptions
*apiserverconfig.Config
schemeOnce sync.Once
DebugMode bool
ConfigFile string
DebugMode bool
}
func NewServerRunOptions() *ServerRunOptions {
s := &ServerRunOptions{
func NewAPIServerOptions() *APIServerOptions {
return &APIServerOptions{
GenericServerRunOptions: genericoptions.NewServerRunOptions(),
Config: apiserverconfig.New(),
schemeOnce: sync.Once{},
}
return s
}
func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
func (s *APIServerOptions) Flags() (fss cliflag.NamedFlagSets) {
fs := fss.FlagSet("generic")
fs.BoolVar(&s.DebugMode, "debug", false, "Don't enable this if you don't know what it means.")
s.GenericServerRunOptions.AddFlags(fs, s.GenericServerRunOptions)
s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
s.AuthenticationOptions.AddFlags(fss.FlagSet("authentication"), s.AuthenticationOptions)
s.AuthorizationOptions.AddFlags(fss.FlagSet("authorization"), s.AuthorizationOptions)
s.DevopsOptions.AddFlags(fss.FlagSet("devops"), s.DevopsOptions)
s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"), s.SonarQubeOptions)
s.S3Options.AddFlags(fss.FlagSet("s3"), s.S3Options)
s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"), s.OpenPitrixOptions)
s.NetworkOptions.AddFlags(fss.FlagSet("network"), s.NetworkOptions)
s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"), s.ServiceMeshOptions)
s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"), s.MonitoringOptions)
s.LoggingOptions.AddFlags(fss.FlagSet("logging"), s.LoggingOptions)
s.MultiClusterOptions.AddFlags(fss.FlagSet("multicluster"), s.MultiClusterOptions)
s.EventsOptions.AddFlags(fss.FlagSet("events"), s.EventsOptions)
s.AuditingOptions.AddFlags(fss.FlagSet("auditing"), s.AuditingOptions)
s.AlertingOptions.AddFlags(fss.FlagSet("alerting"), s.AlertingOptions)
fs = fss.FlagSet("klog")
local := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -105,78 +69,67 @@ func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
}
// NewAPIServer creates an APIServer instance using given options
func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIServer, error) {
func (s *APIServerOptions) NewAPIServer(ctx context.Context) (*apiserver.APIServer, error) {
apiServer := &apiserver.APIServer{
Config: s.Config,
Options: s.Options,
}
kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
if err != nil {
return nil, err
}
apiServer.KubernetesClient = kubernetesClient
ctrl.SetLogger(klog.NewKlogr())
informerFactory := informers.NewInformerFactories(kubernetesClient.Kubernetes(), kubernetesClient.KubeSphere(),
kubernetesClient.Istio(), kubernetesClient.Snapshot(), kubernetesClient.ApiExtensions(), kubernetesClient.Prometheus())
apiServer.InformerFactory = informerFactory
if s.MonitoringOptions == nil || len(s.MonitoringOptions.Endpoint) == 0 {
return nil, fmt.Errorf("moinitoring service address in configuration MUST not be empty, please check configmap/kubesphere-config in kubesphere-system namespace")
} else {
if apiServer.MonitoringClient, err = prometheus.NewPrometheus(s.MonitoringOptions); err != nil {
return nil, fmt.Errorf("failed to connect to prometheus, please check prometheus status, error: %v", err)
}
var err error
if apiServer.K8sClient, err = k8s.NewKubernetesClient(s.KubernetesOptions); err != nil {
return nil, fmt.Errorf("failed to create kubernetes client, error: %v", err)
}
apiServer.MetricsClient = metricsserver.NewMetricsClient(kubernetesClient.Kubernetes(), s.KubernetesOptions)
if s.LoggingOptions.Host != "" {
if apiServer.LoggingClient, err = esclient.NewClient(s.LoggingOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
}
}
if s.DevopsOptions.Host != "" {
if apiServer.DevopsClient, err = jenkins.NewDevopsClient(s.DevopsOptions); err != nil {
return nil, fmt.Errorf("failed to connect to jenkins, please check jenkins status, error: %v", err)
}
}
if s.SonarQubeOptions.Host != "" {
sonarClient, err := sonarqube.NewSonarQubeClient(s.SonarQubeOptions)
if err != nil {
return nil, fmt.Errorf("failed to connecto to sonarqube, please check sonarqube status, error: %v", err)
}
apiServer.SonarClient = sonarqube.NewSonar(sonarClient.SonarQube())
}
if apiServer.CacheClient, err = cache.New(s.CacheOptions, stopCh); err != nil {
if apiServer.CacheClient, err = cache.New(s.CacheOptions, ctx.Done()); err != nil {
return nil, fmt.Errorf("failed to create cache, error: %v", err)
}
if s.EventsOptions.Host != "" {
if apiServer.EventsClient, err = eventsclient.NewClient(s.EventsOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
if c, err := cluster.New(apiServer.K8sClient.Config(), func(options *cluster.Options) {
options.Scheme = scheme.Scheme
}); err != nil {
return nil, fmt.Errorf("unable to create controller runtime cluster: %v", err)
} else {
apiServer.RuntimeCache = c.GetCache()
key := "involvedObject.name"
indexerFunc := func(obj client.Object) []string {
e := obj.(*corev1.Event)
return []string{e.InvolvedObject.Name}
}
}
if s.AuditingOptions.Host != "" {
if apiServer.AuditingClient, err = auditingclient.NewClient(s.AuditingOptions); err != nil {
return nil, fmt.Errorf("failed to connect to elasticsearch, please check elasticsearch status, error: %v", err)
if err = apiServer.RuntimeCache.IndexField(ctx, &corev1.Event{}, key, indexerFunc); err != nil {
klog.Fatalf("unable to create index field: %v", err)
}
apiServer.RuntimeClient = c.GetClient()
}
if s.AlertingOptions != nil && (s.AlertingOptions.PrometheusEndpoint != "" || s.AlertingOptions.ThanosRulerEndpoint != "") {
if apiServer.AlertingClient, err = alerting.NewRuleClient(s.AlertingOptions); err != nil {
return nil, fmt.Errorf("failed to init alerting client: %v", err)
}
apiServer.ResourceManager, err = resourcev1beta1.New(ctx, apiServer.RuntimeClient, apiServer.RuntimeCache)
if err != nil {
return nil, fmt.Errorf("unable to create resource manager: %v", err)
}
if s.Config.MultiClusterOptions.Enable {
apiServer.ClusterClient = clusterclient.NewClusterClient(informerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters())
if err := identityprovider.SharedIdentityProviderController.WatchConfigurationChanges(ctx, apiServer.RuntimeCache); err != nil {
return nil, fmt.Errorf("unable to setup identity provider: %v", err)
}
apiServer.OpenpitrixClient = openpitrixv1.NewOpenpitrixClient(informerFactory, apiServer.KubernetesClient.KubeSphere(), s.OpenPitrixOptions, apiServer.ClusterClient)
if apiServer.ClusterClient, err = clusterclient.NewClusterClientSet(apiServer.RuntimeCache); err != nil {
return nil, fmt.Errorf("unable to create cluster client: %v", err)
}
if apiServer.TokenOperator, err = auth.NewTokenOperator(apiServer.CacheClient, s.Options.AuthenticationOptions); err != nil {
return nil, fmt.Errorf("unable to create issuer: %v", err)
}
k8sVersionInfo, err := apiServer.K8sClient.Discovery().ServerVersion()
if err != nil {
return nil, fmt.Errorf("unable to fetch k8s version info: %v", err)
}
k8sVersion, err := semver.NewVersion(k8sVersionInfo.GitVersion)
if err != nil {
return nil, err
}
apiServer.K8sVersionInfo = k8sVersionInfo
apiServer.K8sVersion = k8sVersion
server := &http.Server{
Addr: fmt.Sprintf(":%d", s.GenericServerRunOptions.InsecurePort),
@@ -187,41 +140,46 @@ func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIS
if err != nil {
return nil, err
}
server.TLSConfig = &tls.Config{
Certificates: []tls.Certificate{certificate},
}
server.Addr = fmt.Sprintf(":%d", s.GenericServerRunOptions.SecurePort)
}
sch := scheme.Scheme
s.schemeOnce.Do(func() {
if err := apis.AddToScheme(sch); err != nil {
klog.Fatalf("unable add APIs to scheme: %v", err)
}
})
mapper, err := apiutil.NewDynamicRESTMapper(apiServer.KubernetesClient.Config())
if err != nil {
klog.Fatalf("unable create dynamic RESTMapper: %v", err)
}
apiServer.RuntimeCache, err = runtimecache.New(apiServer.KubernetesClient.Config(), runtimecache.Options{Scheme: sch, Mapper: mapper})
if err != nil {
klog.Fatalf("unable to create controller runtime cache: %v", err)
}
apiServer.RuntimeClient, err = runtimeclient.New(apiServer.KubernetesClient.Config(), runtimeclient.Options{Scheme: sch})
if err != nil {
klog.Fatalf("unable to create controller runtime client: %v", err)
}
apiServer.Issuer, err = token.NewIssuer(s.AuthenticationOptions)
if err != nil {
klog.Fatalf("unable to create issuer: %v", err)
}
apiServer.Server = server
return apiServer, nil
}
func (s *APIServerOptions) Merge(conf *config.Config) {
if conf == nil {
return
}
if conf.KubernetesOptions != nil {
s.KubernetesOptions = conf.KubernetesOptions
}
if conf.CacheOptions != nil {
s.CacheOptions = conf.CacheOptions
}
if conf.AuthenticationOptions != nil {
s.AuthenticationOptions = conf.AuthenticationOptions
}
if conf.AuthorizationOptions != nil {
s.AuthorizationOptions = conf.AuthorizationOptions
}
if conf.MultiClusterOptions != nil {
s.MultiClusterOptions = conf.MultiClusterOptions
}
if conf.AuditingOptions != nil {
s.AuditingOptions = conf.AuditingOptions
}
if conf.TerminalOptions != nil {
s.TerminalOptions = conf.TerminalOptions
}
if conf.S3Options != nil {
s.S3Options = conf.S3Options
}
if conf.ExperimentalOptions != nil {
s.ExperimentalOptions = conf.ExperimentalOptions
}
}

View File

@@ -1,41 +1,18 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
// Validate validates server run options, to find
// options' misconfiguration
func (s *ServerRunOptions) Validate() []error {
func (s *APIServerOptions) Validate() []error {
var errors []error
errors = append(errors, s.GenericServerRunOptions.Validate()...)
errors = append(errors, s.DevopsOptions.Validate()...)
errors = append(errors, s.KubernetesOptions.Validate()...)
errors = append(errors, s.ServiceMeshOptions.Validate()...)
errors = append(errors, s.MonitoringOptions.Validate()...)
errors = append(errors, s.SonarQubeOptions.Validate()...)
errors = append(errors, s.S3Options.Validate()...)
errors = append(errors, s.OpenPitrixOptions.Validate()...)
errors = append(errors, s.NetworkOptions.Validate()...)
errors = append(errors, s.LoggingOptions.Validate()...)
errors = append(errors, s.AuthenticationOptions.Validate()...)
errors = append(errors, s.AuthorizationOptions.Validate()...)
errors = append(errors, s.EventsOptions.Validate()...)
errors = append(errors, s.AuditingOptions.Validate()...)
errors = append(errors, s.AlertingOptions.Validate()...)
return errors
}

View File

@@ -1,23 +1,13 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package app
import (
"context"
"errors"
"fmt"
"net/http"
@@ -26,28 +16,25 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
"kubesphere.io/kubesphere/cmd/ks-apiserver/app/options"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/utils/term"
"kubesphere.io/kubesphere/pkg/version"
)
func NewAPIServerCommand() *cobra.Command {
s := options.NewServerRunOptions()
// Load configuration from file
conf, err := apiserverconfig.TryLoadFromDisk()
if err == nil {
s.Config = conf
s := options.NewAPIServerOptions()
if conf, err := config.TryLoadFromDisk(); err == nil {
s.Merge(conf)
} else {
klog.Fatalf("Failed to load configuration from disk: %v", err)
}
cmd := &cobra.Command{
Use: "ks-apiserver",
Use: constants.KubeSphereAPIServerName,
Long: `The KubeSphere API server validates and configures data for the API objects.
The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
@@ -60,11 +47,11 @@ cluster's shared state through which all other components interact.`,
// Add agent to report additional information such as the current stack trace, Go version, memory stats, etc.
// Bind to a random port on address 127.0.0.1.
if err := agent.Listen(agent.Options{}); err != nil {
klog.Fatal(err)
klog.Fatalln(err)
}
}
return Run(s, apiserverconfig.WatchConfigChange(), signals.SetupSignalHandler())
return Run(signals.SetupSignalHandler(), s)
},
SilenceUsage: true,
}
@@ -78,7 +65,7 @@ cluster's shared state through which all other components interact.`,
usageFmt := "Usage:\n %s\n"
cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
_, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
})
@@ -91,62 +78,20 @@ cluster's shared state through which all other components interact.`,
}
cmd.AddCommand(versionCmd)
return cmd
}
func Run(s *options.ServerRunOptions, configCh <-chan apiserverconfig.Config, ctx context.Context) error {
ictx, cancelFunc := context.WithCancel(context.TODO())
errCh := make(chan error)
defer close(errCh)
go func() {
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
// The ctx (signals.SetupSignalHandler()) is to control the entire program life cycle,
// The ictx(internal context) is created here to control the life cycle of the ks-apiserver(http server, sharedInformer etc.)
// when config change, stop server and renew context, start new server
for {
select {
case <-ctx.Done():
cancelFunc()
return nil
case cfg := <-configCh:
cancelFunc()
s.Config = &cfg
ictx, cancelFunc = context.WithCancel(context.TODO())
go func() {
if errs := s.Validate(); len(errs) != 0 {
for _, err := range errs {
errCh <- err
}
}
if err := run(s, ictx); err != nil {
errCh <- err
}
}()
case err := <-errCh:
cancelFunc()
return err
}
}
}
func run(s *options.ServerRunOptions, ctx context.Context) error {
apiserver, err := s.NewAPIServer(ctx.Done())
func Run(ctx context.Context, s *options.APIServerOptions) error {
apiServer, err := s.NewAPIServer(ctx)
if err != nil {
return err
}
err = apiserver.PrepareRun(ctx.Done())
if err != nil {
if err = apiServer.PrepareRun(ctx.Done()); err != nil {
return err
}
err = apiserver.Run(ctx)
if err == http.ErrServerClosed {
if errors.Is(apiServer.Run(ctx), http.ErrServerClosed) {
return nil
}
return err