kubesphere/pkg/apiserver/apiserver.go

/*
 * Copyright 2024 the KubeSphere Authors.
 * Please refer to the LICENSE file in the root directory of the project.
 * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
 */

package apiserver

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	rt "runtime"

	"github.com/Masterminds/semver/v3"
	restfulspec "github.com/emicklei/go-restful-openapi/v2"
	"github.com/emicklei/go-restful/v3"
	"k8s.io/apimachinery/pkg/runtime/schema"
	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	k8sversion "k8s.io/apimachinery/pkg/version"
	unionauth "k8s.io/apiserver/pkg/authentication/request/union"
	"k8s.io/klog/v2"
	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
	iamv1beta1 "kubesphere.io/api/iam/v1beta1"
	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

	"kubesphere.io/kubesphere/kube/pkg/openapi"
	openapiv2 "kubesphere.io/kubesphere/kube/pkg/openapi/v2"
	openapiv3 "kubesphere.io/kubesphere/kube/pkg/openapi/v3"
	"kubesphere.io/kubesphere/pkg/apiserver/auditing"
	"kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/basic"
	"kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/jwt"
	oauth2 "kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/anonymous"
	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken"
	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/bearertoken"
	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
	"kubesphere.io/kubesphere/pkg/apiserver/authorization/path"
	"kubesphere.io/kubesphere/pkg/apiserver/authorization/rbac"
	unionauthorizer "kubesphere.io/kubesphere/pkg/apiserver/authorization/union"
	"kubesphere.io/kubesphere/pkg/apiserver/filters"
	"kubesphere.io/kubesphere/pkg/apiserver/metrics"
	"kubesphere.io/kubesphere/pkg/apiserver/options"
	"kubesphere.io/kubesphere/pkg/apiserver/request"
	"kubesphere.io/kubesphere/pkg/apiserver/rest"
	openapicontroller "kubesphere.io/kubesphere/pkg/controller/openapi"
	appv2 "kubesphere.io/kubesphere/pkg/kapis/application/v2"
	clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
	configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
	gatewayv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha2"
	iamapiv1beta1 "kubesphere.io/kubesphere/pkg/kapis/iam/v1beta1"
	"kubesphere.io/kubesphere/pkg/kapis/oauth"
	operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
	packagev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/package/v1alpha1"
	resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2"
	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha3"
	"kubesphere.io/kubesphere/pkg/kapis/static"
	tenantapiv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha3"
	tenantapiv1beta1 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1beta1"
	terminalv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/terminal/v1alpha2"
	"kubesphere.io/kubesphere/pkg/kapis/version"
	workloadtemplatev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/workloadtemplate/v1alpha1"
	"kubesphere.io/kubesphere/pkg/models/auth"
	"kubesphere.io/kubesphere/pkg/models/iam/am"
	"kubesphere.io/kubesphere/pkg/models/iam/im"
	resourcev1beta1 "kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
	"kubesphere.io/kubesphere/pkg/server/healthz"
	"kubesphere.io/kubesphere/pkg/simple/client/cache"
	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
	overviewclient "kubesphere.io/kubesphere/pkg/simple/client/overview"
	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)

type APIServer struct {
	Server *http.Server

	options.Options

	// webservice container, where all webservice defines
	container *restful.Container

	// K8sClient is a collection of all kubernetes(include CRDs) objects clientset
	K8sClient k8s.Client

	// cache is used for short-lived objects, like session
	CacheClient cache.Interface

	// controller-runtime cache
	RuntimeCache runtimecache.Cache

	TokenOperator auth.TokenManagementInterface

	// controller-runtime client with informer cache
	RuntimeClient runtimeclient.Client

	ClusterClient clusterclient.Interface

	ResourceManager resourcev1beta1.ResourceManager

	K8sVersionInfo *k8sversion.Info
	K8sVersion     *semver.Version

	OpenAPIConfig    *restfulspec.Config
	openAPIV2Service openapi.APIServiceManager
	openAPIV3Service openapi.APIServiceManager
}

func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
	s.container = restful.NewContainer()
	s.container.Router(restful.CurlyRouter{})
	s.container.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
		logStackOnRecover(panicReason, httpWriter)
	})
	s.installDynamicResourceAPI()
	s.installKubeSphereAPIs()
	s.installMetricsAPI()
	s.installHealthz()
	s.installLivez()
	if err := s.installOpenAPI(); err != nil {
		return err
	}

	for _, ws := range s.container.RegisteredWebServices() {
		klog.V(2).Infof("%s", ws.RootPath())
	}

	combinedHandler, err := s.buildHandlerChain(s.container, stopCh)
	if err != nil {
		return fmt.Errorf("failed to build handler chain: %v", err)
	}
	s.Server.Handler = filters.WithGlobalFilter(combinedHandler)
	return nil
}

func (s *APIServer) installOpenAPI() error {
	s.OpenAPIConfig = &restfulspec.Config{
		WebServices:                   s.container.RegisteredWebServices(),
		PostBuildSwaggerObjectHandler: openapicontroller.EnrichSwaggerObject,
	}

	openapiV2Services, err := openapiv2.BuildAndRegisterAggregator(s.OpenAPIConfig, s.container)
	if err != nil {
		klog.Errorf("failed to install openapi v2 service : %s", err)
	}
	s.openAPIV2Service = openapiV2Services
	openapiV3Services, err := openapiv3.BuildAndRegisterAggregator(s.OpenAPIConfig, s.container)
	if err != nil {
		klog.Errorf("failed to install openapi v3 service : %s", err)
	}
	s.openAPIV3Service = openapiV3Services
	return openapicontroller.SharedOpenAPIController.WatchOpenAPIChanges(context.Background(), s.RuntimeCache, s.openAPIV2Service, s.openAPIV3Service)
}

func (s *APIServer) installMetricsAPI() {
	metrics.Install(s.container)
}

// Install all kubesphere api groups
// Installations happens before all informers start to cache objects,
// so any attempt to list objects using listers will get empty results.
func (s *APIServer) installKubeSphereAPIs() {
	imOperator := im.NewOperator(s.RuntimeClient, s.ResourceManager, s.AuthenticationOptions)
	amOperator := am.NewOperator(s.ResourceManager)
	rbacAuthorizer := rbac.NewRBACAuthorizer(amOperator)
	counter := overviewclient.New(s.RuntimeClient)
	counter.RegisterResource(overviewclient.NewDefaultRegisterOptions(s.K8sVersion)...)

	handlers := []rest.Handler{
		configv1alpha2.NewHandler(&s.Options, s.RuntimeClient),
		resourcev1alpha3.NewHandler(s.RuntimeCache, counter, s.K8sVersion),
		operationsv1alpha2.NewHandler(s.RuntimeClient),
		resourcesv1alpha2.NewHandler(s.RuntimeClient, s.K8sVersion, s.K8sClient.Master(), s.TerminalOptions),
		tenantapiv1alpha3.NewHandler(s.RuntimeClient, s.K8sVersion, s.ClusterClient, amOperator, imOperator, rbacAuthorizer),
		tenantapiv1beta1.NewHandler(s.RuntimeClient, s.K8sVersion, s.ClusterClient, amOperator, imOperator, rbacAuthorizer, counter),
		terminalv1alpha2.NewHandler(s.K8sClient, rbacAuthorizer, s.K8sClient.Config(), s.TerminalOptions),
		clusterkapisv1alpha1.NewHandler(s.RuntimeClient),
		iamapiv1beta1.NewHandler(imOperator, amOperator),
		oauth.NewHandler(imOperator, s.TokenOperator, auth.NewPasswordAuthenticator(s.RuntimeClient, s.AuthenticationOptions),
			auth.NewOAuthAuthenticator(s.RuntimeClient),
			auth.NewLoginRecorder(s.RuntimeClient), s.AuthenticationOptions,
			oauth2.NewOAuthClientGetter(s.RuntimeClient)),
		version.NewHandler(s.K8sVersionInfo),
		packagev1alpha1.NewHandler(s.RuntimeCache),
		gatewayv1alpha2.NewHandler(s.RuntimeCache),
		appv2.NewHandler(s.RuntimeClient, s.ClusterClient, s.S3Options),
		workloadtemplatev1alpha1.NewHandler(s.RuntimeClient, s.K8sVersion, rbacAuthorizer),
		static.NewHandler(s.CacheClient),
	}

	for _, handler := range handlers {
		urlruntime.Must(handler.AddToContainer(s.container))
	}
}

// installHealthz creates the healthz endpoint for this server
func (s *APIServer) installHealthz() {
	urlruntime.Must(healthz.InstallHandler(s.container))
}

// installLivez creates the livez endpoint for this server
func (s *APIServer) installLivez() {
	urlruntime.Must(healthz.InstallLivezHandler(s.container))
}

func (s *APIServer) Run(ctx context.Context) (err error) {
	go func() {
		if err := s.RuntimeCache.Start(ctx); err != nil {
			klog.Errorf("failed to start runtime cache: %s", err)
		}
	}()

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	go func() {
		<-ctx.Done()
		if err := s.Server.Shutdown(ctx); err != nil {
			klog.Errorf("failed to shutdown server: %s", err)
		}
	}()

	klog.V(0).Infof("Start listening on %s", s.Server.Addr)
	if s.Server.TLSConfig != nil {
		err = s.Server.ListenAndServeTLS("", "")
	} else {
		err = s.Server.ListenAndServe()
	}
	return err
}

func (s *APIServer) buildHandlerChain(handler http.Handler, stopCh <-chan struct{}) (http.Handler, error) {
	requestInfoResolver := &request.RequestInfoFactory{
		APIPrefixes:          sets.New("api", "apis", "kapis", "kapi"),
		GrouplessAPIPrefixes: sets.New("api", "kapi"),
		GlobalResources: []schema.GroupResource{
			iamv1beta1.Resource(iamv1beta1.ResourcesPluralUser),
			iamv1beta1.Resource(iamv1beta1.ResourcesPluralGlobalRole),
			iamv1beta1.Resource(iamv1beta1.ResourcesPluralGlobalRoleBinding),
			tenantv1beta1.Resource(tenantv1beta1.ResourcePluralWorkspace),
			tenantv1beta1.Resource(clusterv1alpha1.ResourcesPluralCluster),
			clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralCluster),
			clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralLabel),
			resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralCluster),
			resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralLabel),
		},
	}

	handler = filters.WithKubeAPIServer(handler, s.K8sClient.Config(), s.ExperimentalOptions)
	handler = filters.WithAPIService(handler, s.RuntimeCache)
	handler = filters.WithReverseProxy(handler, s.RuntimeCache)
	handler = filters.WithJSBundle(handler, s.RuntimeCache)

	if s.AuditingOptions.Enable {
		handler = filters.WithAuditing(handler, auditing.NewAuditing(s.K8sClient, s.AuditingOptions, stopCh))
	}

	var authorizers authorizer.Authorizer
	switch s.AuthorizationOptions.Mode {
	case authorization.AlwaysAllow:
		authorizers = authorizerfactory.NewAlwaysAllowAuthorizer()
	case authorization.AlwaysDeny:
		authorizers = authorizerfactory.NewAlwaysDenyAuthorizer()
	default:
		fallthrough
	case authorization.RBAC:
		excludedPaths := []string{"/oauth/*", "/dist/*", "/.well-known/openid-configuration", "/version", "/metrics", "/livez", "/healthz", "/openapi/v2", "/openapi/v3"}
		pathAuthorizer, _ := path.NewAuthorizer(excludedPaths)
		amOperator := am.NewReadOnlyOperator(s.ResourceManager)
		authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator))
	}

	handler = filters.WithAuthorization(handler, authorizers)
	handler = filters.WithMulticluster(handler, s.ClusterClient, s.MultiClusterOptions)

	// authenticators are unordered
	authn := unionauth.New(anonymous.NewAuthenticator(),
		basictoken.New(basic.NewBasicAuthenticator(
			auth.NewPasswordAuthenticator(s.RuntimeClient, s.AuthenticationOptions),
			auth.NewLoginRecorder(s.RuntimeClient))),
		bearertoken.New(jwt.NewTokenAuthenticator(s.RuntimeCache, s.TokenOperator, s.MultiClusterOptions.ClusterRole)))

	handler = filters.WithAuthentication(handler, authn)
	handler = filters.WithRequestInfo(handler, requestInfoResolver)
	return handler, nil
}

func (s *APIServer) installDynamicResourceAPI() {
	dynamicResourceHandler := filters.NewDynamicResourceHandle(func(err restful.ServiceError, req *restful.Request, resp *restful.Response) {
		for header, values := range err.Header {
			for _, value := range values {
				resp.Header().Add(header, value)
			}
		}
		if err := resp.WriteErrorString(err.Code, err.Message); err != nil {
			klog.Errorf("failed to write error string: %s", err)
		}
	}, s.ResourceManager)
	s.container.ServiceErrorHandler(dynamicResourceHandler.HandleServiceError)
}

func logStackOnRecover(panicReason interface{}, w http.ResponseWriter) {
	var buffer bytes.Buffer
	buffer.WriteString(fmt.Sprintf("recover from panic situation: - %v\r\n", panicReason))
	for i := 2; ; i += 1 {
		_, file, line, ok := rt.Caller(i)
		if !ok {
			break
		}
		buffer.WriteString(fmt.Sprintf("    %s:%d\r\n", file, line))
	}
	klog.Errorln(buffer.String())

	headers := http.Header{}
	if ct := w.Header().Get("Content-Type"); len(ct) > 0 {
		headers.Set("Accept", ct)
	}

	http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}