admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

631
pkg/apiserver/apiserver.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package apiserver
@@ -22,256 +11,187 @@ import (
"fmt"
"net/http"
rt "runtime"
"strconv"
"sync"
"time"
"github.com/Masterminds/semver/v3"
restfulspec "github.com/emicklei/go-restful-openapi/v2"
"github.com/emicklei/go-restful/v3"
"k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
urlruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"
k8sversion "k8s.io/apimachinery/pkg/version"
unionauth "k8s.io/apiserver/pkg/authentication/request/union"
"k8s.io/client-go/discovery"
"k8s.io/client-go/util/retry"
"k8s.io/klog/v2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
notificationv2beta1 "kubesphere.io/api/notification/v2beta1"
notificationv2beta2 "kubesphere.io/api/notification/v2beta2"
tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
typesv1beta1 "kubesphere.io/api/types/v1beta1"
audit "kubesphere.io/kubesphere/pkg/apiserver/auditing"
"kubesphere.io/kubesphere/kube/pkg/openapi"
openapiv2 "kubesphere.io/kubesphere/kube/pkg/openapi/v2"
openapiv3 "kubesphere.io/kubesphere/kube/pkg/openapi/v3"
"kubesphere.io/kubesphere/pkg/apiserver/auditing"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/basic"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/jwt"
oauth2 "kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/anonymous"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/bearertoken"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
"kubesphere.io/kubesphere/pkg/apiserver/authorization"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/path"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/rbac"
unionauthorizer "kubesphere.io/kubesphere/pkg/apiserver/authorization/union"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/apiserver/filters"
"kubesphere.io/kubesphere/pkg/apiserver/metrics"
"kubesphere.io/kubesphere/pkg/apiserver/options"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/informers"
alertingv1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v1"
alertingv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v2alpha1"
alertingv2beta1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v2beta1"
"kubesphere.io/kubesphere/pkg/apiserver/rest"
openapicontroller "kubesphere.io/kubesphere/pkg/controller/openapi"
appv2 "kubesphere.io/kubesphere/pkg/kapis/application/v2"
clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
kapisdevops "kubesphere.io/kubesphere/pkg/kapis/devops"
edgeruntimev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/edgeruntime/v1alpha1"
gatewayv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha1"
iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
kubeedgev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/kubeedge/v1alpha1"
meteringv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/metering/v1alpha1"
monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1"
notificationkapisv2beta1 "kubesphere.io/kubesphere/pkg/kapis/notification/v2beta1"
notificationkapisv2beta2 "kubesphere.io/kubesphere/pkg/kapis/notification/v2beta2"
gatewayv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha2"
iamapiv1beta1 "kubesphere.io/kubesphere/pkg/kapis/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/kapis/oauth"
openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
openpitrixv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v2alpha1"
operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
packagev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/package/v1alpha1"
resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2"
resourcev1alpha3 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha3"
servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/servicemesh/metrics/v1alpha2"
tenantv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha2"
tenantv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha3"
"kubesphere.io/kubesphere/pkg/kapis/static"
tenantapiv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha3"
tenantapiv1beta1 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1beta1"
terminalv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/terminal/v1alpha2"
"kubesphere.io/kubesphere/pkg/kapis/version"
"kubesphere.io/kubesphere/pkg/models/auth"
"kubesphere.io/kubesphere/pkg/models/iam/am"
"kubesphere.io/kubesphere/pkg/models/iam/group"
"kubesphere.io/kubesphere/pkg/models/iam/im"
"kubesphere.io/kubesphere/pkg/models/openpitrix"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/loginrecord"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/user"
resourcev1beta1 "kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
"kubesphere.io/kubesphere/pkg/server/healthz"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
"kubesphere.io/kubesphere/pkg/simple/client/auditing"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/devops"
"kubesphere.io/kubesphere/pkg/simple/client/events"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
overviewclient "kubesphere.io/kubesphere/pkg/simple/client/overview"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
"kubesphere.io/kubesphere/pkg/utils/iputil"
"kubesphere.io/kubesphere/pkg/utils/metrics"
)
var initMetrics sync.Once
type APIServer struct {
// number of kubesphere apiserver
ServerCount int
Server *http.Server
Config *apiserverconfig.Config
options.Options
// webservice container, where all webservice defines
container *restful.Container
// kubeClient is a collection of all kubernetes(include CRDs) objects clientset
KubernetesClient k8s.Client
// K8sClient is a collection of all kubernetes(include CRDs) objects clientset
K8sClient k8s.Client
// informerFactory is a collection of all kubernetes(include CRDs) objects informers,
// mainly for fast query
InformerFactory informers.InformerFactory
// cache is used for short lived objects, like session
// cache is used for short-lived objects, like session
CacheClient cache.Interface
// monitoring client set
MonitoringClient monitoring.Interface
MetricsClient monitoring.Interface
LoggingClient logging.Client
DevopsClient devops.Interface
SonarClient sonarqube.SonarInterface
EventsClient events.Client
AuditingClient auditing.Client
AlertingClient alerting.RuleClient
// controller-runtime cache
RuntimeCache runtimecache.Cache
// entity that issues tokens
Issuer token.Issuer
TokenOperator auth.TokenManagementInterface
// controller-runtime client
// controller-runtime client with informer cache
RuntimeClient runtimeclient.Client
ClusterClient clusterclient.ClusterClients
ClusterClient clusterclient.Interface
OpenpitrixClient openpitrix.Interface
ResourceManager resourcev1beta1.ResourceManager
K8sVersionInfo *k8sversion.Info
K8sVersion *semver.Version
OpenAPIConfig *restfulspec.Config
openAPIV2Service openapi.APIServiceManager
openAPIV3Service openapi.APIServiceManager
}
func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
s.container = restful.NewContainer()
s.container.Filter(logRequestAndResponse)
s.container.Filter(monitorRequest)
s.container.Router(restful.CurlyRouter{})
s.container.RecoverHandler(func(panicReason interface{}, httpWriter http.ResponseWriter) {
logStackOnRecover(panicReason, httpWriter)
})
s.installDynamicResourceAPI()
s.installKubeSphereAPIs(stopCh)
s.installKubeSphereAPIs()
s.installMetricsAPI()
s.installHealthz()
if err := s.installOpenAPI(); err != nil {
return err
}
for _, ws := range s.container.RegisteredWebServices() {
klog.V(2).Infof("%s", ws.RootPath())
}
s.Server.Handler = s.container
s.buildHandlerChain(stopCh)
combinedHandler, err := s.buildHandlerChain(s.container, stopCh)
if err != nil {
return fmt.Errorf("failed to build handler chain: %v", err)
}
s.Server.Handler = filters.WithGlobalFilter(combinedHandler)
return nil
}
func monitorRequest(r *restful.Request, response *restful.Response, chain *restful.FilterChain) {
start := time.Now()
chain.ProcessFilter(r, response)
reqInfo, exists := request.RequestInfoFrom(r.Request.Context())
if exists && reqInfo.APIGroup != "" {
RequestCounter.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(response.StatusCode())).Inc()
elapsedSeconds := time.Since(start).Seconds()
RequestLatencies.WithLabelValues(reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource).Observe(elapsedSeconds)
func (s *APIServer) installOpenAPI() error {
s.OpenAPIConfig = &restfulspec.Config{
WebServices: s.container.RegisteredWebServices(),
PostBuildSwaggerObjectHandler: openapicontroller.EnrichSwaggerObject,
}
openapiV2Services, err := openapiv2.BuildAndRegisterAggregator(s.OpenAPIConfig, s.container)
if err != nil {
klog.Errorf("failed to install openapi v2 service : %s", err)
}
s.openAPIV2Service = openapiV2Services
openapiV3Services, err := openapiv3.BuildAndRegisterAggregator(s.OpenAPIConfig, s.container)
if err != nil {
klog.Errorf("failed to install openapi v3 service : %s", err)
}
s.openAPIV3Service = openapiV3Services
return openapicontroller.SharedOpenAPIController.WatchOpenAPIChanges(context.Background(), s.RuntimeCache, s.openAPIV2Service, s.openAPIV3Service)
}
func (s *APIServer) installMetricsAPI() {
initMetrics.Do(registerMetrics)
metrics.Defaults.Install(s.container)
metrics.Install(s.container)
}
// Install all kubesphere api groups
// Installation happens before all informers start to cache objects, so
//
// any attempt to list objects using listers will get empty results.
func (s *APIServer) installKubeSphereAPIs(stopCh <-chan struct{}) {
imOperator := im.NewOperator(s.KubernetesClient.KubeSphere(),
user.New(s.InformerFactory.KubeSphereSharedInformerFactory(),
s.InformerFactory.KubernetesSharedInformerFactory()),
loginrecord.New(s.InformerFactory.KubeSphereSharedInformerFactory()),
s.Config.AuthenticationOptions)
amOperator := am.NewOperator(s.KubernetesClient.KubeSphere(),
s.KubernetesClient.Kubernetes(),
s.InformerFactory,
s.DevopsClient)
// Installations happens before all informers start to cache objects,
// so any attempt to list objects using listers will get empty results.
func (s *APIServer) installKubeSphereAPIs() {
imOperator := im.NewOperator(s.RuntimeClient, s.ResourceManager, s.AuthenticationOptions)
amOperator := am.NewOperator(s.ResourceManager)
rbacAuthorizer := rbac.NewRBACAuthorizer(amOperator)
counter := overviewclient.New(s.RuntimeClient)
counter.RegisterResource(overviewclient.NewDefaultRegisterOptions(s.K8sVersion)...)
urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory, s.RuntimeCache))
urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.MetricsClient, s.InformerFactory, s.OpenpitrixClient, s.RuntimeClient))
urlruntime.Must(meteringv1alpha1.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient, s.RuntimeClient))
urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions, s.OpenpitrixClient))
urlruntime.Must(openpitrixv2alpha1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions))
urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes()))
urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory,
s.KubernetesClient.Master()))
urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, imOperator, rbacAuthorizer, s.MonitoringClient, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient))
urlruntime.Must(tenantv1alpha3.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, imOperator, rbacAuthorizer, s.MonitoringClient, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient))
urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), rbacAuthorizer, s.KubernetesClient.Config(), s.Config.TerminalOptions))
urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
s.KubernetesClient.KubeSphere(),
s.InformerFactory.KubernetesSharedInformerFactory(),
s.InformerFactory.KubeSphereSharedInformerFactory(),
s.Config.MultiClusterOptions.ProxyPublishService,
s.Config.MultiClusterOptions.ProxyPublishAddress,
s.Config.MultiClusterOptions.AgentImage))
urlruntime.Must(iamapi.AddToContainer(s.container, imOperator, amOperator,
group.New(s.InformerFactory, s.KubernetesClient.KubeSphere(), s.KubernetesClient.Kubernetes()),
rbacAuthorizer))
userLister := s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister()
urlruntime.Must(oauth.AddToContainer(s.container, imOperator,
auth.NewTokenOperator(s.CacheClient, s.Issuer, s.Config.AuthenticationOptions),
auth.NewPasswordAuthenticator(s.KubernetesClient.KubeSphere(), userLister, s.Config.AuthenticationOptions),
auth.NewOAuthAuthenticator(s.KubernetesClient.KubeSphere(), userLister, s.Config.AuthenticationOptions),
auth.NewLoginRecorder(s.KubernetesClient.KubeSphere(), userLister),
s.Config.AuthenticationOptions))
urlruntime.Must(servicemeshv1alpha2.AddToContainer(s.Config.ServiceMeshOptions, s.container, s.KubernetesClient.Kubernetes(), s.CacheClient))
urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
urlruntime.Must(kapisdevops.AddToContainer(s.container, s.Config.DevopsOptions.Endpoint))
urlruntime.Must(alertingv1.AddToContainer(s.container, s.Config.AlertingOptions.Endpoint))
urlruntime.Must(alertingv2alpha1.AddToContainer(s.container, s.InformerFactory,
s.KubernetesClient.Prometheus(), s.AlertingClient, s.Config.AlertingOptions))
urlruntime.Must(alertingv2beta1.AddToContainer(s.container, s.InformerFactory, s.AlertingClient))
urlruntime.Must(version.AddToContainer(s.container, s.KubernetesClient.Kubernetes().Discovery()))
urlruntime.Must(kubeedgev1alpha1.AddToContainer(s.container, s.Config.KubeEdgeOptions.Endpoint))
urlruntime.Must(edgeruntimev1alpha1.AddToContainer(s.container, s.Config.EdgeRuntimeOptions.Endpoint))
if s.Config.NotificationOptions.IsEnabled() {
urlruntime.Must(notificationv1.AddToContainer(s.container, s.Config.NotificationOptions.Endpoint))
urlruntime.Must(notificationkapisv2beta1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
s.KubernetesClient.KubeSphere()))
urlruntime.Must(notificationkapisv2beta2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
s.KubernetesClient.KubeSphere(), s.Config.NotificationOptions))
handlers := []rest.Handler{
configv1alpha2.NewHandler(&s.Options, s.RuntimeClient),
resourcev1alpha3.NewHandler(s.RuntimeCache, counter, s.K8sVersion),
operationsv1alpha2.NewHandler(s.RuntimeClient),
resourcesv1alpha2.NewHandler(s.RuntimeClient, s.K8sVersion, s.K8sClient.Master(), s.TerminalOptions),
tenantapiv1alpha3.NewHandler(s.RuntimeClient, s.K8sVersion, s.ClusterClient, amOperator, imOperator, rbacAuthorizer),
tenantapiv1beta1.NewHandler(s.RuntimeClient, s.K8sVersion, s.ClusterClient, amOperator, imOperator, rbacAuthorizer, counter),
terminalv1alpha2.NewHandler(s.K8sClient, rbacAuthorizer, s.K8sClient.Config(), s.TerminalOptions),
clusterkapisv1alpha1.NewHandler(s.RuntimeClient),
iamapiv1beta1.NewHandler(imOperator, amOperator),
oauth.NewHandler(imOperator, s.TokenOperator, auth.NewPasswordAuthenticator(s.RuntimeClient, s.AuthenticationOptions),
auth.NewOAuthAuthenticator(s.RuntimeClient),
auth.NewLoginRecorder(s.RuntimeClient), s.AuthenticationOptions,
oauth2.NewOAuthClientGetter(s.RuntimeClient)),
version.NewHandler(s.K8sVersionInfo),
packagev1alpha1.NewHandler(s.RuntimeCache),
gatewayv1alpha2.NewHandler(s.RuntimeCache),
appv2.NewHandler(s.RuntimeClient, s.ClusterClient, s.S3Options),
static.NewHandler(s.CacheClient),
}
for _, handler := range handlers {
urlruntime.Must(handler.AddToContainer(s.container))
}
urlruntime.Must(gatewayv1alpha1.AddToContainer(s.container, s.Config.GatewayOptions, s.RuntimeCache, s.RuntimeClient, s.InformerFactory, s.KubernetesClient.Kubernetes(), s.LoggingClient))
}
// installHealthz creates the healthz endpoint for this server
@@ -280,18 +200,20 @@ func (s *APIServer) installHealthz() {
}
func (s *APIServer) Run(ctx context.Context) (err error) {
go func() {
if err := s.RuntimeCache.Start(ctx); err != nil {
klog.Errorf("failed to start runtime cache: %s", err)
}
}()
err = s.waitForResourceSync(ctx)
if err != nil {
return err
}
shutdownCtx, cancel := context.WithCancel(context.Background())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
go func() {
<-ctx.Done()
_ = s.Server.Shutdown(shutdownCtx)
if err := s.Server.Shutdown(ctx); err != nil {
klog.Errorf("failed to shutdown server: %s", err)
}
}()
klog.V(0).Infof("Start listening on %s", s.Server.Addr)
@@ -300,49 +222,38 @@ func (s *APIServer) Run(ctx context.Context) (err error) {
} else {
err = s.Server.ListenAndServe()
}
return err
}
func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
func (s *APIServer) buildHandlerChain(handler http.Handler, stopCh <-chan struct{}) (http.Handler, error) {
requestInfoResolver := &request.RequestInfoFactory{
APIPrefixes: sets.New("api", "apis", "kapis", "kapi"),
GrouplessAPIPrefixes: sets.New("api", "kapi"),
GlobalResources: []schema.GroupResource{
iamv1alpha2.Resource(iamv1alpha2.ResourcesPluralUser),
iamv1alpha2.Resource(iamv1alpha2.ResourcesPluralGlobalRole),
iamv1alpha2.Resource(iamv1alpha2.ResourcesPluralGlobalRoleBinding),
tenantv1alpha1.Resource(tenantv1alpha1.ResourcePluralWorkspace),
tenantv1alpha2.Resource(tenantv1alpha1.ResourcePluralWorkspace),
tenantv1alpha2.Resource(clusterv1alpha1.ResourcesPluralCluster),
iamv1beta1.Resource(iamv1beta1.ResourcesPluralUser),
iamv1beta1.Resource(iamv1beta1.ResourcesPluralGlobalRole),
iamv1beta1.Resource(iamv1beta1.ResourcesPluralGlobalRoleBinding),
tenantv1beta1.Resource(tenantv1beta1.ResourcePluralWorkspace),
tenantv1beta1.Resource(tenantv1beta1.ResourcePluralWorkspace),
tenantv1beta1.Resource(clusterv1alpha1.ResourcesPluralCluster),
clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralCluster),
clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralLabel),
resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralCluster),
resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralLabel),
},
}
if s.Config.NotificationOptions.IsEnabled() {
requestInfoResolver.GlobalResources = append(requestInfoResolver.GlobalResources,
notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralConfig),
notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralReceiver),
notificationv2beta2.Resource(notificationv2beta2.ResourcesPluralNotificationManager),
notificationv2beta2.Resource(notificationv2beta2.ResourcesPluralConfig),
notificationv2beta2.Resource(notificationv2beta2.ResourcesPluralReceiver),
notificationv2beta2.Resource(notificationv2beta2.ResourcesPluralRouter),
notificationv2beta2.Resource(notificationv2beta2.ResourcesPluralSilence),
)
}
handler = filters.WithKubeAPIServer(handler, s.K8sClient.Config(), s.ExperimentalOptions)
handler = filters.WithAPIService(handler, s.RuntimeCache)
handler = filters.WithReverseProxy(handler, s.RuntimeCache)
handler = filters.WithJSBundle(handler, s.RuntimeCache)
handler := s.Server.Handler
handler = filters.WithKubeAPIServer(handler, s.KubernetesClient.Config())
if s.Config.AuditingOptions.Enable {
handler = filters.WithAuditing(handler,
audit.NewAuditing(s.InformerFactory, s.Config.AuditingOptions, stopCh))
if s.AuditingOptions.Enable {
handler = filters.WithAuditing(handler, auditing.NewAuditing(s.K8sClient, s.AuditingOptions, stopCh))
}
var authorizers authorizer.Authorizer
switch s.Config.AuthorizationOptions.Mode {
switch s.AuthorizationOptions.Mode {
case authorization.AlwaysAllow:
authorizers = authorizerfactory.NewAlwaysAllowAuthorizer()
case authorization.AlwaysDeny:
@@ -350,275 +261,25 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
default:
fallthrough
case authorization.RBAC:
excludedPaths := []string{"/oauth/*", "/kapis/config.kubesphere.io/*", "/kapis/version", "/kapis/metrics", "/healthz"}
excludedPaths := []string{"/oauth/*", "/dist/*", "/.well-known/openid-configuration", "/kapis/version", "/version", "/metrics", "/healthz", "/openapi/v2", "/openapi/v3"}
pathAuthorizer, _ := path.NewAuthorizer(excludedPaths)
amOperator := am.NewReadOnlyOperator(s.InformerFactory, s.DevopsClient)
amOperator := am.NewReadOnlyOperator(s.ResourceManager)
authorizers = unionauthorizer.New(pathAuthorizer, rbac.NewRBACAuthorizer(amOperator))
}
handler = filters.WithAuthorization(handler, authorizers)
if s.Config.MultiClusterOptions.Enable {
handler = filters.WithMulticluster(handler, s.ClusterClient)
}
userLister := s.InformerFactory.KubeSphereSharedInformerFactory().Iam().V1alpha2().Users().Lister()
loginRecorder := auth.NewLoginRecorder(s.KubernetesClient.KubeSphere(), userLister)
handler = filters.WithMulticluster(handler, s.ClusterClient, s.MultiClusterOptions)
// authenticators are unordered
authn := unionauth.New(anonymous.NewAuthenticator(),
basictoken.New(basic.NewBasicAuthenticator(auth.NewPasswordAuthenticator(
s.KubernetesClient.KubeSphere(),
userLister,
s.Config.AuthenticationOptions),
loginRecorder)),
bearertoken.New(jwt.NewTokenAuthenticator(
auth.NewTokenOperator(s.CacheClient, s.Issuer, s.Config.AuthenticationOptions),
userLister)))
basictoken.New(basic.NewBasicAuthenticator(
auth.NewPasswordAuthenticator(s.RuntimeClient, s.AuthenticationOptions),
auth.NewLoginRecorder(s.RuntimeClient))),
bearertoken.New(jwt.NewTokenAuthenticator(s.RuntimeCache, s.TokenOperator, s.MultiClusterOptions.ClusterRole)))
handler = filters.WithAuthentication(handler, authn)
handler = filters.WithRequestInfo(handler, requestInfoResolver)
s.Server.Handler = handler
}
func isResourceExists(apiResources []v1.APIResource, resource schema.GroupVersionResource) bool {
for _, apiResource := range apiResources {
if apiResource.Name == resource.Resource {
return true
}
}
return false
}
type informerForResourceFunc func(resource schema.GroupVersionResource) (interface{}, error)
func waitForCacheSync(discoveryClient discovery.DiscoveryInterface, sharedInformerFactory informers.GenericInformerFactory, informerForResourceFunc informerForResourceFunc, GVRs map[schema.GroupVersion][]string, stopCh <-chan struct{}) error {
for groupVersion, resourceNames := range GVRs {
var apiResourceList *v1.APIResourceList
var err error
err = retry.OnError(retry.DefaultRetry, func(err error) bool {
return !errors.IsNotFound(err)
}, func() error {
apiResourceList, err = discoveryClient.ServerResourcesForGroupVersion(groupVersion.String())
return err
})
if err != nil {
if errors.IsNotFound(err) {
klog.Warningf("group version %s not exists in the cluster", groupVersion)
continue
}
return fmt.Errorf("failed to fetch group version %s: %s", groupVersion, err)
}
for _, resourceName := range resourceNames {
groupVersionResource := groupVersion.WithResource(resourceName)
if !isResourceExists(apiResourceList.APIResources, groupVersionResource) {
klog.Warningf("resource %s not exists in the cluster", groupVersionResource)
} else {
// reflect.ValueOf(sharedInformerFactory).MethodByName("ForResource").Call([]reflect.Value{reflect.ValueOf(groupVersionResource)})
if _, err = informerForResourceFunc(groupVersionResource); err != nil {
return fmt.Errorf("failed to create informer for %s: %s", groupVersionResource, err)
}
}
}
}
sharedInformerFactory.Start(stopCh)
sharedInformerFactory.WaitForCacheSync(stopCh)
return nil
}
func (s *APIServer) waitForResourceSync(ctx context.Context) error {
klog.V(0).Info("Start cache objects")
stopCh := ctx.Done()
// resources we have to create informer first
k8sGVRs := map[schema.GroupVersion][]string{
{Group: "", Version: "v1"}: {
"namespaces",
"nodes",
"resourcequotas",
"pods",
"services",
"persistentvolumeclaims",
"persistentvolumes",
"secrets",
"configmaps",
"serviceaccounts",
},
{Group: "rbac.authorization.k8s.io", Version: "v1"}: {
"roles",
"rolebindings",
"clusterroles",
"clusterrolebindings",
},
{Group: "apps", Version: "v1"}: {
"deployments",
"daemonsets",
"replicasets",
"statefulsets",
"controllerrevisions",
},
{Group: "storage.k8s.io", Version: "v1"}: {
"storageclasses",
},
{Group: "batch", Version: "v1"}: {
"jobs",
"cronjobs",
},
{Group: "networking.k8s.io", Version: "v1"}: {
"ingresses",
"networkpolicies",
},
{Group: "autoscaling", Version: "v2"}: {
"horizontalpodautoscalers",
},
}
if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
s.InformerFactory.KubernetesSharedInformerFactory(),
func(resource schema.GroupVersionResource) (interface{}, error) {
return s.InformerFactory.KubernetesSharedInformerFactory().ForResource(resource)
},
k8sGVRs, stopCh); err != nil {
return err
}
ksGVRs := map[schema.GroupVersion][]string{
{Group: "tenant.kubesphere.io", Version: "v1alpha1"}: {
"workspaces",
},
{Group: "tenant.kubesphere.io", Version: "v1alpha2"}: {
"workspacetemplates",
},
{Group: "iam.kubesphere.io", Version: "v1alpha2"}: {
"users",
"globalroles",
"globalrolebindings",
"groups",
"groupbindings",
"workspaceroles",
"workspacerolebindings",
"loginrecords",
},
{Group: "cluster.kubesphere.io", Version: "v1alpha1"}: {
"clusters",
},
{Group: "network.kubesphere.io", Version: "v1alpha1"}: {
"ippools",
},
}
if s.Config.NotificationOptions.IsEnabled() {
ksGVRs[schema.GroupVersion{Group: "notification.kubesphere.io", Version: "v2beta1"}] = []string{
notificationv2beta1.ResourcesPluralConfig,
notificationv2beta1.ResourcesPluralReceiver,
}
ksGVRs[schema.GroupVersion{Group: "notification.kubesphere.io", Version: "v2beta2"}] = []string{
notificationv2beta2.ResourcesPluralNotificationManager,
notificationv2beta2.ResourcesPluralConfig,
notificationv2beta2.ResourcesPluralReceiver,
notificationv2beta2.ResourcesPluralRouter,
notificationv2beta2.ResourcesPluralSilence,
}
}
// skip caching devops resources if devops not enabled
if s.DevopsClient != nil {
ksGVRs[schema.GroupVersion{Group: "devops.kubesphere.io", Version: "v1alpha1"}] = []string{
"s2ibinaries",
"s2ibuildertemplates",
"s2iruns",
"s2ibuilders",
}
ksGVRs[schema.GroupVersion{Group: "devops.kubesphere.io", Version: "v1alpha3"}] = []string{
"devopsprojects",
"pipelines",
}
}
// skip caching servicemesh resources if servicemesh not enabled
if s.KubernetesClient.Istio() != nil {
ksGVRs[schema.GroupVersion{Group: "servicemesh.kubesphere.io", Version: "v1alpha2"}] = []string{
"strategies",
"servicepolicies",
}
}
// federated resources on cached in multi cluster setup
if s.Config.MultiClusterOptions.Enable {
ksGVRs[typesv1beta1.SchemeGroupVersion] = []string{
typesv1beta1.ResourcePluralFederatedClusterRole,
typesv1beta1.ResourcePluralFederatedClusterRoleBindingBinding,
typesv1beta1.ResourcePluralFederatedNamespace,
typesv1beta1.ResourcePluralFederatedService,
typesv1beta1.ResourcePluralFederatedDeployment,
typesv1beta1.ResourcePluralFederatedSecret,
typesv1beta1.ResourcePluralFederatedConfigmap,
typesv1beta1.ResourcePluralFederatedStatefulSet,
typesv1beta1.ResourcePluralFederatedIngress,
typesv1beta1.ResourcePluralFederatedPersistentVolumeClaim,
typesv1beta1.ResourcePluralFederatedApplication,
}
}
if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
s.InformerFactory.KubeSphereSharedInformerFactory(),
func(resource schema.GroupVersionResource) (interface{}, error) {
return s.InformerFactory.KubeSphereSharedInformerFactory().ForResource(resource)
},
ksGVRs, stopCh); err != nil {
return err
}
snapshotGVRs := map[schema.GroupVersion][]string{
{Group: "snapshot.storage.k8s.io", Version: "v1"}: {
"volumesnapshots",
"volumesnapshotcontents",
"volumesnapshotclasses",
},
}
if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
s.InformerFactory.SnapshotSharedInformerFactory(), func(resource schema.GroupVersionResource) (interface{}, error) {
return s.InformerFactory.SnapshotSharedInformerFactory().ForResource(resource)
},
snapshotGVRs, stopCh); err != nil {
return err
}
apiextensionsGVRs := map[schema.GroupVersion][]string{
{Group: "apiextensions.k8s.io", Version: "v1"}: {
"customresourcedefinitions",
},
}
if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
s.InformerFactory.ApiExtensionSharedInformerFactory(), func(resource schema.GroupVersionResource) (interface{}, error) {
return s.InformerFactory.ApiExtensionSharedInformerFactory().ForResource(resource)
},
apiextensionsGVRs, stopCh); err != nil {
return err
}
if promFactory := s.InformerFactory.PrometheusSharedInformerFactory(); promFactory != nil {
prometheusGVRs := map[schema.GroupVersion][]string{
{Group: "monitoring.coreos.com", Version: "v1"}: {
"prometheuses",
"prometheusrules",
"thanosrulers",
},
}
if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
promFactory, func(resource schema.GroupVersionResource) (interface{}, error) {
return promFactory.ForResource(resource)
},
prometheusGVRs, stopCh); err != nil {
return err
}
}
go s.RuntimeCache.Start(ctx)
s.RuntimeCache.WaitForCacheSync(ctx)
klog.V(0).Info("Finished caching objects")
return nil
return handler, nil
}
func (s *APIServer) installDynamicResourceAPI() {
@@ -628,8 +289,10 @@ func (s *APIServer) installDynamicResourceAPI() {
resp.Header().Add(header, value)
}
}
resp.WriteErrorString(err.Code, err.Message)
}, resourcev1beta1.New(s.RuntimeClient, s.RuntimeCache))
if err := resp.WriteErrorString(err.Code, err.Message); err != nil {
klog.Errorf("failed to write error string: %s", err)
}
}, s.ResourceManager)
s.container.ServiceErrorHandler(dynamicResourceHandler.HandleServiceError)
}
@@ -650,27 +313,5 @@ func logStackOnRecover(panicReason interface{}, w http.ResponseWriter) {
headers.Set("Accept", ct)
}
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Internal server error"))
}
func logRequestAndResponse(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
start := time.Now()
chain.ProcessFilter(req, resp)
// Always log error response
logWithVerbose := klog.V(4)
if resp.StatusCode() > http.StatusBadRequest {
logWithVerbose = klog.V(0)
}
logWithVerbose.Infof("%s - \"%s %s %s\" %d %d %dms",
iputil.RemoteIp(req.Request),
req.Request.Method,
req.Request.URL,
req.Request.Proto,
resp.StatusCode(),
resp.ContentLength(),
time.Since(start)/time.Millisecond,
)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
}

216
pkg/apiserver/auditing/backend.go
View File

@@ -1,216 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auditing
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"net/http"
"time"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
options "kubesphere.io/kubesphere/pkg/simple/client/auditing"
)
const (
GetSenderTimeout = time.Second
SendTimeout = time.Second * 3
DefaultSendersNum = 100
DefaultBatchSize = 100
DefaultBatchInterval = time.Second * 3
WebhookURL = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event"
)
type Backend struct {
url string
senderCh chan interface{}
cache chan *v1alpha1.Event
client http.Client
sendTimeout time.Duration
getSenderTimeout time.Duration
eventBatchSize int
eventBatchInterval time.Duration
stopCh <-chan struct{}
}
func NewBackend(opts *options.Options, cache chan *v1alpha1.Event, stopCh <-chan struct{}) *Backend {
b := Backend{
url: opts.WebhookUrl,
getSenderTimeout: GetSenderTimeout,
cache: cache,
sendTimeout: SendTimeout,
eventBatchSize: opts.EventBatchSize,
eventBatchInterval: opts.EventBatchInterval,
stopCh: stopCh,
}
if len(b.url) == 0 {
b.url = WebhookURL
}
if b.eventBatchInterval == 0 {
b.eventBatchInterval = DefaultBatchInterval
}
if b.eventBatchSize == 0 {
b.eventBatchSize = DefaultBatchSize
}
sendersNum := opts.EventSendersNum
if sendersNum == 0 {
sendersNum = DefaultSendersNum
}
b.senderCh = make(chan interface{}, sendersNum)
b.client = http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
Timeout: b.sendTimeout,
}
go b.worker()
return &b
}
func (b *Backend) worker() {
for {
events := b.getEvents()
if events == nil {
break
}
if len(events.Items) == 0 {
continue
}
go b.sendEvents(events)
}
}
func (b *Backend) getEvents() *v1alpha1.EventList {
ctx, cancel := context.WithTimeout(context.Background(), b.eventBatchInterval)
defer cancel()
events := &v1alpha1.EventList{}
for {
select {
case event := <-b.cache:
if event == nil {
break
}
events.Items = append(events.Items, *event)
if len(events.Items) >= b.eventBatchSize {
return events
}
case <-ctx.Done():
return events
case <-b.stopCh:
return nil
}
}
}
func (b *Backend) sendEvents(events *v1alpha1.EventList) {
ctx, cancel := context.WithTimeout(context.Background(), b.sendTimeout)
defer cancel()
stopCh := make(chan struct{})
skipReturnSender := false
send := func() {
ctx, cancel := context.WithTimeout(context.Background(), b.getSenderTimeout)
defer cancel()
select {
case <-ctx.Done():
klog.Error("Get auditing event sender timeout")
skipReturnSender = true
return
case b.senderCh <- struct{}{}:
}
start := time.Now()
defer func() {
stopCh <- struct{}{}
klog.V(8).Infof("send %d auditing logs used %d", len(events.Items), time.Since(start).Milliseconds())
}()
bs, err := b.eventToBytes(events)
if err != nil {
klog.Errorf("json marshal error, %s", err)
return
}
klog.V(8).Infof("%s", string(bs))
response, err := b.client.Post(b.url, "application/json", bytes.NewBuffer(bs))
if err != nil {
klog.Errorf("send audit events error, %s", err)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
klog.Errorf("send audit events error[%d]", response.StatusCode)
return
}
}
go send()
defer func() {
if !skipReturnSender {
<-b.senderCh
}
}()
select {
case <-ctx.Done():
klog.Error("send audit events timeout")
case <-stopCh:
}
}
func (b *Backend) eventToBytes(event *v1alpha1.EventList) ([]byte, error) {
bs, err := json.Marshal(event)
if err != nil {
// Normally, the serialization failure is caused by the failure of ResponseObject serialization.
// To ensure the integrity of the auditing event to the greatest extent,
// it is necessary to delete ResponseObject and and then try to serialize again.
if event.Items[0].ResponseObject != nil {
event.Items[0].ResponseObject = nil
return json.Marshal(event)
}
return nil, err
}
return bs, err
}

298
pkg/apiserver/auditing/types.go → pkg/apiserver/auditing/client.go
View File

@@ -1,92 +1,149 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"time"
"github.com/google/uuid"
"github.com/modern-go/reflect2"
v1 "k8s.io/api/authentication/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/klog/v2"
devopsv1alpha3 "kubesphere.io/api/devops/v1alpha3"
"kubesphere.io/api/iam/v1alpha2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
"kubesphere.io/api/iam/v1beta1"
auditv1alpha1 "kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/log"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/webhook"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/client/listers/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/devops"
options "kubesphere.io/kubesphere/pkg/simple/client/auditing"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/utils/iputil"
)
const (
DefaultWebhook = "kube-auditing-webhook"
DefaultCacheCapacity = 10000
CacheTimeout = time.Second
DefaultBatchSize = 100
DefaultBatchInterval = time.Second * 3
)
type Auditing interface {
Enabled() bool
K8sAuditingEnabled() bool
LogRequestObject(req *http.Request, info *request.RequestInfo) *auditv1alpha1.Event
LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCapture)
LogRequestObject(req *http.Request, info *request.RequestInfo) *Event
LogResponseObject(e *Event, resp *ResponseCapture)
}
type auditing struct {
webhookLister v1alpha1.WebhookLister
devopsGetter v1alpha3.Interface
cache chan *auditv1alpha1.Event
backend *Backend
k8sClient k8s.Client
stopCh <-chan struct{}
auditLevel audit.Level
events chan *Event
backend []internal.Backend
hostname string
hostIP string
cluster string
eventBatchSize int
eventBatchInterval time.Duration
}
func NewAuditing(informers informers.InformerFactory, opts *options.Options, stopCh <-chan struct{}) Auditing {
func NewAuditing(kubernetesClient k8s.Client, opts *Options, stopCh <-chan struct{}) Auditing {
a := &auditing{
webhookLister: informers.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
devopsGetter: devops.New(informers.KubeSphereSharedInformerFactory()),
cache: make(chan *auditv1alpha1.Event, DefaultCacheCapacity),
k8sClient: kubernetesClient,
stopCh: stopCh,
auditLevel: opts.AuditLevel,
events: make(chan *Event, DefaultCacheCapacity),
hostname: os.Getenv("HOSTNAME"),
hostIP: getHostIP(),
eventBatchInterval: opts.EventBatchInterval,
eventBatchSize: opts.EventBatchSize,
}
a.backend = NewBackend(opts, a.cache, stopCh)
if a.eventBatchInterval == 0 {
a.eventBatchInterval = DefaultBatchInterval
}
if a.eventBatchSize == 0 {
a.eventBatchSize = DefaultBatchSize
}
a.cluster = a.getClusterName()
if opts.WebhookOptions.WebhookUrl != "" {
a.backend = append(a.backend, webhook.NewBackend(opts.WebhookOptions.WebhookUrl,
opts.WebhookOptions.EventSendersNum))
}
if opts.LogOptions.Path != "" {
a.backend = append(a.backend, log.NewBackend(opts.LogOptions.Path,
opts.LogOptions.MaxAge,
opts.LogOptions.MaxBackups,
opts.LogOptions.MaxSize))
}
go a.Start()
return a
}
func (a *auditing) getAuditLevel() audit.Level {
wh, err := a.webhookLister.Get(DefaultWebhook)
if err != nil {
klog.V(8).Info(err)
return audit.LevelNone
func getHostIP() string {
addrs, err := net.InterfaceAddrs()
hostip := ""
if err == nil {
for _, addr := range addrs {
if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
if ipnet.IP.To4() != nil {
hostip = ipnet.IP.String()
break
}
}
}
}
return (audit.Level)(wh.Spec.AuditLevel)
return hostip
}
func (a *auditing) getClusterName() string {
ns, err := a.k8sClient.CoreV1().Namespaces().Get(context.Background(), constants.KubeSphereNamespace, metav1.GetOptions{})
if err != nil {
klog.Errorf("get %s error: %s", constants.KubeSphereNamespace, err)
return ""
}
if ns.Annotations != nil {
return ns.Annotations[clusterv1alpha1.AnnotationClusterName]
}
return ""
}
func (a *auditing) getAuditLevel() audit.Level {
if a.auditLevel != "" {
return a.auditLevel
}
return audit.LevelMetadata
}
func (a *auditing) Enabled() bool {
@@ -95,16 +152,6 @@ func (a *auditing) Enabled() bool {
return !level.Less(audit.LevelMetadata)
}
func (a *auditing) K8sAuditingEnabled() bool {
wh, err := a.webhookLister.Get(DefaultWebhook)
if err != nil {
klog.V(8).Info(err)
return false
}
return wh.Spec.K8sAuditingEnabled
}
// If the request is not a standard request, or a resource request,
// or part of the audit information cannot be obtained through url,
// the function that handles the request can obtain Event from
@@ -116,7 +163,8 @@ func (a *auditing) K8sAuditingEnabled() bool {
// info.Verb = "post"
// info.Name = created.Name
// }
func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo) *auditv1alpha1.Event {
func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo) *Event {
// Ignore the dryRun k8s request.
if info.IsKubernetesRequest {
@@ -126,10 +174,11 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
}
}
e := &auditv1alpha1.Event{
Devops: info.DevOps,
e := &Event{
HostName: a.hostname,
HostIP: a.hostIP,
Workspace: info.Workspace,
Cluster: info.Cluster,
Cluster: a.cluster,
Event: audit.Event{
RequestURI: info.Path,
Verb: info.Verb,
@@ -153,25 +202,6 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
},
}
// Get the workspace which the devops project be in.
if len(e.Devops) > 0 && len(e.Workspace) == 0 {
res, err := a.devopsGetter.List("", query.New())
if err != nil {
klog.Error(err)
}
for _, obj := range res.Items {
d := obj.(*devopsv1alpha3.DevOpsProject)
if d.Name == e.Devops {
e.Workspace = d.Labels["kubesphere.io/workspace"]
} else if d.Status.AdminNamespace == e.Devops {
e.Workspace = d.Labels["kubesphere.io/workspace"]
e.Devops = d.Name
}
}
}
ips := make([]string, 1)
ips[0] = iputil.RemoteIp(req)
e.SourceIPs = ips
@@ -203,7 +233,7 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
// For resource creating request, get resource name from the request body.
if info.Verb == "create" {
obj := &auditv1alpha1.Object{}
obj := &Object{}
if err := json.Unmarshal(body, obj); err == nil {
e.ObjectRef.Name = obj.Name
}
@@ -211,21 +241,47 @@ func (a *auditing) LogRequestObject(req *http.Request, info *request.RequestInfo
// for recording disable and enable user
if e.ObjectRef.Resource == "users" && e.Verb == "update" {
u := &v1alpha2.User{}
u := &v1beta1.User{}
if err := json.Unmarshal(body, u); err == nil {
if u.Status.State == v1alpha2.UserActive {
if u.Status.State == v1beta1.UserActive {
e.Verb = "enable"
} else if u.Status.State == v1alpha2.UserDisabled {
} else if u.Status.State == v1beta1.UserDisabled {
e.Verb = "disable"
}
}
}
}
a.getWorkspace(e)
return e
}
func (a *auditing) needAnalyzeRequestBody(e *auditv1alpha1.Event, req *http.Request) bool {
func (a *auditing) getWorkspace(e *Event) {
if e.Workspace != "" {
return
}
ns := e.ObjectRef.Namespace
if e.ObjectRef.Resource == "namespaces" {
ns = e.ObjectRef.Name
}
if ns == "" {
return
}
namespace, err := a.k8sClient.CoreV1().Namespaces().Get(context.Background(), ns, metav1.GetOptions{})
if err != nil && !apierrors.IsNotFound(err) {
klog.Errorf("get %s error: %s", ns, err)
return
}
if namespace.Labels != nil {
e.Workspace = namespace.Labels[constants.WorkspaceLabelKey]
}
}
func (a *auditing) needAnalyzeRequestBody(e *Event, req *http.Request) bool {
if req.ContentLength <= 0 {
return false
@@ -247,7 +303,7 @@ func (a *auditing) needAnalyzeRequestBody(e *auditv1alpha1.Event, req *http.Requ
return false
}
func (a *auditing) LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCapture) {
func (a *auditing) LogResponseObject(e *Event, resp *ResponseCapture) {
e.StageTimestamp = metav1.NowMicro()
e.ResponseStatus = &metav1.Status{Code: int32(resp.StatusCode())}
@@ -258,10 +314,9 @@ func (a *auditing) LogResponseObject(e *auditv1alpha1.Event, resp *ResponseCaptu
a.cacheEvent(*e)
}
func (a *auditing) cacheEvent(e auditv1alpha1.Event) {
func (a *auditing) cacheEvent(e Event) {
select {
case a.cache <- &e:
case a.events <- &e:
return
case <-time.After(CacheTimeout):
klog.V(8).Infof("cache audit event %s timeout", e.AuditID)
@@ -269,6 +324,80 @@ func (a *auditing) cacheEvent(e auditv1alpha1.Event) {
}
}
func (a *auditing) Start() {
for {
events, exit := a.getEvents()
if exit {
break
}
if len(events) == 0 {
continue
}
byteEvents := a.eventToBytes(events)
if len(byteEvents) == 0 {
continue
}
for _, b := range a.backend {
if reflect2.IsNil(b) {
continue
}
b.ProcessEvents(byteEvents...)
}
}
}
func (a *auditing) getEvents() ([]*Event, bool) {
ctx, cancel := context.WithTimeout(context.Background(), a.eventBatchInterval)
defer cancel()
var events []*Event
for {
select {
case event := <-a.events:
if event == nil {
break
}
events = append(events, event)
if len(events) >= a.eventBatchSize {
return events, false
}
case <-ctx.Done():
return events, false
case <-a.stopCh:
return nil, true
}
}
}
func (a *auditing) eventToBytes(events []*Event) [][]byte {
var res [][]byte
for _, event := range events {
bs, err := json.Marshal(event)
if err != nil {
// Normally, the serialization failure is caused by the failure of ResponseObject serialization.
// To ensure the integrity of the auditing event to the greatest extent,
// it is necessary to delete ResponseObject and and then try to serialize again.
if event.ResponseObject != nil {
event.ResponseObject = nil
bs, err = json.Marshal(event)
}
}
if err != nil {
klog.Errorf("serialize audit event error: %s", err)
continue
}
res = append(res, bs)
}
return res
}
type ResponseCapture struct {
http.ResponseWriter
wroteHeader bool
@@ -289,7 +418,6 @@ func (c *ResponseCapture) Header() http.Header {
}
func (c *ResponseCapture) Write(data []byte) (int, error) {
c.WriteHeader(http.StatusOK)
c.body.Write(data)
return c.ResponseWriter.Write(data)

28
pkg/apiserver/auditing/event.go Normal file
View File

@@ -0,0 +1,28 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
)
type Event struct {
// The workspace which this audit event happened
Workspace string
// The cluster which this audit event happened
Cluster string
// Message send to user.
Message string
HostName string
HostIP string
audit.Event
}
type Object struct {
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
}

10
pkg/apiserver/auditing/internal/backend.go Normal file
View File

@@ -0,0 +1,10 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package internal
type Backend interface {
ProcessEvents(events ...[]byte)
}

94
pkg/apiserver/auditing/log/backend.go Normal file
View File

@@ -0,0 +1,94 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package log
import (
"fmt"
"io"
"os"
"path/filepath"
"time"
"gopkg.in/natefinch/lumberjack.v2"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
)
const (
WriteTimeout = time.Second * 3
DefaultMaxAge = 7
DefaultMaxBackups = 10
DefaultMaxSize = 100
)
type backend struct {
path string
maxAge int
maxBackups int
maxSize int
timeout time.Duration
writer io.Writer
}
func NewBackend(path string, maxAge, maxBackups, maxSize int) internal.Backend {
b := backend{
path: path,
maxAge: maxAge,
maxBackups: maxBackups,
maxSize: maxSize,
timeout: WriteTimeout,
}
if b.maxAge == 0 {
b.maxAge = DefaultMaxAge
}
if b.maxBackups == 0 {
b.maxBackups = DefaultMaxBackups
}
if b.maxSize == 0 {
b.maxSize = DefaultMaxSize
}
if err := b.ensureLogFile(); err != nil {
klog.Errorf("ensure audit log file error, %s", err)
return nil
}
b.writer = &lumberjack.Logger{
Filename: b.path,
MaxAge: b.maxAge,
MaxBackups: b.maxBackups,
MaxSize: b.maxSize,
Compress: false,
}
return &b
}
func (b *backend) ensureLogFile() error {
if err := os.MkdirAll(filepath.Dir(b.path), 0700); err != nil {
return err
}
mode := os.FileMode(0600)
f, err := os.OpenFile(b.path, os.O_CREATE|os.O_APPEND|os.O_RDWR, mode)
if err != nil {
return err
}
return f.Close()
}
func (b *backend) ProcessEvents(events ...[]byte) {
for _, event := range events {
if _, err := fmt.Fprint(b.writer, string(event)+"\n"); err != nil {
klog.Errorf("Log audit event error, %s. affecting audit event: %v\nImpacted event:\n", err, event)
klog.Error(string(event))
}
}
}

69
pkg/apiserver/auditing/options.go Normal file
View File

@@ -0,0 +1,69 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auditing
import (
"time"
"k8s.io/apiserver/pkg/apis/audit"
"github.com/spf13/pflag"
)
type WebhookOptions struct {
WebhookUrl string `json:"webhookUrl" yaml:"webhookUrl"`
// The maximum concurrent senders which send auditing events to the auditing webhook.
EventSendersNum int `json:"eventSendersNum" yaml:"eventSendersNum"`
}
type LogOptions struct {
Path string `json:"path" yaml:"path"`
MaxAge int `json:"maxAge" yaml:"maxAge"`
MaxBackups int `json:"maxBackups" yaml:"maxBackups"`
MaxSize int `json:"maxSize" yaml:"maxSize"`
}
type Options struct {
Enable bool `json:"enable" yaml:"enable"`
AuditLevel audit.Level `json:"auditLevel" yaml:"auditLevel"`
// The batch size of auditing events.
EventBatchSize int `json:"eventBatchSize" yaml:"eventBatchSize"`
// The batch interval of auditing events.
EventBatchInterval time.Duration `json:"eventBatchInterval" yaml:"eventBatchInterval"`
WebhookOptions WebhookOptions `json:"webhookOptions" yaml:"webhookOptions"`
LogOptions LogOptions `json:"logOptions" yaml:"logOptions"`
}
func NewAuditingOptions() *Options {
return &Options{}
}
func (s *Options) Validate() []error {
errs := make([]error, 0)
return errs
}
func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
fs.BoolVar(&s.Enable, "auditing-enabled", c.Enable, "Enable auditing component or not. ")
fs.IntVar(&s.EventBatchSize, "auditing-event-batch-size", c.EventBatchSize,
"The batch size of auditing events.")
fs.DurationVar(&s.EventBatchInterval, "auditing-event-batch-interval", c.EventBatchInterval,
"The batch interval of auditing events.")
fs.StringVar(&s.WebhookOptions.WebhookUrl, "auditing-webhook-url", c.WebhookOptions.WebhookUrl, "Auditing wehook url")
fs.IntVar(&s.WebhookOptions.EventSendersNum, "auditing-event-senders-num", c.WebhookOptions.EventSendersNum,
"The maximum concurrent senders which send auditing events to the auditing webhook.")
fs.StringVar(&s.LogOptions.Path, "audit-log-path", s.LogOptions.Path,
"If set, all requests coming to the apiserver will be logged to this file. '-' means standard out.")
fs.IntVar(&s.LogOptions.MaxAge, "audit-log-maxage", s.LogOptions.MaxAge,
"The maximum number of days to retain old audit log files based on the timestamp encoded in their filename.")
fs.IntVar(&s.LogOptions.MaxBackups, "audit-log-maxbackup", s.LogOptions.MaxBackups,
"The maximum number of old audit log files to retain. Setting a value of 0 will mean there's no restriction on the number of files.")
fs.IntVar(&s.LogOptions.MaxSize, "audit-log-maxsize", s.LogOptions.MaxSize,
"The maximum size in megabytes of the audit log file before it gets rotated.")
}

351
pkg/apiserver/auditing/types_test.go
View File

@@ -1,351 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auditing
import (
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"testing"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/authentication/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authentication/user"
k8srequest "k8s.io/apiserver/pkg/endpoints/request"
fakek8s "k8s.io/client-go/kubernetes/fake"
auditingv1alpha1 "kubesphere.io/api/auditing/v1alpha1"
v1alpha12 "kubesphere.io/kubesphere/pkg/apiserver/auditing/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/utils/iputil"
)
func TestGetAuditLevel(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelRequestResponse,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, string(webhook.Spec.AuditLevel), string(a.getAuditLevel()))
}
func TestAuditing_Enabled(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelNone,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, false, a.Enabled())
}
func TestAuditing_K8sAuditingEnabled(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelNone,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
assert.Equal(t, true, a.K8sAuditingEnabled())
}
func TestAuditing_LogRequestObject(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelRequestResponse,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
req := &http.Request{}
u, err := url.Parse("http://139.198.121.143:32306//kapis/tenant.kubesphere.io/v1alpha2/workspaces")
if err != nil {
panic(err)
}
req.URL = u
req.Header = http.Header{}
req.Header.Add(iputil.XClientIP, "192.168.0.2")
req = req.WithContext(request.WithUser(req.Context(), &user.DefaultInfo{
Name: "admin",
Groups: []string{
"system",
},
}))
info := &request.RequestInfo{
RequestInfo: &k8srequest.RequestInfo{
IsResourceRequest: false,
Path: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
Verb: "create",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
Resource: "workspaces",
Name: "test",
},
}
e := a.LogRequestObject(req, info)
expectedEvent := &v1alpha12.Event{
Event: audit.Event{
AuditID: e.AuditID,
Level: "RequestResponse",
Verb: "create",
Stage: "ResponseComplete",
User: v1.UserInfo{
Username: "admin",
Groups: []string{
"system",
},
Extra: make(map[string]v1.ExtraValue),
},
SourceIPs: []string{
"192.168.0.2",
},
RequestURI: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
RequestReceivedTimestamp: e.RequestReceivedTimestamp,
ObjectRef: &audit.ObjectReference{
Resource: "workspaces",
Namespace: "",
Name: "test",
UID: "",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
ResourceVersion: "",
Subresource: "",
},
},
}
assert.Equal(t, expectedEvent, e)
}
func TestAuditing_LogResponseObject(t *testing.T) {
webhook := &auditingv1alpha1.Webhook{
TypeMeta: metav1.TypeMeta{
APIVersion: auditingv1alpha1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: "kube-auditing-webhook",
},
Spec: auditingv1alpha1.WebhookSpec{
AuditLevel: auditingv1alpha1.LevelMetadata,
K8sAuditingEnabled: true,
},
}
ksClient := fake.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
a := auditing{
webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
}
err := fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Informer().GetIndexer().Add(webhook)
if err != nil {
panic(err)
}
req := &http.Request{}
u, err := url.Parse("http://139.198.121.143:32306//kapis/tenant.kubesphere.io/v1alpha2/workspaces")
if err != nil {
panic(err)
}
req.URL = u
req.Header = http.Header{}
req.Header.Add(iputil.XClientIP, "192.168.0.2")
req = req.WithContext(request.WithUser(req.Context(), &user.DefaultInfo{
Name: "admin",
Groups: []string{
"system",
},
}))
info := &request.RequestInfo{
RequestInfo: &k8srequest.RequestInfo{
IsResourceRequest: false,
Path: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
Verb: "create",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
Resource: "workspaces",
Name: "test",
},
}
e := a.LogRequestObject(req, info)
resp := NewResponseCapture(httptest.NewRecorder())
resp.WriteHeader(200)
a.LogResponseObject(e, resp)
expectedEvent := &v1alpha12.Event{
Event: audit.Event{
Verb: "create",
AuditID: e.AuditID,
Level: "Metadata",
Stage: "ResponseComplete",
User: v1.UserInfo{
Username: "admin",
Groups: []string{
"system",
},
},
SourceIPs: []string{
"192.168.0.2",
},
ObjectRef: &audit.ObjectReference{
Resource: "workspaces",
Name: "test",
APIGroup: "tenant.kubesphere.io",
APIVersion: "v1alpha2",
},
RequestReceivedTimestamp: e.RequestReceivedTimestamp,
StageTimestamp: e.StageTimestamp,
RequestURI: "/kapis/tenant.kubesphere.io/v1alpha2/workspaces",
ResponseStatus: &metav1.Status{
Code: 200,
},
},
}
expectedBs, err := json.Marshal(expectedEvent)
if err != nil {
panic(err)
}
bs, err := json.Marshal(e)
if err != nil {
panic(err)
}
assert.EqualValues(t, string(expectedBs), string(bs))
}
func TestResponseCapture_WriteHeader(t *testing.T) {
record := httptest.NewRecorder()
resp := NewResponseCapture(record)
resp.WriteHeader(404)
assert.EqualValues(t, 404, resp.StatusCode())
assert.EqualValues(t, 404, record.Code)
}
func TestResponseCapture_Write(t *testing.T) {
record := httptest.NewRecorder()
resp := NewResponseCapture(record)
body := []byte("123")
_, err := resp.Write(body)
if err != nil {
panic(err)
}
assert.EqualValues(t, body, resp.Bytes())
assert.EqualValues(t, body, record.Body.Bytes())
}

43
pkg/apiserver/auditing/v1alpha1/event.go
View File

@@ -1,43 +0,0 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
)
type Event struct {
// Devops project
Devops string
// The workspace which this audit event happened
Workspace string
// The cluster which this audit event happened
Cluster string
// Message send to user.
Message string
audit.Event
}
type EventList struct {
Items []Event
}
type Object struct {
v1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
}

129
pkg/apiserver/auditing/webhook/backend.go Normal file
View File

@@ -0,0 +1,129 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package webhook
import (
"bytes"
"context"
"crypto/tls"
"net/http"
"time"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/auditing/internal"
)
const (
GetSenderTimeout = time.Second
SendTimeout = time.Second * 3
DefaultSendersNum = 100
WebhookURL = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event"
)
type backend struct {
url string
senderCh chan interface{}
client http.Client
sendTimeout time.Duration
getSenderTimeout time.Duration
}
func NewBackend(url string, sendersNum int) internal.Backend {
b := backend{
url: url,
getSenderTimeout: GetSenderTimeout,
sendTimeout: SendTimeout,
}
if len(b.url) == 0 {
b.url = WebhookURL
}
num := sendersNum
if num == 0 {
num = DefaultSendersNum
}
b.senderCh = make(chan interface{}, num)
b.client = http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
Timeout: b.sendTimeout,
}
return &b
}
func (b *backend) ProcessEvents(events ...[]byte) {
go b.sendEvents(events...)
}
func (b *backend) sendEvents(events ...[]byte) {
ctx, cancel := context.WithTimeout(context.Background(), b.sendTimeout)
defer cancel()
stopCh := make(chan struct{})
skipReturnSender := false
send := func() {
ctx, cancel := context.WithTimeout(context.Background(), b.getSenderTimeout)
defer cancel()
select {
case <-ctx.Done():
klog.Error("Get auditing event sender timeout")
skipReturnSender = true
return
case b.senderCh <- struct{}{}:
}
start := time.Now()
defer func() {
stopCh <- struct{}{}
klog.V(8).Infof("send %d auditing events used %d", len(events), time.Since(start).Milliseconds())
}()
var body bytes.Buffer
for _, event := range events {
if _, err := body.Write(event); err != nil {
klog.Errorf("send auditing event error %s", err)
return
}
}
response, err := b.client.Post(b.url, "application/json", &body)
if err != nil {
klog.Errorf("send audit events error, %s", err)
return
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
klog.Errorf("send audit events error[%d]", response.StatusCode)
return
}
}
go send()
defer func() {
if !skipReturnSender {
<-b.senderCh
}
}()
select {
case <-ctx.Done():
klog.Error("send audit events timeout")
case <-stopCh:
}
}

32
pkg/apiserver/authentication/authenticators/basic/basic.go
View File

@@ -1,34 +1,22 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package basic
import (
"context"
"errors"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/models/auth"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
)
// TokenAuthenticator implements kubernetes token authenticate interface with our custom logic.
@@ -49,15 +37,15 @@ func NewBasicAuthenticator(authenticator auth.PasswordAuthenticator, loginRecord
}
func (t *basicAuthenticator) AuthenticatePassword(ctx context.Context, username, password string) (*authenticator.Response, bool, error) {
authenticated, provider, err := t.authenticator.Authenticate(ctx, "", username, password)
authenticated, err := t.authenticator.Authenticate(ctx, "", username, password)
if err != nil {
if t.loginRecorder != nil && err == auth.IncorrectPasswordError {
if t.loginRecorder != nil && errors.Is(err, auth.IncorrectPasswordError) {
var sourceIP, userAgent string
if requestInfo, ok := request.RequestInfoFrom(ctx); ok {
sourceIP = requestInfo.SourceIP
userAgent = requestInfo.UserAgent
}
if err := t.loginRecorder.RecordLogin(username, iamv1alpha2.Password, provider, sourceIP, userAgent, err); err != nil {
if err := t.loginRecorder.RecordLogin(ctx, username, iamv1beta1.Password, "", sourceIP, userAgent, err); err != nil {
klog.Errorf("Failed to record unsuccessful login attempt for user %s, error: %v", username, err)
}
}

80
pkg/apiserver/authentication/authenticators/jwt/jwt.go
View File

@@ -1,33 +1,25 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package jwt
import (
"context"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
corev1alpha1 "kubesphere.io/api/core/v1alpha1"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
"kubesphere.io/kubesphere/pkg/models/auth"
iamv1alpha2listers "kubesphere.io/kubesphere/pkg/client/listers/iam/v1alpha2"
"kubesphere.io/kubesphere/pkg/utils/serviceaccount"
)
// TokenAuthenticator implements kubernetes token authenticate interface with our custom logic.
@@ -37,13 +29,15 @@ import (
// because some resources are public accessible.
type tokenAuthenticator struct {
tokenOperator auth.TokenManagementInterface
userLister iamv1alpha2listers.UserLister
cache runtimecache.Cache
clusterRole string
}
func NewTokenAuthenticator(tokenOperator auth.TokenManagementInterface, userLister iamv1alpha2listers.UserLister) authenticator.Token {
func NewTokenAuthenticator(cache runtimecache.Cache, tokenOperator auth.TokenManagementInterface, clusterRole string) authenticator.Token {
return &tokenAuthenticator{
tokenOperator: tokenOperator,
userLister: userLister,
cache: cache,
clusterRole: clusterRole,
}
}
@@ -54,25 +48,51 @@ func (t *tokenAuthenticator) AuthenticateToken(ctx context.Context, token string
return nil, false, err
}
if verified.User.GetName() == iamv1alpha2.PreRegistrationUser {
if serviceaccount.IsServiceAccountToken(verified.Subject) {
if t.clusterRole == string(clusterv1alpha1.ClusterRoleHost) {
_, err = t.validateServiceAccount(ctx, verified)
if err != nil {
return nil, false, err
}
}
return &authenticator.Response{
User: verified.User,
}, true, nil
}
userInfo, err := t.userLister.Get(verified.User.GetName())
if err != nil {
return nil, false, err
if verified.User.GetName() == iamv1beta1.PreRegistrationUser {
return &authenticator.Response{
User: verified.User,
}, true, nil
}
// AuthLimitExceeded state should be ignored
if userInfo.Status.State == iamv1alpha2.UserDisabled {
return nil, false, auth.AccountIsNotActiveError
if t.clusterRole == string(clusterv1alpha1.ClusterRoleHost) {
userInfo := &iamv1beta1.User{}
if err := t.cache.Get(ctx, types.NamespacedName{Name: verified.User.GetName()}, userInfo); err != nil {
return nil, false, err
}
// AuthLimitExceeded state should be ignored
if userInfo.Status.State == iamv1beta1.UserDisabled {
return nil, false, auth.AccountIsNotActiveError
}
}
return &authenticator.Response{
User: &user.DefaultInfo{
Name: userInfo.GetName(),
Groups: append(userInfo.Spec.Groups, user.AllAuthenticated),
Name: verified.User.GetName(),
// TODO(wenhaozhou) Add user`s groups(can be searched by GroupBinding)
Groups: []string{user.AllAuthenticated},
},
}, true, nil
}
func (t *tokenAuthenticator) validateServiceAccount(ctx context.Context, verify *token.VerifiedResponse) (*corev1alpha1.ServiceAccount, error) {
// Ensure the relative service account exist
name, namespace := serviceaccount.SplitUsername(verify.Username)
sa := &corev1alpha1.ServiceAccount{}
if err := t.cache.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, sa); err != nil {
return nil, err
}
return sa, nil
}

20
pkg/apiserver/authentication/identityprovider/aliyunidaas/idaas.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package aliyunidaas
@@ -23,7 +12,6 @@ import (
"net/http"
"github.com/mitchellh/mapstructure"
"golang.org/x/oauth2"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
@@ -31,7 +19,7 @@ import (
)
func init() {
identityprovider.RegisterOAuthProvider(&idaasProviderFactory{})
identityprovider.RegisterOAuthProviderFactory(&idaasProviderFactory{})
}
type aliyunIDaaS struct {

17
pkg/apiserver/authentication/identityprovider/aliyunidaas/idaas_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package aliyunidaas

19
pkg/apiserver/authentication/identityprovider/cas/cas.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package cas
@@ -30,7 +19,7 @@ import (
)
func init() {
identityprovider.RegisterOAuthProvider(&casProviderFactory{})
identityprovider.RegisterOAuthProviderFactory(&casProviderFactory{})
}
type cas struct {

120
pkg/apiserver/authentication/identityprovider/configuration.go Normal file
View File

@@ -0,0 +1,120 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider
import (
"context"
"errors"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/server/options"
)
const (
MappingMethodManual MappingMethod = "manual"
MappingMethodAuto MappingMethod = "auto"
// MappingMethodLookup Looks up an existing identity, user identity mapping, and user, but does not automatically
// provision users or identities. Using this method requires you to manually provision users.
MappingMethodLookup MappingMethod = "lookup"
ConfigTypeIdentityProvider = "identityprovider"
SecretTypeIdentityProvider = "config.kubesphere.io/" + ConfigTypeIdentityProvider
SecretDataKey = "configuration.yaml"
)
var ErrorIdentityProviderNotFound = errors.New("the Identity provider was not found")
type MappingMethod string
type Configuration struct {
// The provider name.
Name string `json:"name" yaml:"name"`
// Defines how new identities are mapped to users when they login. Allowed values are:
// - manual: The user needs to confirm the mapped username on the onboarding page.
// - auto: Skip the onboarding screen, so the user cannot change its username.
// Fails if a user with that username is already mapped to another identity.
// - lookup: Looks up an existing identity, user identity mapping, and user, but does not automatically
// provision users or identities. Using this method requires you to manually provision users.
MappingMethod MappingMethod `json:"mappingMethod" yaml:"mappingMethod"`
// The type of identity provider
Type string `json:"type" yaml:"type"`
// The options of identify provider
ProviderOptions options.DynamicOptions `json:"provider" yaml:"provider"`
}
type ConfigurationGetter interface {
GetConfiguration(ctx context.Context, name string) (*Configuration, error)
ListConfigurations(ctx context.Context) ([]*Configuration, error)
}
func NewConfigurationGetter(client client.Client) ConfigurationGetter {
return &configurationGetter{client}
}
type configurationGetter struct {
client.Client
}
func (o *configurationGetter) ListConfigurations(ctx context.Context) ([]*Configuration, error) {
configurations := make([]*Configuration, 0)
secrets := &v1.SecretList{}
if err := o.List(ctx, secrets, client.InNamespace(constants.KubeSphereNamespace), client.MatchingLabels{constants.GenericConfigTypeLabel: ConfigTypeIdentityProvider}); err != nil {
klog.Errorf("failed to list secrets: %v", err)
return nil, err
}
for _, secret := range secrets.Items {
if secret.Type != SecretTypeIdentityProvider {
continue
}
if c, err := UnmarshalFrom(&secret); err != nil {
klog.Errorf("failed to unmarshal secret data: %s", err)
continue
} else {
configurations = append(configurations, c)
}
}
return configurations, nil
}
func (o *configurationGetter) GetConfiguration(ctx context.Context, name string) (*Configuration, error) {
configurations, err := o.ListConfigurations(ctx)
if err != nil {
klog.Errorf("failed to list identity providers: %v", err)
return nil, err
}
for _, c := range configurations {
if c.Name == name {
return c, nil
}
}
return nil, ErrorIdentityProviderNotFound
}
func UnmarshalFrom(secret *v1.Secret) (*Configuration, error) {
c := &Configuration{}
if err := yaml.Unmarshal(secret.Data[SecretDataKey], c); err != nil {
return nil, err
}
return c, nil
}
func IsIdentityProviderConfiguration(secret *v1.Secret) bool {
if secret.Namespace != constants.KubeSphereNamespace {
return false
}
return secret.Type == SecretTypeIdentityProvider
}

21
pkg/apiserver/authentication/identityprovider/generic_provider.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider
@@ -30,6 +17,6 @@ type GenericProvider interface {
type GenericProviderFactory interface {
// Type unique type of the provider
Type() string
// Apply the dynamic options from kubesphere-config
// Create generic identity provider
Create(options options.DynamicOptions) (GenericProvider, error)
}

19
pkg/apiserver/authentication/identityprovider/github/github.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package github
@@ -38,7 +27,7 @@ const (
)
func init() {
identityprovider.RegisterOAuthProvider(&ldapProviderFactory{})
identityprovider.RegisterOAuthProviderFactory(&ldapProviderFactory{})
}
type github struct {

26
pkg/apiserver/authentication/identityprovider/github/github_test.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package github
@@ -29,7 +16,7 @@ import (
"kubesphere.io/kubesphere/pkg/server/options"
. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
"golang.org/x/oauth2"
@@ -45,7 +32,7 @@ func TestGithub(t *testing.T) {
RunSpecs(t, "GitHub Identity Provider Suite")
}
var _ = BeforeSuite(func(done Done) {
var _ = BeforeSuite(func() {
githubServer = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var data map[string]interface{}
switch r.RequestURI {
@@ -69,8 +56,7 @@ var _ = BeforeSuite(func(done Done) {
w.Header().Add("Content-Type", "application/json")
json.NewEncoder(w).Encode(data)
}))
close(done)
}, 60)
})
var _ = AfterSuite(func() {
By("tearing down the test environment")

177
pkg/apiserver/authentication/identityprovider/gitlab/gitlab.go
View File

@@ -1,177 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package gitlab
import (
"context"
"crypto/tls"
"encoding/json"
"io"
"net/http"
"strconv"
"github.com/mitchellh/mapstructure"
"golang.org/x/oauth2"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/server/options"
)
const (
userInfoURL = "https://gitlab.com/api/v4/user"
authURL = "https://gitlab.com/oauth/authorize"
tokenURL = "https://gitlab.com/oauth/token"
)
func init() {
identityprovider.RegisterOAuthProvider(&gitlabProviderFactory{})
}
type gitlab struct {
// ClientID is the application's ID.
ClientID string `json:"clientID" yaml:"clientID"`
// ClientSecret is the application's secret.
ClientSecret string `json:"-" yaml:"clientSecret"`
// Endpoint contains the resource server's token endpoint
// URLs. These are constants specific to each server and are
// often available via site-specific packages, such as
// google.Endpoint or sso.endpoint.
Endpoint endpoint `json:"endpoint" yaml:"endpoint"`
// RedirectURL is the URL to redirect users going through
// the OAuth flow, after the resource owner's URLs.
RedirectURL string `json:"redirectURL" yaml:"redirectURL"`
// Used to turn off TLS certificate checks
InsecureSkipVerify bool `json:"insecureSkipVerify" yaml:"insecureSkipVerify"`
// Scope specifies optional requested permissions.
Scopes []string `json:"scopes" yaml:"scopes"`
Config *oauth2.Config `json:"-" yaml:"-"`
}
// endpoint represents an OAuth 2.0 provider's authorization and token
// endpoint URLs.
type endpoint struct {
AuthURL string `json:"authURL" yaml:"authURL"`
TokenURL string `json:"tokenURL" yaml:"tokenURL"`
UserInfoURL string `json:"userInfoURL" yaml:"userInfoURL"`
}
type gitlabIdentity struct {
ID int64 `json:"id"`
Username string `json:"username"`
Email string `json:"email"`
Name string `json:"name"`
State string `json:"state"`
AvatarURL string `json:"avatar_url"`
WebURL string `json:"web_url"`
}
type gitlabProviderFactory struct {
}
func (g *gitlabProviderFactory) Type() string {
return "GitlabIdentityProvider"
}
func (g *gitlabProviderFactory) Create(opts options.DynamicOptions) (identityprovider.OAuthProvider, error) {
var gitlab gitlab
if err := mapstructure.Decode(opts, &gitlab); err != nil {
return nil, err
}
if gitlab.Endpoint.AuthURL == "" {
gitlab.Endpoint.AuthURL = authURL
}
if gitlab.Endpoint.TokenURL == "" {
gitlab.Endpoint.TokenURL = tokenURL
}
if gitlab.Endpoint.UserInfoURL == "" {
gitlab.Endpoint.UserInfoURL = userInfoURL
}
// fixed options
opts["endpoint"] = options.DynamicOptions{
"authURL": gitlab.Endpoint.AuthURL,
"tokenURL": gitlab.Endpoint.TokenURL,
"userInfoURL": gitlab.Endpoint.UserInfoURL,
}
gitlab.Config = &oauth2.Config{
ClientID: gitlab.ClientID,
ClientSecret: gitlab.ClientSecret,
Endpoint: oauth2.Endpoint{
AuthURL: gitlab.Endpoint.AuthURL,
TokenURL: gitlab.Endpoint.TokenURL,
},
RedirectURL: gitlab.RedirectURL,
Scopes: gitlab.Scopes,
}
return &gitlab, nil
}
func (g gitlabIdentity) GetUserID() string {
return strconv.FormatInt(g.ID, 10)
}
func (g gitlabIdentity) GetUsername() string {
return g.Username
}
func (g gitlabIdentity) GetEmail() string {
return g.Email
}
func (g *gitlab) IdentityExchangeCallback(req *http.Request) (identityprovider.Identity, error) {
// OAuth2 callback, see also https://tools.ietf.org/html/rfc6749#section-4.1.2
code := req.URL.Query().Get("code")
ctx := req.Context()
if g.InsecureSkipVerify {
client := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
ctx = context.WithValue(ctx, oauth2.HTTPClient, client)
}
token, err := g.Config.Exchange(ctx, code)
if err != nil {
return nil, err
}
resp, err := oauth2.NewClient(ctx, oauth2.StaticTokenSource(token)).Get(g.Endpoint.UserInfoURL)
if err != nil {
return nil, err
}
data, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var gitlabIdentity gitlabIdentity
err = json.Unmarshal(data, &gitlabIdentity)
if err != nil {
return nil, err
}
return gitlabIdentity, nil
}

82
pkg/apiserver/authentication/identityprovider/gitlab/gitlab_test.go
View File

@@ -1,82 +0,0 @@
package gitlab
import (
"reflect"
"testing"
"golang.org/x/oauth2"
"gopkg.in/yaml.v3"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/server/options"
)
func Test_gitlabProviderFactory_Create(t *testing.T) {
type args struct {
opts options.DynamicOptions
}
mustUnmarshalYAML := func(data string) options.DynamicOptions {
var dynamicOptions options.DynamicOptions
_ = yaml.Unmarshal([]byte(data), &dynamicOptions)
return dynamicOptions
}
tests := []struct {
name string
args args
want identityprovider.OAuthProvider
wantErr bool
}{
{
name: "should create successfully",
args: args{opts: mustUnmarshalYAML(`
clientID: 035c18fc229c686e4652d7034
clientSecret: 75c82b42e54aaf25186140f5
endpoint:
userInfoUrl: "https://gitlab.com/api/v4/user"
authURL: "https://gitlab.com/oauth/authorize"
tokenURL: "https://gitlab.com/oauth/token"
redirectURL: "https://ks-console.kubesphere-system.svc/oauth/redirect/gitlab"
scopes:
- read
`)},
want: &gitlab{
ClientID: "035c18fc229c686e4652d7034",
ClientSecret: "75c82b42e54aaf25186140f5",
Endpoint: endpoint{
AuthURL: "https://gitlab.com/oauth/authorize",
TokenURL: "https://gitlab.com/oauth/token",
UserInfoURL: "https://gitlab.com/api/v4/user",
},
RedirectURL: "https://ks-console.kubesphere-system.svc/oauth/redirect/gitlab",
Scopes: []string{"read"},
Config: &oauth2.Config{
ClientID: "035c18fc229c686e4652d7034",
ClientSecret: "75c82b42e54aaf25186140f5",
Endpoint: oauth2.Endpoint{
AuthURL: "https://gitlab.com/oauth/authorize",
TokenURL: "https://gitlab.com/oauth/token",
AuthStyle: oauth2.AuthStyleAutoDetect,
},
RedirectURL: "https://ks-console.kubesphere-system.svc/oauth/redirect/gitlab",
Scopes: []string{"read"},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := &gitlabProviderFactory{}
got, err := g.Create(tt.args.opts)
if (err != nil) != tt.wantErr {
t.Errorf("Create() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Create() got = %v, want %v", got, tt.want)
}
})
}
}

18
pkg/apiserver/authentication/identityprovider/identity.go Normal file
View File

@@ -0,0 +1,18 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider
// Identity represents the account mapped to kubesphere
type Identity interface {
// GetUserID required
// Identifier for the End-User at the Issuer.
GetUserID() string
// GetUsername optional
// The username which the End-User wishes to be referred to kubesphere.
GetUsername() string
// GetEmail optional
GetEmail() string
}

110
pkg/apiserver/authentication/identityprovider/identity_provider.go
View File

@@ -1,110 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package identityprovider
import (
"errors"
"fmt"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
)
var (
oauthProviderFactories = make(map[string]OAuthProviderFactory)
genericProviderFactories = make(map[string]GenericProviderFactory)
identityProviderNotFound = errors.New("identity provider not found")
oauthProviders = make(map[string]OAuthProvider)
genericProviders = make(map[string]GenericProvider)
)
// Identity represents the account mapped to kubesphere
type Identity interface {
// GetUserID required
// Identifier for the End-User at the Issuer.
GetUserID() string
// GetUsername optional
// The username which the End-User wishes to be referred to kubesphere.
GetUsername() string
// GetEmail optional
GetEmail() string
}
// SetupWithOptions will verify the configuration and initialize the identityProviders
func SetupWithOptions(options []oauth.IdentityProviderOptions) error {
// Clear all providers when reloading configuration
oauthProviders = make(map[string]OAuthProvider)
genericProviders = make(map[string]GenericProvider)
for _, o := range options {
if oauthProviders[o.Name] != nil || genericProviders[o.Name] != nil {
err := fmt.Errorf("duplicate identity provider found: %s, name must be unique", o.Name)
klog.Error(err)
return err
}
if genericProviderFactories[o.Type] == nil && oauthProviderFactories[o.Type] == nil {
err := fmt.Errorf("identity provider %s with type %s is not supported", o.Name, o.Type)
klog.Error(err)
return err
}
if factory, ok := oauthProviderFactories[o.Type]; ok {
if provider, err := factory.Create(o.Provider); err != nil {
// don’t return errors, decoupling external dependencies
klog.Error(fmt.Sprintf("failed to create identity provider %s: %s", o.Name, err))
} else {
oauthProviders[o.Name] = provider
klog.V(4).Infof("create identity provider %s successfully", o.Name)
}
}
if factory, ok := genericProviderFactories[o.Type]; ok {
if provider, err := factory.Create(o.Provider); err != nil {
klog.Error(fmt.Sprintf("failed to create identity provider %s: %s", o.Name, err))
} else {
genericProviders[o.Name] = provider
klog.V(4).Infof("create identity provider %s successfully", o.Name)
}
}
}
return nil
}
// GetGenericProvider returns GenericProvider with given name
func GetGenericProvider(providerName string) (GenericProvider, error) {
if provider, ok := genericProviders[providerName]; ok {
return provider, nil
}
return nil, identityProviderNotFound
}
// GetOAuthProvider returns OAuthProvider with given name
func GetOAuthProvider(providerName string) (OAuthProvider, error) {
if provider, ok := oauthProviders[providerName]; ok {
return provider, nil
}
return nil, identityProviderNotFound
}
// RegisterOAuthProvider register OAuthProviderFactory with the specified type
func RegisterOAuthProvider(factory OAuthProviderFactory) {
oauthProviderFactories[factory.Type()] = factory
}
// RegisterGenericProvider registers GenericProviderFactory with the specified type
func RegisterGenericProvider(factory GenericProviderFactory) {
genericProviderFactories[factory.Type()] = factory
}

21
pkg/apiserver/authentication/identityprovider/identity_provider_factory.go Normal file
View File

@@ -0,0 +1,21 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider
var (
oauthProviderFactories = make(map[string]OAuthProviderFactory)
genericProviderFactories = make(map[string]GenericProviderFactory)
)
// RegisterOAuthProviderFactory register OAuthProviderFactory with the specified type
func RegisterOAuthProviderFactory(factory OAuthProviderFactory) {
oauthProviderFactories[factory.Type()] = factory
}
// RegisterGenericProviderFactory registers GenericProviderFactory with the specified type
func RegisterGenericProviderFactory(factory GenericProviderFactory) {
genericProviderFactories[factory.Type()] = factory
}

145
pkg/apiserver/authentication/identityprovider/identity_provider_test.go
View File

@@ -1,145 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package identityprovider
import (
"net/http"
"testing"
"kubesphere.io/kubesphere/pkg/server/options"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
)
type emptyOAuthProviderFactory struct {
typeName string
}
func (e emptyOAuthProviderFactory) Type() string {
return e.typeName
}
type emptyOAuthProvider struct {
}
type emptyIdentity struct {
}
func (e emptyIdentity) GetUserID() string {
return "test"
}
func (e emptyIdentity) GetUsername() string {
return "test"
}
func (e emptyIdentity) GetEmail() string {
return "test@test.com"
}
func (e emptyOAuthProvider) IdentityExchangeCallback(req *http.Request) (Identity, error) {
return emptyIdentity{}, nil
}
func (e emptyOAuthProviderFactory) Create(options options.DynamicOptions) (OAuthProvider, error) {
return emptyOAuthProvider{}, nil
}
type emptyGenericProviderFactory struct {
typeName string
}
func (e emptyGenericProviderFactory) Type() string {
return e.typeName
}
type emptyGenericProvider struct {
}
func (e emptyGenericProvider) Authenticate(username string, password string) (Identity, error) {
return emptyIdentity{}, nil
}
func (e emptyGenericProviderFactory) Create(options options.DynamicOptions) (GenericProvider, error) {
return emptyGenericProvider{}, nil
}
func TestSetupWith(t *testing.T) {
RegisterOAuthProvider(emptyOAuthProviderFactory{typeName: "GitHubIdentityProvider"})
RegisterOAuthProvider(emptyOAuthProviderFactory{typeName: "OIDCIdentityProvider"})
RegisterGenericProvider(emptyGenericProviderFactory{typeName: "LDAPIdentityProvider"})
type args struct {
options []oauth.IdentityProviderOptions
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "ldap",
args: args{options: []oauth.IdentityProviderOptions{
{
Name: "ldap",
MappingMethod: "auto",
Type: "LDAPIdentityProvider",
Provider: options.DynamicOptions{},
},
}},
wantErr: false,
},
{
name: "conflict",
args: args{options: []oauth.IdentityProviderOptions{
{
Name: "ldap",
MappingMethod: "auto",
Type: "LDAPIdentityProvider",
Provider: options.DynamicOptions{},
},
{
Name: "ldap",
MappingMethod: "auto",
Type: "LDAPIdentityProvider",
Provider: options.DynamicOptions{},
},
}},
wantErr: true,
},
{
name: "not supported",
args: args{options: []oauth.IdentityProviderOptions{
{
Name: "test",
MappingMethod: "auto",
Type: "NotSupported",
Provider: options.DynamicOptions{},
},
}},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := SetupWithOptions(tt.args.options); (err != nil) != tt.wantErr {
t.Errorf("SetupWithOptions() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

131
pkg/apiserver/authentication/identityprovider/identityprovider.go Normal file
View File

@@ -0,0 +1,131 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider
import (
"context"
"fmt"
"sync"
v1 "k8s.io/api/core/v1"
toolscache "k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
)
var SharedIdentityProviderController = NewController()
type Controller struct {
identityProviders *sync.Map
identityProviderConfigs *sync.Map
}
func NewController() *Controller {
return &Controller{identityProviders: &sync.Map{}, identityProviderConfigs: &sync.Map{}}
}
func (c *Controller) WatchConfigurationChanges(ctx context.Context, cache runtimecache.Cache) error {
informer, err := cache.GetInformer(ctx, &v1.Secret{})
if err != nil {
return fmt.Errorf("get informer failed: %w", err)
}
_, err = informer.AddEventHandler(toolscache.FilteringResourceEventHandler{
FilterFunc: func(obj interface{}) bool {
return IsIdentityProviderConfiguration(obj.(*v1.Secret))
},
Handler: &toolscache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
c.OnConfigurationChange(obj.(*v1.Secret))
},
UpdateFunc: func(old, new interface{}) {
c.OnConfigurationChange(new.(*v1.Secret))
},
DeleteFunc: func(obj interface{}) {
c.OnConfigurationDelete(obj.(*v1.Secret))
},
},
})
if err != nil {
return fmt.Errorf("add event handler failed: %w", err)
}
return nil
}
func (c *Controller) OnConfigurationDelete(secret *v1.Secret) {
configuration, err := UnmarshalFrom(secret)
if err != nil {
klog.Errorf("failed to unmarshal secret data: %s", err)
return
}
c.identityProviders.Delete(configuration.Name)
c.identityProviderConfigs.Delete(configuration.Name)
}
func (c *Controller) OnConfigurationChange(secret *v1.Secret) {
configuration, err := UnmarshalFrom(secret)
if err != nil {
klog.Errorf("failed to unmarshal secret data: %s", err)
return
}
if genericProviderFactories[configuration.Type] == nil && oauthProviderFactories[configuration.Type] == nil {
klog.Errorf("identity provider %s with type %s is not supported", configuration.Name, configuration.Type)
return
}
if factory, ok := oauthProviderFactories[configuration.Type]; ok {
if provider, err := factory.Create(configuration.ProviderOptions); err != nil {
// don’t return errors, decoupling external dependencies
klog.Error(fmt.Sprintf("failed to create identity provider %s: %s", configuration.Name, err))
} else {
c.identityProviders.Store(configuration.Name, provider)
c.identityProviderConfigs.Store(configuration.Name, configuration)
klog.Infof("create identity provider %s successfully", configuration.Name)
}
}
if factory, ok := genericProviderFactories[configuration.Type]; ok {
if provider, err := factory.Create(configuration.ProviderOptions); err != nil {
klog.Error(fmt.Sprintf("failed to create identity provider %s: %s", configuration.Name, err))
} else {
c.identityProviders.Store(configuration.Name, provider)
c.identityProviderConfigs.Store(configuration.Name, configuration)
klog.V(4).Infof("create identity provider %s successfully", configuration.Name)
}
}
}
func (c *Controller) GetGenericProvider(providerName string) (GenericProvider, bool) {
if obj, ok := c.identityProviders.Load(providerName); ok {
if provider, ok := obj.(GenericProvider); ok {
return provider, true
}
}
return nil, false
}
func (c *Controller) GetOAuthProvider(providerName string) (OAuthProvider, bool) {
if obj, ok := c.identityProviders.Load(providerName); ok {
if provider, ok := obj.(OAuthProvider); ok {
return provider, true
}
}
return nil, false
}
func (c *Controller) ListConfigurations() []*Configuration {
configurations := make([]*Configuration, 0)
c.identityProviderConfigs.Range(func(key, value any) bool {
if configuration, ok := value.(*Configuration); ok {
configurations = append(configurations, configuration)
}
return true
})
return configurations
}

19
pkg/apiserver/authentication/identityprovider/ldap/ldap.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package ldap
@@ -39,7 +28,7 @@ const (
)
func init() {
identityprovider.RegisterGenericProvider(&ldapProviderFactory{})
identityprovider.RegisterGenericProviderFactory(&ldapProviderFactory{})
}
type ldapProvider struct {

17
pkg/apiserver/authentication/identityprovider/ldap/ldap_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package ldap

17
pkg/apiserver/authentication/identityprovider/oauth_provider.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package identityprovider

19
pkg/apiserver/authentication/identityprovider/oidc/oidc.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oidc
@@ -36,7 +25,7 @@ import (
)
func init() {
identityprovider.RegisterOAuthProvider(&oidcProviderFactory{})
identityprovider.RegisterOAuthProviderFactory(&oidcProviderFactory{})
}
type oidcProvider struct {

26
pkg/apiserver/authentication/identityprovider/oidc/oidc_test.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oidc
@@ -36,7 +23,7 @@ import (
"kubesphere.io/kubesphere/pkg/server/options"
"github.com/golang-jwt/jwt/v4"
. "github.com/onsi/ginkgo"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/gexec"
"gopkg.in/square/go-jose.v2"
@@ -53,7 +40,7 @@ func TestOIDC(t *testing.T) {
RunSpecs(t, "OIDC Identity Provider Suite")
}
var _ = BeforeSuite(func(done Done) {
var _ = BeforeSuite(func() {
privateKey, err := rsa.GenerateKey(cryptorand.Reader, 2048)
Expect(err).Should(BeNil())
jwk := jose.JSONWebKey{
@@ -152,8 +139,7 @@ var _ = BeforeSuite(func(done Done) {
w.Header().Add("Content-Type", "application/json")
json.NewEncoder(w).Encode(data)
}))
close(done)
}, 60)
})
var _ = AfterSuite(func() {
By("tearing down the test environment")

126
pkg/apiserver/authentication/oauth/error.go
View File

@@ -1,26 +1,17 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oauth
import "fmt"
// The following error type is defined in https://datatracker.ietf.org/doc/html/rfc6749#section-5.2
var (
// ErrorInvalidClient
type ErrorType string
// The following error type is defined in https://datatracker.ietf.org/doc/html/rfc6749#section-4.1.2.1
const (
// InvalidClient
// Client authentication failed (e.g., unknown client, no
// client authentication included, or unsupported
// authentication method). The authorization server MAY
@@ -31,79 +22,98 @@ var (
// respond with an HTTP 401 (Unauthorized) status code and
// include the "WWW-Authenticate" response header field
// matching the authentication scheme used by the client.
ErrorInvalidClient = Error{Type: "invalid_client"}
InvalidClient ErrorType = "invalid_client"
// ErrorInvalidRequest The request is missing a required parameter,
// includes an unsupported parameter value (other than grant type),
// repeats a parameter, includes multiple credentials,
// utilizes more than one mechanism for authenticating the client,
// InvalidRequest
// The request is missing a required parameter, includes an unsupported parameter value (other than grant type),
// repeats a parameter, includes multiple credentials, utilizes more than one mechanism for authenticating the client,
// or is otherwise malformed.
ErrorInvalidRequest = Error{Type: "invalid_request"}
InvalidRequest ErrorType = "invalid_request"
// ErrorInvalidGrant
// InvalidGrant
// The provided authorization grant (e.g., authorization code,
// resource owner credentials) or refresh token is invalid, expired, revoked,
// does not match the redirection URI used in the authorization request,
// or was issued to another client.
ErrorInvalidGrant = Error{Type: "invalid_grant"}
InvalidGrant ErrorType = "invalid_grant"
// ErrorUnsupportedGrantType
// UnsupportedGrantType
// The authorization grant type is not supported by the authorization server.
ErrorUnsupportedGrantType = Error{Type: "unsupported_grant_type"}
UnsupportedGrantType ErrorType = "unsupported_grant_type"
ErrorUnsupportedResponseType = Error{Type: "unsupported_response_type"}
// UnsupportedResponseType
// The authorization server does not support obtaining an authorization code using this method.
UnsupportedResponseType ErrorType = "unsupported_response_type"
// ErrorUnauthorizedClient
// UnauthorizedClient
// The authenticated client is not authorized to use this authorization grant type.
ErrorUnauthorizedClient = Error{Type: "unauthorized_client"}
UnauthorizedClient ErrorType = "unauthorized_client"
// ErrorInvalidScope The requested scope is invalid, unknown, malformed,
// InvalidScope The requested scope is invalid, unknown, malformed,
// or exceeds the scope granted by the resource owner.
ErrorInvalidScope = Error{Type: "invalid_scope"}
InvalidScope ErrorType = "invalid_scope"
// ErrorLoginRequired The Authorization Server requires End-User authentication.
// LoginRequired The Authorization Server requires End-User authentication.
// This error MAY be returned when the prompt parameter value in the Authentication Request is none,
// but the Authentication Request cannot be completed without displaying a user interface
// for End-User authentication.
ErrorLoginRequired = Error{Type: "login_required"}
LoginRequired ErrorType = "login_required"
// ErrorServerError
// InteractionRequired
// The Authorization Server requires End-User interaction of some form to proceed.
// This error MAY be returned when the prompt parameter value in the Authentication Request is none,
// but the Authentication Request cannot be completed without displaying a user interface for End-User interaction.
InteractionRequired ErrorType = "interaction_required"
// ServerError
// The authorization server encountered an unexpected
// condition that prevented it from fulfilling the request.
// (This error code is needed because a 500 Internal Server
// Error HTTP status code cannot be returned to the client
// via an HTTP redirect.)
ErrorServerError = Error{Type: "server_error"}
ServerError ErrorType = "server_error"
)
func NewInvalidRequest(error error) Error {
err := ErrorInvalidRequest
err.Description = error.Error()
return err
func NewError(errorType ErrorType, description string) *Error {
return &Error{
Type: errorType,
Description: description,
}
}
func NewInvalidScope(error error) Error {
err := ErrorInvalidScope
err.Description = error.Error()
return err
func NewInvalidRequest(description string) *Error {
return &Error{
Type: InvalidRequest,
Description: description,
}
}
func NewInvalidClient(error error) Error {
err := ErrorInvalidClient
err.Description = error.Error()
return err
func NewInvalidScope(description string) *Error {
return &Error{
Type: InvalidScope,
Description: description,
}
}
func NewInvalidGrant(error error) Error {
err := ErrorInvalidGrant
err.Description = error.Error()
return err
func NewInvalidClient(description string) *Error {
return &Error{
Type: InvalidClient,
Description: description,
}
}
func NewServerError(error error) Error {
err := ErrorServerError
err.Description = error.Error()
return err
func NewInvalidGrant(description string) *Error {
return &Error{
Type: InvalidGrant,
Description: description,
}
}
func NewServerError(description string) *Error {
return &Error{
Type: ServerError,
Description: description,
}
}
// Error wrapped OAuth error Response, for more details: https://datatracker.ietf.org/doc/html/rfc6749#section-5.2
@@ -115,7 +125,7 @@ type Error struct {
// A single ASCII [USASCII] error code from the following:
// Values for the "error" parameter MUST NOT include characters
// outside the set %x20-21 / %x23-5B / %x5D-7E.
Type string `json:"error"`
Type ErrorType `json:"error"`
// Description OPTIONAL. Human-readable ASCII [USASCII] text providing
// additional information, used to assist the client developer in
// understanding the error that occurred.
@@ -124,6 +134,6 @@ type Error struct {
Description string `json:"error_description,omitempty"`
}
func (e Error) Error() string {
func (e *Error) Error() string {
return fmt.Sprintf("error=\"%s\", error_description=\"%s\"", e.Type, e.Description)
}

239
pkg/apiserver/authentication/oauth/oauth_client.go Normal file
View File

@@ -0,0 +1,239 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oauth
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
errorsutil "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
)
const (
GrantMethodAuto = "auto"
GrantMethodPrompt = "prompt"
GrantMethodDeny = "deny"
ConfigTypeOAuthClient = "oauthclient"
SecretTypeOAuthClient = "config.kubesphere.io/" + ConfigTypeOAuthClient
SecretDataKey = "configuration.yaml"
)
var (
ErrorClientNotFound = errors.New("the OAuth client was not found")
ErrorRedirectURLNotAllowed = errors.New("redirect URL is not allowed")
ValidGrantMethods = []string{GrantMethodAuto, GrantMethodPrompt, GrantMethodDeny}
)
// Client represents an OAuth client configuration.
type Client struct {
// Name is the unique identifier for the OAuth client. It is used as the client_id parameter
// when making requests to <master>/oauth/authorize.
Name string `json:"name" yaml:"name"`
// Secret is the unique secret associated with the client for secure communication.
Secret string `json:"-" yaml:"secret"`
// Trusted indicates whether the client is considered a trusted client.
Trusted bool `json:"trusted" yaml:"trusted"`
// GrantMethod determines how grant requests for this client should be handled. If no method is provided,
// the cluster default grant handling method will be used. Valid grant handling methods are:
// - auto: Always approves grant requests, useful for trusted clients.
// - prompt: Prompts the end user for approval of grant requests, useful for third-party clients.
// - deny: Always denies grant requests, useful for black-listed clients.
GrantMethod string `json:"grantMethod" yaml:"grantMethod"`
// RespondWithChallenges indicates whether the client prefers authentication needed responses
// in the form of challenges instead of redirects.
RespondWithChallenges bool `json:"respondWithChallenges,omitempty" yaml:"respondWithChallenges,omitempty"`
// ScopeRestrictions describes which scopes this client can request. Each requested scope
// is checked against each restriction. If any restriction matches, then the scope is allowed.
// If no restriction matches, then the scope is denied.
ScopeRestrictions []string `json:"scopeRestrictions,omitempty" yaml:"scopeRestrictions,omitempty"`
// RedirectURIs is a list of valid redirection URIs associated with the client.
RedirectURIs []string `json:"redirectURIs,omitempty" yaml:"redirectURIs,omitempty"`
// AccessTokenMaxAge overrides the default maximum age for access tokens granted to this client.
// The default value is 7200 seconds, and the minimum allowed value is 600 seconds.
AccessTokenMaxAgeSeconds int64 `json:"accessTokenMaxAgeSeconds,omitempty" yaml:"accessTokenMaxAgeSeconds,omitempty"`
// AccessTokenInactivityTimeout overrides the default token inactivity timeout
// for tokens granted to this client.
AccessTokenInactivityTimeoutSeconds int64 `json:"accessTokenInactivityTimeoutSeconds,omitempty" yaml:"accessTokenInactivityTimeoutSeconds,omitempty"`
}
type ClientGetter interface {
GetOAuthClient(ctx context.Context, name string) (*Client, error)
ListOAuthClients(ctx context.Context) ([]*Client, error)
}
func NewOAuthClientGetter(reader client.Reader) ClientGetter {
return &oauthClientGetter{reader}
}
type oauthClientGetter struct {
client.Reader
}
func (o *oauthClientGetter) ListOAuthClients(ctx context.Context) ([]*Client, error) {
clients := make([]*Client, 0)
secrets := &v1.SecretList{}
if err := o.List(ctx, secrets, client.InNamespace(constants.KubeSphereNamespace),
client.MatchingLabels{constants.GenericConfigTypeLabel: ConfigTypeOAuthClient}); err != nil {
return nil, err
}
for _, secret := range secrets.Items {
if secret.Type != SecretTypeOAuthClient {
continue
}
if c, err := UnmarshalFrom(&secret); err != nil {
klog.Errorf("failed to unmarshal secret data: %s", err)
continue
} else {
clients = append(clients, c)
}
}
return clients, nil
}
// GetOAuthClient retrieves an OAuth client by name from the underlying storage.
// It returns the OAuth client if found; otherwise, returns an error.
func (o *oauthClientGetter) GetOAuthClient(ctx context.Context, name string) (*Client, error) {
clients, err := o.ListOAuthClients(ctx)
if err != nil {
klog.Errorf("failed to list OAuth clients: %v", err)
return nil, err
}
for _, c := range clients {
if c.Name == name {
return c, nil
}
}
return nil, ErrorClientNotFound
}
// ValidateClient validates the properties of the provided OAuth 2.0 client.
// It checks the client's grant method, access token inactivity timeout, and access
// token max age for validity. If any validation fails, it returns an aggregated error.
func ValidateClient(client Client) error {
var validationErrors []error
// Validate grant method.
if !sliceutil.HasString(ValidGrantMethods, client.GrantMethod) {
validationErrors = append(validationErrors, fmt.Errorf("invalid grant method: %s", client.GrantMethod))
}
// Validate access token inactivity timeout.
if client.AccessTokenInactivityTimeoutSeconds != 0 && client.AccessTokenInactivityTimeoutSeconds < 600 {
validationErrors = append(validationErrors, fmt.Errorf("invalid access token inactivity timeout: %d, the minimum value can only be 600", client.AccessTokenInactivityTimeoutSeconds))
}
// Validate access token max age.
if client.AccessTokenMaxAgeSeconds != 0 && client.AccessTokenMaxAgeSeconds < 600 {
validationErrors = append(validationErrors, fmt.Errorf("invalid access token max age: %d, the minimum value can only be 600", client.AccessTokenMaxAgeSeconds))
}
// Aggregate validation errors and return.
return errorsutil.NewAggregate(validationErrors)
}
// ResolveRedirectURL resolves the redirect URL for the OAuth 2.0 authorization process.
// It takes an expected URL as a parameter and returns the resolved URL if it's allowed.
// If the expected URL is not provided, it uses the first available RedirectURI from the client.
func (c *Client) ResolveRedirectURL(expectURL string) (*url.URL, error) {
// Check if RedirectURIs are specified for the client.
if len(c.RedirectURIs) == 0 {
return nil, ErrorRedirectURLNotAllowed
}
// Get the list of redirectable URIs for the client.
redirectAbleURIs := filterValidRedirectURIs(c.RedirectURIs)
// If the expected URL is not provided, use the first available RedirectURI.
if expectURL == "" {
if len(redirectAbleURIs) > 0 {
return url.Parse(redirectAbleURIs[0])
} else {
// No RedirectURIs available for the client.
return nil, ErrorRedirectURLNotAllowed
}
}
// Check if the provided expected URL is allowed.
if sliceutil.HasString(redirectAbleURIs, expectURL) {
return url.Parse(expectURL)
}
// The provided expected URL is not allowed.
return nil, ErrorRedirectURLNotAllowed
}
// IsValidScope checks whether the requested scope is valid for the client.
// It compares each individual scope in the requested scope string with the client's
// allowed scope restrictions. If all scopes are allowed, it returns true; otherwise, false.
func (c *Client) IsValidScope(requestedScope string) bool {
// Split the requested scope string into individual scopes.
scopes := strings.Split(requestedScope, " ")
// Check each individual scope against the client's scope restrictions.
for _, scope := range scopes {
if !sliceutil.HasString(c.ScopeRestrictions, scope) {
// Log a message indicating the disallowed scope.
klog.V(4).Infof("Invalid scope: %s is not allowed for client %s", scope, c.Name)
return false
}
}
// All scopes are valid.
return true
}
// filterValidRedirectURIs filters out invalid redirect URIs from the given slice.
// It returns a new slice containing only valid URIs.
func filterValidRedirectURIs(redirectURIs []string) []string {
validURIs := make([]string, 0)
for _, uri := range redirectURIs {
// Check if the URI is valid by attempting to parse it.
_, err := url.Parse(uri)
if err == nil {
// The URI is valid, add it to the list of valid URIs.
validURIs = append(validURIs, uri)
}
}
return validURIs
}
func UnmarshalFrom(secret *v1.Secret) (*Client, error) {
oc := &Client{}
if err := yaml.Unmarshal(secret.Data[SecretDataKey], oc); err != nil {
return nil, err
}
return oc, nil
}
func MarshalInto(client *Client, secret *v1.Secret) error {
data, err := yaml.Marshal(client)
if err != nil {
return err
}
secret.Data = map[string][]byte{SecretDataKey: data}
return nil
}

39
pkg/apiserver/authentication/oauth/oauth_client_test.go Normal file
View File

@@ -0,0 +1,39 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oauth
import (
"reflect"
"testing"
v1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
)
func TestMarshalInto(t *testing.T) {
want := &Client{
Name: "test",
Secret: "test",
Trusted: false,
GrantMethod: "auto",
RedirectURIs: []string{"test"},
AccessTokenMaxAgeSeconds: 10000,
AccessTokenInactivityTimeoutSeconds: 10000,
}
secret := &v1.Secret{}
if err := MarshalInto(want, secret); err != nil {
t.Errorf("Error: %v", err)
}
got, err := UnmarshalFrom(secret)
if err != nil {
klog.Errorf("failed to unmarshal secret data: %s", err)
}
if !reflect.DeepEqual(want, got) {
t.Errorf("got %v, want %v", got, want)
}
}

38
pkg/apiserver/authentication/oauth/oidc.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oauth
@@ -32,18 +19,19 @@ const (
// ScopeProfile This scope value requests access to the End-User's default profile Claims,
// which are: name, family_name, given_name, middle_name, nickname, preferred_username,
// profile, picture, website, gender, birthdate, zoneinfo, locale, and updated_at.
ScopeProfile = "profile"
// ScopePhone This scope value requests access to the phone_number and phone_number_verified Claims.
ScopePhone = "phone"
// ScopeAddress This scope value requests access to the address Claim.
ScopeAddress = "address"
ResponseCode = "code"
ResponseIDToken = "id_token"
ResponseToken = "token"
ScopeProfile = "profile"
ResponseTypeCode = "code"
ResponseTypeIDToken = "id_token"
ResponseTypeToken = "token"
GrantTypePassword = "password"
GrantTypeRefreshToken = "refresh_token"
GrantTypeCode = "code"
GrantTypeAuthorizationCode = "authorization_code"
GrantTypeOTP = "otp"
)
var ValidScopes = []string{ScopeOpenID, ScopeEmail, ScopeProfile}
var ValidResponseTypes = []string{ResponseCode, ResponseIDToken, ResponseToken}
var ValidResponseTypes = []string{ResponseTypeCode, ResponseTypeIDToken, ResponseTypeToken}
func IsValidScopes(scopes []string) bool {
for _, scope := range scopes {

16
pkg/apiserver/authentication/oauth/oidc_test.go
View File

@@ -1,18 +1,8 @@
/*
Copyright 2021 The KubeSphere Authors.
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oauth
import "testing"

186
pkg/apiserver/authentication/oauth/options.go
View File

@@ -1,29 +1,14 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package oauth
import (
"errors"
"net/url"
"time"
"kubesphere.io/kubesphere/pkg/server/options"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
)
type GrantHandlerType string
@@ -31,36 +16,22 @@ type MappingMethod string
type IdentityProviderType string
const (
// GrantHandlerAuto auto-approves client authorization grant requests
GrantHandlerAuto GrantHandlerType = "auto"
// GrantHandlerPrompt prompts the user to approve new client authorization grant requests
GrantHandlerPrompt GrantHandlerType = "prompt"
// GrantHandlerDeny auto-denies client authorization grant requests
GrantHandlerDeny GrantHandlerType = "deny"
// MappingMethodAuto The default value.
// The user will automatically create and mapping when login successful.
// Fails if a user with that username is already mapped to another identity.
MappingMethodAuto MappingMethod = "auto"
// MappingMethodLookup Looks up an existing identity, user identity mapping, and user, but does not automatically
// provision users or identities. Using this method requires you to manually provision users.
MappingMethodLookup MappingMethod = "lookup"
// MappingMethodMixed A user entity can be mapped with multiple identifyProvider.
// MappingMethodMixed A user entity can be mapped with multiple identifyProvider.
// not supported yet.
MappingMethodMixed MappingMethod = "mixed"
DefaultIssuer string = "kubesphere"
)
var (
ErrorClientNotFound = errors.New("the OAuth client was not found")
ErrorProviderNotFound = errors.New("the identity provider was not found")
ErrorRedirectURLNotAllowed = errors.New("redirect URL is not allowed")
)
type Options struct {
// An Issuer Identifier is a case-sensitive URL using the https scheme that contains scheme,
type IssuerOptions struct {
// URL is a case-sensitive URL using the https scheme that contains scheme,
// host, and optionally, port number and path components and no query or fragment components.
Issuer string `json:"issuer,omitempty" yaml:"issuer,omitempty"`
URL string `json:"url,omitempty" yaml:"url,omitempty"`
// secret to sign jwt token
JWTSecret string `json:"-" yaml:"jwtSecret"`
// RSA private key file used to sign the id token
SignKey string `json:"signKey,omitempty" yaml:"signKey,omitempty"`
@@ -68,14 +39,9 @@ type Options struct {
// Raw RSA private key. Base64 encoded PEM file
SignKeyData string `json:"-,omitempty" yaml:"signKeyData,omitempty"`
// Register identity providers.
IdentityProviders []IdentityProviderOptions `json:"identityProviders,omitempty" yaml:"identityProviders,omitempty"`
// Register additional OAuth clients.
Clients []Client `json:"clients,omitempty" yaml:"clients,omitempty"`
// AccessTokenMaxAgeSeconds control the lifetime of access tokens. The default lifetime is 24 hours.
// 0 means no expiration.
// AccessTokenMaxAgeSeconds control the lifetime of access tokens.
// The default lifetime is 24 hours.
// Zero means no expiration.
AccessTokenMaxAge time.Duration `json:"accessTokenMaxAge" yaml:"accessTokenMaxAge"`
// Inactivity timeout for tokens
@@ -89,26 +55,34 @@ type Options struct {
// - X: Tokens time out if there is no activity
// The current minimum allowed value for X is 5 minutes
AccessTokenInactivityTimeout time.Duration `json:"accessTokenInactivityTimeout" yaml:"accessTokenInactivityTimeout"`
// Token verification maximum time difference, default to 10s.
// You should consider allowing a clock skew when checking the time-based values.
// This should be values of a few seconds, and we don’t recommend using more than 30 seconds for this purpose,
// as this would rather indicate problems with the server, rather than a common clock skew.
MaximumClockSkew time.Duration `json:"maximumClockSkew" yaml:"maximumClockSkew"`
}
type IdentityProviderOptions struct {
// The provider name.
Name string `json:"name" yaml:"name"`
// Defines how new identities are mapped to users when they login. Allowed values are:
// - auto: The default value.The user will automatically create and mapping when login successful.
// Fails if a user with that user name is already mapped to another identity.
// Defines how new identities are mapped to users when they login.
// Allowed values are:
// - auto: The default value.The user will automatically create and mapping when login is successful.
// Fails if a user with that username is already mapped to another identity.
// - lookup: Looks up an existing identity, user identity mapping, and user, but does not automatically
// provision users or identities. Using this method requires you to manually provision users.
// - mixed: A user entity can be mapped with multiple identifyProvider.
// provision users or identities.
// Using this method requires you to manually provision users.
// - mixed: A user entity can be mapped with multiple identifyProvider.
MappingMethod MappingMethod `json:"mappingMethod" yaml:"mappingMethod"`
// DisableLoginConfirmation means that when the user login successfully,
// reconfirm the account information is not required.
// DisableLoginConfirmation Skip the login confirmation screen, so user cannot change its username.
// Username is provided from ID Token.
// Username from IDP must math [a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*
DisableLoginConfirmation bool `json:"disableLoginConfirmation" yaml:"disableLoginConfirmation"`
// The type of identify provider
// The type of identity provider
// OpenIDIdentityProvider LDAPIdentityProvider GitHubIdentityProvider
Type string `json:"type" yaml:"type"`
@@ -125,7 +99,7 @@ type Token struct {
// The Type method returns either this or "Bearer", the default.
TokenType string `json:"token_type,omitempty"`
// RefreshToken is a token that's used by the application
// RefreshToken is a token used by the application
// (as opposed to the user) to refresh the access token
// if it expires.
RefreshToken string `json:"refresh_token,omitempty"`
@@ -137,104 +111,10 @@ type Token struct {
ExpiresIn int `json:"expires_in,omitempty"`
}
type Client struct {
// The name of the OAuth client is used as the client_id parameter when making requests to <master>/oauth/authorize
// and <master>/oauth/token.
Name string `json:"name,omitempty" yaml:"name,omitempty"`
// Secret is the unique secret associated with a client
Secret string `json:"-" yaml:"secret,omitempty"`
// RespondWithChallenges indicates whether the client wants authentication needed responses made
// in the form of challenges instead of redirects
RespondWithChallenges bool `json:"respondWithChallenges,omitempty" yaml:"respondWithChallenges,omitempty"`
// RedirectURIs is the valid redirection URIs associated with a client
RedirectURIs []string `json:"redirectURIs,omitempty" yaml:"redirectURIs,omitempty"`
// GrantMethod determines how to handle grants for this client. If no method is provided, the
// cluster default grant handling method will be used. Valid grant handling methods are:
// - auto: always approves grant requests, useful for trusted clients
// - prompt: prompts the end user for approval of grant requests, useful for third-party clients
// - deny: always denies grant requests, useful for black-listed clients
GrantMethod GrantHandlerType `json:"grantMethod,omitempty" yaml:"grantMethod,omitempty"`
// ScopeRestrictions describes which scopes this client can request. Each requested scope
// is checked against each restriction. If any restriction matches, then the scope is allowed.
// If no restriction matches, then the scope is denied.
ScopeRestrictions []string `json:"scopeRestrictions,omitempty" yaml:"scopeRestrictions,omitempty"`
// AccessTokenMaxAge overrides the default access token max age for tokens granted to this client.
AccessTokenMaxAge *time.Duration `json:"accessTokenMaxAge,omitempty" yaml:"accessTokenMaxAge,omitempty"`
// AccessTokenInactivityTimeout overrides the default token
// inactivity timeout for tokens granted to this client.
AccessTokenInactivityTimeout *time.Duration `json:"accessTokenInactivityTimeout,omitempty" yaml:"accessTokenInactivityTimeout,omitempty"`
}
var (
// AllowAllRedirectURI Allow any redirect URI if the redirectURI is defined in request
AllowAllRedirectURI = "*"
)
func (o *Options) OAuthClient(name string) (Client, error) {
for _, found := range o.Clients {
if found.Name == name {
return found, nil
}
}
return Client{}, ErrorClientNotFound
}
func (o *Options) IdentityProviderOptions(name string) (*IdentityProviderOptions, error) {
for _, found := range o.IdentityProviders {
if found.Name == name {
return &found, nil
}
}
return nil, ErrorProviderNotFound
}
func (c Client) anyRedirectAbleURI() []string {
uris := make([]string, 0)
for _, uri := range c.RedirectURIs {
_, err := url.Parse(uri)
if err == nil {
uris = append(uris, uri)
}
}
return uris
}
func (c Client) ResolveRedirectURL(expectURL string) (*url.URL, error) {
// RedirectURIs is empty
if len(c.RedirectURIs) == 0 {
return nil, ErrorRedirectURLNotAllowed
}
allowAllRedirectURI := sliceutil.HasString(c.RedirectURIs, AllowAllRedirectURI)
redirectAbleURIs := c.anyRedirectAbleURI()
if expectURL == "" {
// Need to specify at least one RedirectURI
if len(redirectAbleURIs) > 0 {
return url.Parse(redirectAbleURIs[0])
} else {
return nil, ErrorRedirectURLNotAllowed
}
}
if allowAllRedirectURI || sliceutil.HasString(redirectAbleURIs, expectURL) {
return url.Parse(expectURL)
}
return nil, ErrorRedirectURLNotAllowed
}
func NewOptions() *Options {
return &Options{
Issuer: DefaultIssuer,
IdentityProviders: make([]IdentityProviderOptions, 0),
Clients: make([]Client, 0),
func NewIssuerOptions() *IssuerOptions {
return &IssuerOptions{
AccessTokenMaxAge: time.Hour * 2,
AccessTokenInactivityTimeout: time.Hour * 2,
MaximumClockSkew: 10 * time.Second,
}
}

107
pkg/apiserver/authentication/oauth/options_test.go
View File

@@ -1,107 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package oauth
import (
"encoding/json"
"testing"
"gopkg.in/yaml.v3"
)
func TestClientResolveRedirectURL(t *testing.T) {
tests := []struct {
Name string
client Client
wantErr bool
expectURL string
}{
{
Name: "custom client test",
client: Client{
Name: "custom",
RespondWithChallenges: true,
RedirectURIs: []string{AllowAllRedirectURI, "https://foo.bar.com/oauth/cb"},
GrantMethod: GrantHandlerAuto,
},
wantErr: false,
expectURL: "https://foo.bar.com/oauth/cb",
},
{
Name: "custom client test",
client: Client{
Name: "custom",
RespondWithChallenges: true,
RedirectURIs: []string{"https://foo.bar.com/oauth/cb"},
GrantMethod: GrantHandlerAuto,
},
wantErr: true,
expectURL: "https://foo.bar.com/oauth/cb2",
},
}
for _, test := range tests {
redirectURL, err := test.client.ResolveRedirectURL(test.expectURL)
if (err != nil) != test.wantErr {
t.Errorf("ResolveRedirectURL() error = %+v, wantErr %+v", err, test.wantErr)
return
}
if redirectURL != nil && test.expectURL != redirectURL.String() {
t.Errorf("expected redirect url: %s, got: %s", test.expectURL, redirectURL)
}
}
}
func TestDynamicOptions_MarshalJSON(t *testing.T) {
config := `
accessTokenMaxAge: 1h
accessTokenInactivityTimeout: 30m
identityProviders:
- name: ldap
type: LDAPIdentityProvider
mappingMethod: auto
provider:
host: xxxx.sn.mynetname.net:389
managerDN: uid=root,cn=users,dc=xxxx,dc=sn,dc=mynetname,dc=net
managerPassword: xxxx
userSearchBase: dc=xxxx,dc=sn,dc=mynetname,dc=net
loginAttribute: uid
mailAttribute: mail
- name: github
type: GitHubIdentityProvider
mappingMethod: mixed
provider:
clientID: 'xxxxxx'
clientSecret: 'xxxxxx'
endpoint:
authURL: 'https://github.com/login/oauth/authorize'
tokenURL: 'https://github.com/login/oauth/access_token'
redirectURL: 'https://ks-console/oauth/redirect'
scopes:
- user
`
var options Options
if err := yaml.Unmarshal([]byte(config), &options); err != nil {
t.Error(err)
}
expected := `{"identityProviders":[{"name":"ldap","mappingMethod":"auto","disableLoginConfirmation":false,"type":"LDAPIdentityProvider","provider":{"host":"xxxx.sn.mynetname.net:389","loginAttribute":"uid","mailAttribute":"mail","managerDN":"uid=root,cn=users,dc=xxxx,dc=sn,dc=mynetname,dc=net","userSearchBase":"dc=xxxx,dc=sn,dc=mynetname,dc=net"}},{"name":"github","mappingMethod":"mixed","disableLoginConfirmation":false,"type":"GitHubIdentityProvider","provider":{"clientID":"xxxxxx","endpoint":{"authURL":"https://github.com/login/oauth/authorize","tokenURL":"https://github.com/login/oauth/access_token"},"redirectURL":"https://ks-console/oauth/redirect","scopes":["user"]}}],"accessTokenMaxAge":3600000000000,"accessTokenInactivityTimeout":1800000000000}`
output, _ := json.Marshal(options)
if expected != string(output) {
t.Errorf("expected: %s, but got: %s", expected, output)
}
}

53
pkg/apiserver/authentication/options.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package authentication
@@ -24,11 +11,9 @@ import (
"github.com/spf13/pflag"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/aliyunidaas"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/cas"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/github"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/gitlab"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/ldap"
_ "kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider/oidc"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
@@ -43,11 +28,7 @@ type Options struct {
// A user will be blocked for 10m if he/she logins with incorrect credentials for at least 5 times in 10m.
AuthenticateRateLimiterMaxTries int `json:"authenticateRateLimiterMaxTries" yaml:"authenticateRateLimiterMaxTries"`
AuthenticateRateLimiterDuration time.Duration `json:"authenticateRateLimiterDuration" yaml:"authenticateRateLimiterDuration"`
// Token verification maximum time difference, default to 10s.
// You should consider allowing a clock skew when checking the time-based values.
// This should be values of a few seconds, and we don’t recommend using more than 30 seconds for this purpose,
// as this would rather indicate problems with the server, rather than a common clock skew.
MaximumClockSkew time.Duration `json:"maximumClockSkew" yaml:"maximumClockSkew"`
// retention login history, records beyond this amount will be deleted
LoginHistoryRetentionPeriod time.Duration `json:"loginHistoryRetentionPeriod" yaml:"loginHistoryRetentionPeriod"`
// retention login history, records beyond this amount will be deleted
@@ -55,39 +36,30 @@ type Options struct {
LoginHistoryMaximumEntries int `json:"loginHistoryMaximumEntries,omitempty" yaml:"loginHistoryMaximumEntries,omitempty"`
// allow multiple users login from different location at the same time
MultipleLogin bool `json:"multipleLogin" yaml:"multipleLogin"`
// secret to sign jwt token
JwtSecret string `json:"-" yaml:"jwtSecret"`
// OAuthOptions defines options needed for integrated oauth plugins
OAuthOptions *oauth.Options `json:"oauthOptions" yaml:"oauthOptions"`
// KubectlImage is the image address we use to create kubectl pod for users who have admin access to the cluster.
KubectlImage string `json:"kubectlImage" yaml:"kubectlImage"`
// Issuer defines options needed for integrated oauth plugins
Issuer *oauth.IssuerOptions `json:"issuer" yaml:"issuer"`
}
func NewOptions() *Options {
return &Options{
AuthenticateRateLimiterMaxTries: 5,
AuthenticateRateLimiterDuration: time.Minute * 30,
MaximumClockSkew: 10 * time.Second,
LoginHistoryRetentionPeriod: time.Hour * 24 * 7,
LoginHistoryMaximumEntries: 100,
OAuthOptions: oauth.NewOptions(),
Issuer: oauth.NewIssuerOptions(),
MultipleLogin: false,
JwtSecret: "",
KubectlImage: "kubesphere/kubectl:v1.0.0",
}
}
func (options *Options) Validate() []error {
var errs []error
if len(options.JwtSecret) == 0 {
if len(options.Issuer.JWTSecret) == 0 {
errs = append(errs, errors.New("JWT secret MUST not be empty"))
}
if options.AuthenticateRateLimiterMaxTries > options.LoginHistoryMaximumEntries {
errs = append(errs, errors.New("authenticateRateLimiterMaxTries MUST not be greater than loginHistoryMaximumEntries"))
}
if err := identityprovider.SetupWithOptions(options.OAuthOptions.IdentityProviders); err != nil {
errs = append(errs, err)
}
return errs
}
@@ -95,10 +67,9 @@ func (options *Options) AddFlags(fs *pflag.FlagSet, s *Options) {
fs.IntVar(&options.AuthenticateRateLimiterMaxTries, "authenticate-rate-limiter-max-retries", s.AuthenticateRateLimiterMaxTries, "")
fs.DurationVar(&options.AuthenticateRateLimiterDuration, "authenticate-rate-limiter-duration", s.AuthenticateRateLimiterDuration, "")
fs.BoolVar(&options.MultipleLogin, "multiple-login", s.MultipleLogin, "Allow multiple login with the same account, disable means only one user can login at the same time.")
fs.StringVar(&options.JwtSecret, "jwt-secret", s.JwtSecret, "Secret to sign jwt token, must not be empty.")
fs.StringVar(&options.Issuer.JWTSecret, "jwt-secret", s.Issuer.JWTSecret, "Secret to sign jwt token, must not be empty.")
fs.DurationVar(&options.LoginHistoryRetentionPeriod, "login-history-retention-period", s.LoginHistoryRetentionPeriod, "login-history-retention-period defines how long login history should be kept.")
fs.IntVar(&options.LoginHistoryMaximumEntries, "login-history-maximum-entries", s.LoginHistoryMaximumEntries, "login-history-maximum-entries defines how many entries of login history should be kept.")
fs.DurationVar(&options.OAuthOptions.AccessTokenMaxAge, "access-token-max-age", s.OAuthOptions.AccessTokenMaxAge, "access-token-max-age control the lifetime of access tokens, 0 means no expiration.")
fs.StringVar(&s.KubectlImage, "kubectl-image", s.KubectlImage, "Setup the image used by kubectl terminal pod")
fs.DurationVar(&options.MaximumClockSkew, "maximum-clock-skew", s.MaximumClockSkew, "The maximum time difference between the system clocks of the ks-apiserver that issued a JWT and the ks-apiserver that verified the JWT.")
fs.DurationVar(&options.Issuer.AccessTokenMaxAge, "access-token-max-age", s.Issuer.AccessTokenMaxAge, "access-token-max-age control the lifetime of access tokens, 0 means no expiration.")
fs.DurationVar(&options.Issuer.MaximumClockSkew, "maximum-clock-skew", s.Issuer.MaximumClockSkew, "The maximum time difference between the system clocks of the ks-apiserver that issued a JWT and the ks-apiserver that verified the JWT.")
}

17
pkg/apiserver/authentication/request/anonymous/anonymous.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package anonymous

17
pkg/apiserver/authentication/request/anonymous/anonymous_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package anonymous

53
pkg/apiserver/authentication/token/issuer.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package token
@@ -33,7 +22,7 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
)
const (
@@ -93,9 +82,9 @@ type Claims struct {
// The following is well-known ID Token fields
// End-User's full name in displayable form including all name parts,
// End-User's full url in displayable form including all url parts,
// possibly including titles and suffixes, ordered according to the End-User's locale and preferences.
Name string `json:"name,omitempty"`
Name string `json:"url,omitempty"`
// String value used to associate a Client session with an ID Token, and to mitigate replay attacks.
// The value is passed through unmodified from the Authentication Request to the ID Token.
Nonce string `json:"nonce,omitempty"`
@@ -103,13 +92,13 @@ type Claims struct {
Email string `json:"email,omitempty"`
// End-User's locale, represented as a BCP47 [RFC5646] language tag.
Locale string `json:"locale,omitempty"`
// Shorthand name by which the End-User wishes to be referred to at the RP,
// Shorthand url by which the End-User wishes to be referred to at the RP,
PreferredUsername string `json:"preferred_username,omitempty"`
}
type issuer struct {
// Issuer Identity
name string
// Issuer Identifier
url string
// signing access_token and refresh_token
secret []byte
// signing id_token
@@ -127,7 +116,7 @@ func (s *issuer) IssueTo(request *IssueRequest) (string, error) {
RegisteredClaims: jwt.RegisteredClaims{
IssuedAt: jwt.NewNumericDate(issueAt),
Subject: request.User.GetName(),
Issuer: s.name,
Issuer: s.url,
},
}
@@ -253,19 +242,19 @@ func generatePrivateKeyData() ([]byte, error) {
return pemData, nil
}
func loadSignKey(options *authentication.Options) (*rsa.PrivateKey, string, error) {
func loadSignKey(config *oauth.IssuerOptions) (*rsa.PrivateKey, string, error) {
var signKey *rsa.PrivateKey
var signKeyData []byte
var err error
if options.OAuthOptions.SignKey != "" {
signKeyData, err = os.ReadFile(options.OAuthOptions.SignKey)
if config.SignKey != "" {
signKeyData, err = os.ReadFile(config.SignKey)
if err != nil {
klog.Errorf("issuer: failed to read private key file %s: %v", options.OAuthOptions.SignKey, err)
klog.Errorf("issuer: failed to read private key file %s: %v", config.SignKey, err)
return nil, "", err
}
} else if options.OAuthOptions.SignKeyData != "" {
signKeyData, err = base64.StdEncoding.DecodeString(options.OAuthOptions.SignKeyData)
} else if config.SignKeyData != "" {
signKeyData, err = base64.StdEncoding.DecodeString(config.SignKeyData)
if err != nil {
klog.Errorf("issuer: failed to decode sign key data: %s", err)
return nil, "", err
@@ -292,16 +281,16 @@ func loadSignKey(options *authentication.Options) (*rsa.PrivateKey, string, erro
return signKey, keyID, nil
}
func NewIssuer(options *authentication.Options) (Issuer, error) {
func NewIssuer(config *oauth.IssuerOptions) (Issuer, error) {
// TODO(hongming) automatically rotates keys
signKey, keyID, err := loadSignKey(options)
signKey, keyID, err := loadSignKey(config)
if err != nil {
return nil, err
}
return &issuer{
name: options.OAuthOptions.Issuer,
secret: []byte(options.JwtSecret),
maximumClockSkew: options.MaximumClockSkew,
url: config.URL,
secret: []byte(config.JWTSecret),
maximumClockSkew: config.MaximumClockSkew,
signKey: &Keys{
SigningKey: &jose.JSONWebKey{
Key: signKey,

71
pkg/apiserver/authentication/token/issuer_test.go
View File

@@ -1,18 +1,8 @@
/*
Copyright 2021 The KubeSphere Authors.
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package token
import (
@@ -26,7 +16,6 @@ import (
"gopkg.in/square/go-jose.v2"
"k8s.io/apiserver/pkg/authentication/user"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
)
@@ -62,28 +51,26 @@ PsSsOHhPx0g+Wl8K2+Edg3FQRZ1m0rQFAZn66jd96u85aA9NH/bw3A3VYUdVJyHh
func TestNewIssuer(t *testing.T) {
signKeyData := base64.StdEncoding.EncodeToString([]byte(privateKeyData))
options := &authentication.Options{
config := &oauth.IssuerOptions{
URL: "https://ks-console.kubesphere-system.svc",
SignKeyData: signKeyData,
MaximumClockSkew: 10 * time.Second,
JwtSecret: "test-secret",
OAuthOptions: &oauth.Options{
Issuer: "kubesphere",
SignKeyData: signKeyData,
},
JWTSecret: "test-secret",
}
got, err := NewIssuer(options)
got, err := NewIssuer(config)
if err != nil {
t.Fatal(err)
}
signKey, keyID, err := loadSignKey(options)
signKey, keyID, err := loadSignKey(config)
if err != nil {
t.Fatal(err)
}
want := &issuer{
name: options.OAuthOptions.Issuer,
secret: []byte(options.JwtSecret),
maximumClockSkew: options.MaximumClockSkew,
url: config.URL,
secret: []byte(config.JWTSecret),
maximumClockSkew: config.MaximumClockSkew,
signKey: &Keys{
SigningKey: &jose.JSONWebKey{
Key: signKey,
@@ -100,21 +87,19 @@ func TestNewIssuer(t *testing.T) {
},
}
if !reflect.DeepEqual(got, want) {
t.Errorf("NewIssuer() got = %v, want %v", got, want)
t.Errorf("NewIssuerOptions() got = %v, want %v", got, want)
return
}
}
func TestNewIssuerGenerateSignKey(t *testing.T) {
options := &authentication.Options{
config := &oauth.IssuerOptions{
URL: "https://ks-console.kubesphere-system.svc",
MaximumClockSkew: 10 * time.Second,
JwtSecret: "test-secret",
OAuthOptions: &oauth.Options{
Issuer: "kubesphere",
},
JWTSecret: "test-secret",
}
got, err := NewIssuer(options)
got, err := NewIssuer(config)
if err != nil {
t.Fatal(err)
}
@@ -129,7 +114,7 @@ func TestNewIssuerGenerateSignKey(t *testing.T) {
func Test_issuer_IssueTo(t *testing.T) {
type fields struct {
name string
url string
secret []byte
maximumClockSkew time.Duration
}
@@ -146,7 +131,7 @@ func Test_issuer_IssueTo(t *testing.T) {
{
name: "token is successfully issued",
fields: fields{
name: "kubesphere",
url: "kubesphere",
secret: []byte("kubesphere"),
maximumClockSkew: 0,
},
@@ -173,7 +158,7 @@ func Test_issuer_IssueTo(t *testing.T) {
{
name: "token is successfully issued",
fields: fields{
name: "kubesphere",
url: "kubesphere",
secret: []byte("kubesphere"),
maximumClockSkew: 0,
},
@@ -202,7 +187,7 @@ func Test_issuer_IssueTo(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &issuer{
name: tt.fields.name,
url: tt.fields.url,
secret: tt.fields.secret,
maximumClockSkew: tt.fields.maximumClockSkew,
}
@@ -220,7 +205,7 @@ func Test_issuer_IssueTo(t *testing.T) {
return
}
assert.Equal(t, got.TokenType, tt.want.TokenType)
assert.Equal(t, got.Issuer, tt.fields.name)
assert.Equal(t, got.Issuer, tt.fields.url)
assert.Equal(t, got.Username, tt.want.Username)
assert.Equal(t, got.Subject, tt.want.User.GetName())
assert.NotZero(t, got.IssuedAt)
@@ -230,7 +215,7 @@ func Test_issuer_IssueTo(t *testing.T) {
func Test_issuer_Verify(t *testing.T) {
type fields struct {
name string
url string
secret []byte
maximumClockSkew time.Duration
}
@@ -247,7 +232,7 @@ func Test_issuer_Verify(t *testing.T) {
{
name: "token validation failed",
fields: fields{
name: "kubesphere",
url: "kubesphere",
secret: []byte("kubesphere"),
maximumClockSkew: 0,
},
@@ -257,7 +242,7 @@ func Test_issuer_Verify(t *testing.T) {
{
name: "token is successfully verified",
fields: fields{
name: "kubesphere",
url: "kubesphere",
secret: []byte("kubesphere"),
maximumClockSkew: 0,
},
@@ -277,7 +262,7 @@ func Test_issuer_Verify(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &issuer{
name: tt.fields.name,
url: tt.fields.url,
secret: tt.fields.secret,
maximumClockSkew: tt.fields.maximumClockSkew,
}
@@ -290,7 +275,7 @@ func Test_issuer_Verify(t *testing.T) {
return
}
assert.Equal(t, got.TokenType, tt.want.TokenType)
assert.Equal(t, got.Issuer, tt.fields.name)
assert.Equal(t, got.Issuer, tt.fields.url)
assert.Equal(t, got.Username, tt.want.Username)
assert.Equal(t, got.Subject, tt.want.User.GetName())
assert.NotZero(t, got.IssuedAt)
@@ -335,7 +320,7 @@ func Test_issuer_keyFunc(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s, err := NewIssuer(authentication.NewOptions())
s, err := NewIssuer(oauth.NewIssuerOptions())
if err != nil {
t.Error(err)
return

24
pkg/apiserver/authorization/authorizer/interfaces.go
View File

@@ -1,19 +1,3 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package authorizer
import (
@@ -51,9 +35,6 @@ type Attributes interface {
// The namespace of the object, if a request is for a REST object.
GetNamespace() string
// The devops project of the object, if a request is for a REST object.
GetDevOps() string
// The kind of object, if a request is for a REST object.
GetResource() string
@@ -112,7 +93,6 @@ type AttributesRecord struct {
Cluster string
Workspace string
Namespace string
DevOps string
APIGroup string
APIVersion string
Resource string
@@ -148,10 +128,6 @@ func (a AttributesRecord) GetNamespace() string {
return a.Namespace
}
func (a AttributesRecord) GetDevOps() string {
return a.DevOps
}
func (a AttributesRecord) GetResource() string {
return a.Resource
}

18
pkg/apiserver/authorization/options.go
View File

@@ -1,21 +1,3 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package authorization
import (

178
pkg/apiserver/authorization/rbac/rbac.go
View File

@@ -1,19 +1,3 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// NOTE: This file is copied from k8s.io/kubernetes/plugin/pkg/auth/authorizer/rbac.
package rbac
@@ -24,20 +8,20 @@ import (
"fmt"
"github.com/open-policy-agent/opa/rego"
rbacv1 "k8s.io/api/rbac/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/authentication/serviceaccount"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
corev1alpha1 "kubesphere.io/api/core/v1alpha1"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/models/iam/am"
ksserviceaccount "kubesphere.io/kubesphere/pkg/utils/serviceaccount"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
"k8s.io/klog/v2"
rbacv1 "k8s.io/api/rbac/v1"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/authentication/user"
)
const (
@@ -45,7 +29,7 @@ const (
defaultRegoFileName = "authz.rego"
)
type RBACAuthorizer struct {
type Authorizer struct {
am am.AccessManagementInterface
}
@@ -90,7 +74,7 @@ func (r *ruleAccumulator) visit(_ fmt.Stringer, _ string, rule *rbacv1.PolicyRul
return true
}
func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (authorizer.Decision, string, error) {
func (r *Authorizer) Authorize(requestAttributes authorizer.Attributes) (authorizer.Decision, string, error) {
ruleCheckingVisitor := &authorizingVisitor{requestAttributes: requestAttributes}
r.visitRulesFor(requestAttributes, ruleCheckingVisitor.visit)
@@ -100,7 +84,7 @@ func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (aut
}
// Build a detailed log of the denial.
// Make the whole block conditional so we don't do a lot of string-building we won't use.
// Make the whole block conditional, so we don't do a lot of string-building we won't use.
if klog.V(4).Enabled() {
var operation string
if requestAttributes.IsResourceRequest() {
@@ -129,10 +113,10 @@ func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (aut
}
var scope string
if ns := requestAttributes.GetNamespace(); len(ns) > 0 {
scope = fmt.Sprintf("in namespace %q", ns)
} else if ws := requestAttributes.GetWorkspace(); len(ws) > 0 {
scope = fmt.Sprintf("in workspace %q", ws)
if requestAttributes.GetResourceScope() == request.NamespaceScope {
scope = fmt.Sprintf("in namespace %q", requestAttributes.GetNamespace())
} else if requestAttributes.GetResourceScope() == request.WorkspaceScope {
scope = fmt.Sprintf("in workspace %q", requestAttributes.GetWorkspace())
} else if requestAttributes.GetResourceScope() == request.ClusterScope {
scope = "cluster scope"
} else {
@@ -149,8 +133,8 @@ func (r *RBACAuthorizer) Authorize(requestAttributes authorizer.Attributes) (aut
return authorizer.DecisionNoOpinion, reason, nil
}
func NewRBACAuthorizer(am am.AccessManagementInterface) *RBACAuthorizer {
return &RBACAuthorizer{am: am}
func NewRBACAuthorizer(am am.AccessManagementInterface) *Authorizer {
return &Authorizer{am: am}
}
func ruleAllows(requestAttributes authorizer.Attributes, rule *rbacv1.PolicyRule) bool {
@@ -195,18 +179,16 @@ func regoPolicyAllows(requestAttributes authorizer.Attributes, regoPolicy string
return false
}
func (r *RBACAuthorizer) rulesFor(requestAttributes authorizer.Attributes) ([]rbacv1.PolicyRule, error) {
func (r *Authorizer) rulesFor(requestAttributes authorizer.Attributes) ([]rbacv1.PolicyRule, error) {
visitor := &ruleAccumulator{}
r.visitRulesFor(requestAttributes, visitor.visit)
return visitor.rules, utilerrors.NewAggregate(visitor.errors)
}
func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes, visitor func(source fmt.Stringer, regoPolicy string, rule *rbacv1.PolicyRule, err error) bool) {
if globalRoleBindings, err := r.am.ListGlobalRoleBindings(""); err != nil {
if !visitor(nil, "", nil, err) {
return
}
func (r *Authorizer) visitRulesFor(requestAttributes authorizer.Attributes, visitor func(source fmt.Stringer, regoPolicy string, rule *rbacv1.PolicyRule, err error) bool) {
if globalRoleBindings, err := r.am.ListGlobalRoleBindings("", ""); err != nil {
visitor(nil, "", nil, err)
return
} else {
sourceDescriber := &globalRoleBindingDescriber{}
for _, globalRoleBinding := range globalRoleBindings {
@@ -219,7 +201,7 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
visitor(nil, "", nil, err)
continue
}
sourceDescriber.binding = globalRoleBinding
sourceDescriber.binding = &globalRoleBinding
sourceDescriber.subject = &globalRoleBinding.Subjects[subjectIndex]
if !visitor(sourceDescriber, regoPolicy, nil, nil) {
return
@@ -236,35 +218,25 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
}
}
if requestAttributes.GetResourceScope() == request.WorkspaceScope ||
requestAttributes.GetResourceScope() == request.NamespaceScope ||
requestAttributes.GetResourceScope() == request.DevOpsScope {
var workspace string
var err error
// all of resource under namespace and devops belong to workspace
if requestAttributes.GetResourceScope() == request.NamespaceScope {
if workspace, err = r.am.GetNamespaceControlledWorkspace(requestAttributes.GetNamespace()); err != nil {
if !visitor(nil, "", nil, err) {
return
}
}
} else if requestAttributes.GetResourceScope() == request.DevOpsScope {
if workspace, err = r.am.GetDevOpsControlledWorkspace(requestAttributes.GetDevOps()); err != nil {
if !visitor(nil, "", nil, err) {
return
}
}
var targetWorkspace string
if requestAttributes.GetResourceScope() == request.NamespaceScope {
if workspace, err := r.am.GetNamespaceControlledWorkspace(requestAttributes.GetNamespace()); err != nil {
visitor(nil, "", nil, err)
return
} else {
targetWorkspace = workspace
}
}
if workspace == "" {
workspace = requestAttributes.GetWorkspace()
}
if requestAttributes.GetResourceScope() == request.WorkspaceScope {
targetWorkspace = requestAttributes.GetWorkspace()
}
if workspaceRoleBindings, err := r.am.ListWorkspaceRoleBindings("", nil, workspace); err != nil {
if !visitor(nil, "", nil, err) {
return
}
// workspace managed resources
if targetWorkspace != "" {
if workspaceRoleBindings, err := r.am.ListWorkspaceRoleBindings("", "", nil, targetWorkspace); err != nil {
visitor(nil, "", nil, err)
return
} else {
sourceDescriber := &workspaceRoleBindingDescriber{}
for _, workspaceRoleBinding := range workspaceRoleBindings {
@@ -275,9 +247,9 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
regoPolicy, rules, err := r.am.GetRoleReferenceRules(workspaceRoleBinding.RoleRef, "")
if err != nil {
visitor(nil, "", nil, err)
continue
return
}
sourceDescriber.binding = workspaceRoleBinding
sourceDescriber.binding = &workspaceRoleBinding
sourceDescriber.subject = &workspaceRoleBinding.Subjects[subjectIndex]
if !visitor(sourceDescriber, regoPolicy, nil, nil) {
return
@@ -291,38 +263,28 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
}
}
if requestAttributes.GetResourceScope() == request.NamespaceScope ||
requestAttributes.GetResourceScope() == request.DevOpsScope {
var targetNamespace string
if requestAttributes.GetResourceScope() == request.NamespaceScope {
targetNamespace = requestAttributes.GetNamespace()
}
namespace := requestAttributes.GetNamespace()
// list devops role binding
if requestAttributes.GetResourceScope() == request.DevOpsScope {
if relatedNamespace, err := r.am.GetDevOpsRelatedNamespace(requestAttributes.GetDevOps()); err != nil {
if !visitor(nil, "", nil, err) {
return
}
} else {
namespace = relatedNamespace
}
}
if roleBindings, err := r.am.ListRoleBindings("", nil, namespace); err != nil {
if !visitor(nil, "", nil, err) {
return
}
if targetNamespace != "" {
if roleBindings, err := r.am.ListRoleBindings("", "", nil, targetNamespace); err != nil {
visitor(nil, "", nil, err)
return
} else {
sourceDescriber := &roleBindingDescriber{}
for _, roleBinding := range roleBindings {
subjectIndex, applies := appliesTo(requestAttributes.GetUser(), roleBinding.Subjects, namespace)
subjectIndex, applies := appliesTo(requestAttributes.GetUser(), roleBinding.Subjects, targetNamespace)
if !applies {
continue
}
regoPolicy, rules, err := r.am.GetRoleReferenceRules(roleBinding.RoleRef, namespace)
regoPolicy, rules, err := r.am.GetRoleReferenceRules(roleBinding.RoleRef, targetNamespace)
if err != nil {
visitor(nil, "", nil, err)
continue
return
}
sourceDescriber.binding = roleBinding
sourceDescriber.binding = &roleBinding
sourceDescriber.subject = &roleBinding.Subjects[subjectIndex]
if !visitor(sourceDescriber, regoPolicy, nil, nil) {
return
@@ -336,10 +298,9 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
}
}
if clusterRoleBindings, err := r.am.ListClusterRoleBindings(""); err != nil {
if !visitor(nil, "", nil, err) {
return
}
if clusterRoleBindings, err := r.am.ListClusterRoleBindings("", ""); err != nil {
visitor(nil, "", nil, err)
return
} else {
sourceDescriber := &clusterRoleBindingDescriber{}
for _, clusterRoleBinding := range clusterRoleBindings {
@@ -350,9 +311,9 @@ func (r *RBACAuthorizer) visitRulesFor(requestAttributes authorizer.Attributes,
regoPolicy, rules, err := r.am.GetRoleReferenceRules(clusterRoleBinding.RoleRef, "")
if err != nil {
visitor(nil, "", nil, err)
continue
return
}
sourceDescriber.binding = clusterRoleBinding
sourceDescriber.binding = &clusterRoleBinding
sourceDescriber.subject = &clusterRoleBinding.Subjects[subjectIndex]
if !visitor(sourceDescriber, regoPolicy, nil, nil) {
return
@@ -386,8 +347,9 @@ func appliesToUser(user user.Info, subject rbacv1.Subject, namespace string) boo
return sliceutil.HasString(user.GetGroups(), subject.Name)
case rbacv1.ServiceAccountKind:
// default the namespace to namespace we're working in if its available. This allows rolebindings that reference
// SAs in th local namespace to avoid having to qualify them.
// Default the namespace to namespace we're working in if it's available.
// This allows role bindings that reference
// SAs in the local namespace to avoid having to qualify them.
saNamespace := namespace
if len(subject.Namespace) > 0 {
saNamespace = subject.Namespace
@@ -395,15 +357,23 @@ func appliesToUser(user user.Info, subject rbacv1.Subject, namespace string) boo
if len(saNamespace) == 0 {
return false
}
// use a more efficient comparison for RBAC checking
return serviceaccount.MatchesUsername(saNamespace, subject.Name, user.GetName())
switch subject.APIGroup {
case rbacv1.GroupName:
// use a more efficient comparison for RBAC checking
return serviceaccount.MatchesUsername(saNamespace, subject.Name, user.GetName())
case corev1alpha1.GroupName:
return ksserviceaccount.MatchesUsername(saNamespace, subject.Name, user.GetName())
default:
return false
}
default:
return false
}
}
type globalRoleBindingDescriber struct {
binding *iamv1alpha2.GlobalRoleBinding
binding *iamv1beta1.GlobalRoleBinding
subject *rbacv1.Subject
}
@@ -417,7 +387,7 @@ func (d *globalRoleBindingDescriber) String() string {
}
type clusterRoleBindingDescriber struct {
binding *rbacv1.ClusterRoleBinding
binding *iamv1beta1.ClusterRoleBinding
subject *rbacv1.Subject
}
@@ -431,7 +401,7 @@ func (d *clusterRoleBindingDescriber) String() string {
}
type workspaceRoleBindingDescriber struct {
binding *iamv1alpha2.WorkspaceRoleBinding
binding *iamv1beta1.WorkspaceRoleBinding
subject *rbacv1.Subject
}
@@ -445,7 +415,7 @@ func (d *workspaceRoleBindingDescriber) String() string {
}
type roleBindingDescriber struct {
binding *rbacv1.RoleBinding
binding *iamv1beta1.RoleBinding
subject *rbacv1.Subject
}

227
pkg/apiserver/authorization/rbac/rbac_test.go
View File

@@ -1,24 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package rbac
import (
"context"
"errors"
"hash/fnv"
"io"
@@ -27,36 +10,35 @@ import (
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
fakek8s "k8s.io/client-go/kubernetes/fake"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
"kubesphere.io/kubesphere/pkg/apiserver/request"
fakeks "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/iam/am"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/user"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
runtimefakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/models/iam/am"
"kubesphere.io/kubesphere/pkg/models/resources/v1beta1"
"kubesphere.io/kubesphere/pkg/scheme"
)
// StaticRoles is a rule resolver that resolves from lists of role objects.
type StaticRoles struct {
roles []*rbacv1.Role
roleBindings []*rbacv1.RoleBinding
clusterRoles []*rbacv1.ClusterRole
clusterRoleBindings []*rbacv1.ClusterRoleBinding
workspaceRoles []*iamv1alpha2.WorkspaceRole
workspaceRoleBindings []*iamv1alpha2.WorkspaceRoleBinding
globalRoles []*iamv1alpha2.GlobalRole
globalRoleBindings []*iamv1alpha2.GlobalRoleBinding
roles []*iamv1beta1.Role
roleBindings []*iamv1beta1.RoleBinding
clusterRoles []*iamv1beta1.ClusterRole
clusterRoleBindings []*iamv1beta1.ClusterRoleBinding
workspaceRoles []*iamv1beta1.WorkspaceRole
workspaceRoleBindings []*iamv1beta1.WorkspaceRoleBinding
globalRoles []*iamv1beta1.GlobalRole
globalRoleBindings []*iamv1beta1.GlobalRoleBinding
namespaces []*corev1.Namespace
}
func (r *StaticRoles) GetRole(namespace, name string) (*rbacv1.Role, error) {
func (r *StaticRoles) GetRole(namespace, name string) (*iamv1beta1.Role, error) {
if len(namespace) == 0 {
return nil, errors.New("must provide namespace when getting role")
}
@@ -68,21 +50,21 @@ func (r *StaticRoles) GetRole(namespace, name string) (*rbacv1.Role, error) {
return nil, errors.New("role not found")
}
func (r *StaticRoles) GetClusterRole(name string) (*rbacv1.ClusterRole, error) {
func (r *StaticRoles) GetClusterRole(name string) (*iamv1beta1.ClusterRole, error) {
for _, clusterRole := range r.clusterRoles {
if clusterRole.Name == name {
return clusterRole, nil
}
}
return nil, errors.New("clusterrole not found")
return nil, errors.New("cluster role not found")
}
func (r *StaticRoles) ListRoleBindings(namespace string) ([]*rbacv1.RoleBinding, error) {
func (r *StaticRoles) ListRoleBindings(namespace string) ([]*iamv1beta1.RoleBinding, error) {
if len(namespace) == 0 {
return nil, errors.New("must provide namespace when listing role bindings")
}
var roleBindingList []*rbacv1.RoleBinding
var roleBindingList []*iamv1beta1.RoleBinding
for _, roleBinding := range r.roleBindings {
if roleBinding.Namespace != namespace {
continue
@@ -92,17 +74,17 @@ func (r *StaticRoles) ListRoleBindings(namespace string) ([]*rbacv1.RoleBinding,
return roleBindingList, nil
}
func (r *StaticRoles) ListClusterRoleBindings() ([]*rbacv1.ClusterRoleBinding, error) {
func (r *StaticRoles) ListClusterRoleBindings() ([]*iamv1beta1.ClusterRoleBinding, error) {
return r.clusterRoleBindings, nil
}
// compute a hash of a policy rule so we can sort in a deterministic order
// compute a hash of a policy rule, so we can sort in a deterministic order
func hashOf(p rbacv1.PolicyRule) string {
hash := fnv.New32()
writeStrings := func(slis ...[]string) {
for _, sli := range slis {
for _, s := range sli {
io.WriteString(hash, s)
_, _ = io.WriteString(hash, s)
}
}
}
@@ -140,13 +122,13 @@ func TestRBACAuthorizer(t *testing.T) {
}
staticRoles := StaticRoles{
roles: []*rbacv1.Role{
roles: []*iamv1beta1.Role{
{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace1", Name: "readthings"},
Rules: []rbacv1.PolicyRule{ruleReadPods, ruleReadServices},
},
},
clusterRoles: []*rbacv1.ClusterRole{
clusterRoles: []*iamv1beta1.ClusterRole{
{
ObjectMeta: metav1.ObjectMeta{Name: "cluster-admin"},
Rules: []rbacv1.PolicyRule{ruleAdmin},
@@ -156,16 +138,16 @@ func TestRBACAuthorizer(t *testing.T) {
Rules: []rbacv1.PolicyRule{ruleWriteNodes},
},
},
workspaceRoles: []*iamv1alpha2.WorkspaceRole{
workspaceRoles: []*iamv1beta1.WorkspaceRole{
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-workspace-manager",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
Rules: []rbacv1.PolicyRule{ruleAdmin},
},
},
globalRoles: []*iamv1alpha2.GlobalRole{
globalRoles: []*iamv1beta1.GlobalRole{
{
ObjectMeta: metav1.ObjectMeta{
Name: "global-admin",
@@ -174,9 +156,12 @@ func TestRBACAuthorizer(t *testing.T) {
},
},
roleBindings: []*rbacv1.RoleBinding{
roleBindings: []*iamv1beta1.RoleBinding{
{
ObjectMeta: metav1.ObjectMeta{Namespace: "namespace1"},
ObjectMeta: metav1.ObjectMeta{
Namespace: "namespace1",
Name: "readthings",
},
Subjects: []rbacv1.Subject{
{Kind: rbacv1.UserKind, Name: "foobar"},
{Kind: rbacv1.GroupKind, Name: "group1"},
@@ -184,37 +169,40 @@ func TestRBACAuthorizer(t *testing.T) {
RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "readthings"},
},
},
workspaceRoleBindings: []*iamv1alpha2.WorkspaceRoleBinding{
workspaceRoleBindings: []*iamv1beta1.WorkspaceRoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-workspace-manager-tester",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindWorkspaceRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindWorkspaceRole,
Name: "system-workspace-workspace-manager",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindUser,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Name: "tester",
},
},
},
},
globalRoleBindings: []*iamv1alpha2.GlobalRoleBinding{
globalRoleBindings: []*iamv1beta1.GlobalRoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Name: "admin",
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindGlobalRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindGlobalRole,
Name: "global-admin",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindUser,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Name: "admin",
},
},
@@ -249,6 +237,12 @@ func TestRBACAuthorizer(t *testing.T) {
workspace: "system-workspace",
effectiveRules: []rbacv1.PolicyRule{ruleAdmin},
},
{
StaticRoles: staticRoles,
user: &user.DefaultInfo{Name: "tester"},
workspace: "not-exists-workspace",
effectiveRules: nil,
},
{
StaticRoles: staticRoles,
user: &user.DefaultInfo{Name: "foobar"},
@@ -315,9 +309,9 @@ func TestRBACAuthorizer(t *testing.T) {
}
func TestRBACAuthorizerMakeDecision(t *testing.T) {
t.Skipf("TODO: refactor this test case")
staticRoles := StaticRoles{
roles: []*rbacv1.Role{
roles: []*iamv1beta1.Role{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "kubesphere-system",
@@ -345,7 +339,7 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
},
},
},
clusterRoles: []*rbacv1.ClusterRole{
clusterRoles: []*iamv1beta1.ClusterRole{
{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-viewer",
@@ -371,11 +365,11 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
},
},
},
workspaceRoles: []*iamv1alpha2.WorkspaceRole{
workspaceRoles: []*iamv1beta1.WorkspaceRole{
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-admin",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
Rules: []rbacv1.PolicyRule{
{
@@ -388,7 +382,7 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-viewer",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
Rules: []rbacv1.PolicyRule{
{
@@ -399,7 +393,7 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
},
},
},
globalRoles: []*iamv1alpha2.GlobalRole{
globalRoles: []*iamv1beta1.GlobalRole{
{
ObjectMeta: metav1.ObjectMeta{
Name: "global-admin",
@@ -427,7 +421,7 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
},
},
roleBindings: []*rbacv1.RoleBinding{
roleBindings: []*iamv1beta1.RoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Namespace: "kubesphere-system",
@@ -452,20 +446,20 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "kubesphere-system-viewer"},
},
},
workspaceRoleBindings: []*iamv1alpha2.WorkspaceRoleBinding{
workspaceRoleBindings: []*iamv1beta1.WorkspaceRoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-admin",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindWorkspaceRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindWorkspaceRole,
Name: "system-workspace-admin",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
Kind: iamv1beta1.ResourceKindUser,
Name: "system-workspace-admin",
},
},
@@ -473,22 +467,22 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{
Name: "system-workspace-viewer",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindWorkspaceRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindWorkspaceRole,
Name: "system-workspace-viewer",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
Kind: iamv1beta1.ResourceKindUser,
Name: "system-workspace-viewer",
},
},
},
},
clusterRoleBindings: []*rbacv1.ClusterRoleBinding{
clusterRoleBindings: []*iamv1beta1.ClusterRoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-admin",
@@ -508,20 +502,20 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "cluster-viewer"},
},
},
globalRoleBindings: []*iamv1alpha2.GlobalRoleBinding{
globalRoleBindings: []*iamv1beta1.GlobalRoleBinding{
{
ObjectMeta: metav1.ObjectMeta{
Name: "admin",
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindGlobalRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindGlobalRole,
Name: "global-admin",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindUser,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Name: "admin",
},
},
@@ -531,14 +525,14 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
Name: "viewer",
},
RoleRef: rbacv1.RoleRef{
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1alpha2.ResourceKindGlobalRole,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindGlobalRole,
Name: "global-viewer",
},
Subjects: []rbacv1.Subject{
{
Kind: iamv1alpha2.ResourceKindUser,
APIGroup: iamv1alpha2.SchemeGroupVersion.Group,
Kind: iamv1beta1.ResourceKindUser,
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
Name: "viewer",
},
},
@@ -549,13 +543,13 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{
Name: "kubesphere-system",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "kube-system",
Labels: map[string]string{tenantv1alpha1.WorkspaceLabel: "system-workspace"},
Labels: map[string]string{tenantv1beta1.WorkspaceLabel: "system-workspace"},
},
},
},
@@ -671,7 +665,7 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
APIVersion: "v1alpha2",
Resource: "namespaces",
ResourceRequest: true,
ResourceScope: iamv1alpha2.ScopeWorkspace,
ResourceScope: iamv1beta1.ScopeWorkspace,
},
ExpectedDecision: authorizer.DecisionAllow,
},
@@ -856,71 +850,66 @@ func TestRBACAuthorizerMakeDecision(t *testing.T) {
}
}
func newMockRBACAuthorizer(staticRoles *StaticRoles) (*RBACAuthorizer, error) {
ksClient := fakeks.NewSimpleClientset()
k8sClient := fakek8s.NewSimpleClientset()
fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
k8sInformerFactory := fakeInformerFactory.KubernetesSharedInformerFactory()
ksInformerFactory := fakeInformerFactory.KubeSphereSharedInformerFactory()
func newMockRBACAuthorizer(staticRoles *StaticRoles) (*Authorizer, error) {
client := runtimefakeclient.NewClientBuilder().
WithScheme(scheme.Scheme).Build()
for _, role := range staticRoles.roles {
err := k8sInformerFactory.Rbac().V1().Roles().Informer().GetIndexer().Add(role)
if err != nil {
if err := client.Create(context.Background(), role.DeepCopy()); err != nil {
return nil, err
}
}
for _, roleBinding := range staticRoles.roleBindings {
err := k8sInformerFactory.Rbac().V1().RoleBindings().Informer().GetIndexer().Add(roleBinding)
if err != nil {
if err := client.Create(context.Background(), roleBinding.DeepCopy()); err != nil {
return nil, err
}
}
for _, clusterRole := range staticRoles.clusterRoles {
err := k8sInformerFactory.Rbac().V1().ClusterRoles().Informer().GetIndexer().Add(clusterRole)
if err != nil {
if err := client.Create(context.Background(), clusterRole.DeepCopy()); err != nil {
return nil, err
}
}
for _, clusterRoleBinding := range staticRoles.clusterRoleBindings {
err := k8sInformerFactory.Rbac().V1().ClusterRoleBindings().Informer().GetIndexer().Add(clusterRoleBinding)
if err != nil {
if err := client.Create(context.Background(), clusterRoleBinding.DeepCopy()); err != nil {
return nil, err
}
}
for _, workspaceRole := range staticRoles.workspaceRoles {
err := ksInformerFactory.Iam().V1alpha2().WorkspaceRoles().Informer().GetIndexer().Add(workspaceRole)
if err != nil {
if err := client.Create(context.Background(), workspaceRole.DeepCopy()); err != nil {
return nil, err
}
}
for _, workspaceRoleBinding := range staticRoles.workspaceRoleBindings {
err := ksInformerFactory.Iam().V1alpha2().WorkspaceRoleBindings().Informer().GetIndexer().Add(workspaceRoleBinding)
if err != nil {
if err := client.Create(context.Background(), workspaceRoleBinding.DeepCopy()); err != nil {
return nil, err
}
}
for _, globalRole := range staticRoles.globalRoles {
err := ksInformerFactory.Iam().V1alpha2().GlobalRoles().Informer().GetIndexer().Add(globalRole)
if err != nil {
if err := client.Create(context.Background(), globalRole.DeepCopy()); err != nil {
return nil, err
}
}
for _, globalRoleBinding := range staticRoles.globalRoleBindings {
err := ksInformerFactory.Iam().V1alpha2().GlobalRoleBindings().Informer().GetIndexer().Add(globalRoleBinding)
if err != nil {
if err := client.Create(context.Background(), globalRoleBinding.DeepCopy()); err != nil {
return nil, err
}
}
return NewRBACAuthorizer(am.NewReadOnlyOperator(fakeInformerFactory, nil)), nil
fakeCache := &informertest.FakeInformers{Scheme: scheme.Scheme}
resourceManager, err := v1beta1.New(context.Background(), client, fakeCache)
if err != nil {
return nil, err
}
return NewRBACAuthorizer(am.NewReadOnlyOperator(resourceManager)), nil
}
func TestAppliesTo(t *testing.T) {
@@ -985,7 +974,7 @@ func TestAppliesTo(t *testing.T) {
subjects: []rbacv1.Subject{
{Kind: rbacv1.UserKind, Name: "barfoo"},
{Kind: rbacv1.GroupKind, Name: "foobar"},
{Kind: rbacv1.ServiceAccountKind, Namespace: "kube-system", Name: "default"},
{Kind: rbacv1.ServiceAccountKind, APIGroup: rbacv1.GroupName, Namespace: "kube-system", Name: "default"},
},
user: &user.DefaultInfo{Name: "system:serviceaccount:kube-system:default"},
namespace: "default",

377
pkg/apiserver/config/config.go
View File

@@ -1,377 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"reflect"
"strings"
"sync"
"github.com/fsnotify/fsnotify"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/klog/v2"
networkv1alpha1 "kubesphere.io/api/network/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authorization"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/models/terminal"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
"kubesphere.io/kubesphere/pkg/simple/client/auditing"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
"kubesphere.io/kubesphere/pkg/simple/client/edgeruntime"
"kubesphere.io/kubesphere/pkg/simple/client/events"
"kubesphere.io/kubesphere/pkg/simple/client/gateway"
"kubesphere.io/kubesphere/pkg/simple/client/gpu"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/kubeedge"
"kubesphere.io/kubesphere/pkg/simple/client/ldap"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"kubesphere.io/kubesphere/pkg/simple/client/metering"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
"kubesphere.io/kubesphere/pkg/simple/client/network"
"kubesphere.io/kubesphere/pkg/simple/client/notification"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
)
// Package config saves configuration for running KubeSphere components
//
// Config can be configured from command line flags and configuration file.
// Command line flags hold higher priority than configuration file. But if
// component Endpoint/Host/APIServer was left empty, all of that component
// command line flags will be ignored, use configuration file instead.
// For example, we have configuration file
//
// mysql:
// host: mysql.kubesphere-system.svc
// username: root
// password: password
//
// At the same time, have command line flags like following:
//
// --mysql-host mysql.openpitrix-system.svc --mysql-username king --mysql-password 1234
//
// We will use `king:1234@mysql.openpitrix-system.svc` from command line flags rather
// than `root:password@mysql.kubesphere-system.svc` from configuration file,
// cause command line has higher priority. But if command line flags like following:
//
// --mysql-username root --mysql-password password
//
// we will `root:password@mysql.kubesphere-system.svc` as input, cause
// mysql-host is missing in command line flags, all other mysql command line flags
// will be ignored.
var (
// singleton instance of config package
_config = defaultConfig()
)
const (
// DefaultConfigurationName is the default name of configuration
defaultConfigurationName = "kubesphere"
// DefaultConfigurationPath the default location of the configuration file
defaultConfigurationPath = "/etc/kubesphere"
)
type config struct {
cfg *Config
cfgChangeCh chan Config
watchOnce sync.Once
loadOnce sync.Once
}
func (c *config) watchConfig() <-chan Config {
c.watchOnce.Do(func() {
viper.WatchConfig()
viper.OnConfigChange(func(in fsnotify.Event) {
cfg := New()
if err := viper.Unmarshal(cfg); err != nil {
klog.Warningf("config reload error: %v", err)
} else {
c.cfgChangeCh <- *cfg
}
})
})
return c.cfgChangeCh
}
func (c *config) loadFromDisk() (*Config, error) {
var err error
c.loadOnce.Do(func() {
if err = viper.ReadInConfig(); err != nil {
return
}
err = viper.Unmarshal(c.cfg)
})
return c.cfg, err
}
func defaultConfig() *config {
viper.SetConfigName(defaultConfigurationName)
viper.AddConfigPath(defaultConfigurationPath)
// Load from current working directory, only used for debugging
viper.AddConfigPath(".")
// Load from Environment variables
viper.SetEnvPrefix("kubesphere")
viper.AutomaticEnv()
viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
return &config{
cfg: New(),
cfgChangeCh: make(chan Config),
watchOnce: sync.Once{},
loadOnce: sync.Once{},
}
}
// Config defines everything needed for apiserver to deal with external services
type Config struct {
DevopsOptions *jenkins.Options `json:"devops,omitempty" yaml:"devops,omitempty" mapstructure:"devops"`
SonarQubeOptions *sonarqube.Options `json:"sonarqube,omitempty" yaml:"sonarQube,omitempty" mapstructure:"sonarqube"`
KubernetesOptions *k8s.KubernetesOptions `json:"kubernetes,omitempty" yaml:"kubernetes,omitempty" mapstructure:"kubernetes"`
ServiceMeshOptions *servicemesh.Options `json:"servicemesh,omitempty" yaml:"servicemesh,omitempty" mapstructure:"servicemesh"`
NetworkOptions *network.Options `json:"network,omitempty" yaml:"network,omitempty" mapstructure:"network"`
LdapOptions *ldap.Options `json:"-,omitempty" yaml:"ldap,omitempty" mapstructure:"ldap"`
CacheOptions *cache.Options `json:"cache,omitempty" yaml:"cache,omitempty" mapstructure:"cache"`
S3Options *s3.Options `json:"s3,omitempty" yaml:"s3,omitempty" mapstructure:"s3"`
OpenPitrixOptions *openpitrix.Options `json:"openpitrix,omitempty" yaml:"openpitrix,omitempty" mapstructure:"openpitrix"`
MonitoringOptions *prometheus.Options `json:"monitoring,omitempty" yaml:"monitoring,omitempty" mapstructure:"monitoring"`
LoggingOptions *logging.Options `json:"logging,omitempty" yaml:"logging,omitempty" mapstructure:"logging"`
AuthenticationOptions *authentication.Options `json:"authentication,omitempty" yaml:"authentication,omitempty" mapstructure:"authentication"`
AuthorizationOptions *authorization.Options `json:"authorization,omitempty" yaml:"authorization,omitempty" mapstructure:"authorization"`
MultiClusterOptions *multicluster.Options `json:"multicluster,omitempty" yaml:"multicluster,omitempty" mapstructure:"multicluster"`
EventsOptions *events.Options `json:"events,omitempty" yaml:"events,omitempty" mapstructure:"events"`
AuditingOptions *auditing.Options `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"`
AlertingOptions *alerting.Options `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"`
NotificationOptions *notification.Options `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"`
KubeEdgeOptions *kubeedge.Options `json:"kubeedge,omitempty" yaml:"kubeedge,omitempty" mapstructure:"kubeedge"`
EdgeRuntimeOptions *edgeruntime.Options `json:"edgeruntime,omitempty" yaml:"edgeruntime,omitempty" mapstructure:"edgeruntime"`
MeteringOptions *metering.Options `json:"metering,omitempty" yaml:"metering,omitempty" mapstructure:"metering"`
GatewayOptions *gateway.Options `json:"gateway,omitempty" yaml:"gateway,omitempty" mapstructure:"gateway"`
GPUOptions *gpu.Options `json:"gpu,omitempty" yaml:"gpu,omitempty" mapstructure:"gpu"`
TerminalOptions *terminal.Options `json:"terminal,omitempty" yaml:"terminal,omitempty" mapstructure:"terminal"`
}
// newConfig creates a default non-empty Config
func New() *Config {
return &Config{
DevopsOptions: jenkins.NewDevopsOptions(),
SonarQubeOptions: sonarqube.NewSonarQubeOptions(),
KubernetesOptions: k8s.NewKubernetesOptions(),
ServiceMeshOptions: servicemesh.NewServiceMeshOptions(),
NetworkOptions: network.NewNetworkOptions(),
LdapOptions: ldap.NewOptions(),
CacheOptions: cache.NewCacheOptions(),
S3Options: s3.NewS3Options(),
OpenPitrixOptions: openpitrix.NewOptions(),
MonitoringOptions: prometheus.NewPrometheusOptions(),
AlertingOptions: alerting.NewAlertingOptions(),
NotificationOptions: notification.NewNotificationOptions(),
LoggingOptions: logging.NewLoggingOptions(),
AuthenticationOptions: authentication.NewOptions(),
AuthorizationOptions: authorization.NewOptions(),
MultiClusterOptions: multicluster.NewOptions(),
EventsOptions: events.NewEventsOptions(),
AuditingOptions: auditing.NewAuditingOptions(),
KubeEdgeOptions: kubeedge.NewKubeEdgeOptions(),
EdgeRuntimeOptions: edgeruntime.NewEdgeRuntimeOptions(),
MeteringOptions: metering.NewMeteringOptions(),
GatewayOptions: gateway.NewGatewayOptions(),
GPUOptions: gpu.NewGPUOptions(),
TerminalOptions: terminal.NewTerminalOptions(),
}
}
// TryLoadFromDisk loads configuration from default location after server startup
// return nil error if configuration file not exists
func TryLoadFromDisk() (*Config, error) {
return _config.loadFromDisk()
}
// WatchConfigChange return config change channel
func WatchConfigChange() <-chan Config {
return _config.watchConfig()
}
// convertToMap simply converts config to map[string]bool
// to hide sensitive information
func (conf *Config) ToMap() map[string]bool {
conf.stripEmptyOptions()
result := make(map[string]bool, 0)
if conf == nil {
return result
}
c := reflect.Indirect(reflect.ValueOf(conf))
for i := 0; i < c.NumField(); i++ {
name := strings.Split(c.Type().Field(i).Tag.Get("json"), ",")[0]
if strings.HasPrefix(name, "-") {
continue
}
if name == "network" {
ippoolName := "network.ippool"
nsnpName := "network"
networkTopologyName := "network.topology"
if conf.NetworkOptions == nil {
result[nsnpName] = false
result[ippoolName] = false
} else {
if conf.NetworkOptions.EnableNetworkPolicy {
result[nsnpName] = true
} else {
result[nsnpName] = false
}
if conf.NetworkOptions.IPPoolType == networkv1alpha1.IPPoolTypeNone {
result[ippoolName] = false
} else {
result[ippoolName] = true
}
if conf.NetworkOptions.WeaveScopeHost == "" {
result[networkTopologyName] = false
} else {
result[networkTopologyName] = true
}
}
continue
}
if name == "openpitrix" {
// openpitrix is always true
result[name] = true
if conf.OpenPitrixOptions == nil {
result["openpitrix.appstore"] = false
} else {
result["openpitrix.appstore"] = !conf.OpenPitrixOptions.AppStoreConfIsEmpty()
}
continue
}
if c.Field(i).IsNil() {
result[name] = false
} else {
result[name] = true
}
}
return result
}
// Remove invalid options before serializing to json or yaml
func (conf *Config) stripEmptyOptions() {
if conf.CacheOptions != nil && conf.CacheOptions.Type == "" {
conf.CacheOptions = nil
}
if conf.DevopsOptions != nil && conf.DevopsOptions.Host == "" {
conf.DevopsOptions = nil
}
if conf.MonitoringOptions != nil && conf.MonitoringOptions.Endpoint == "" {
conf.MonitoringOptions = nil
}
if conf.SonarQubeOptions != nil && conf.SonarQubeOptions.Host == "" {
conf.SonarQubeOptions = nil
}
if conf.LdapOptions != nil && conf.LdapOptions.Host == "" {
conf.LdapOptions = nil
}
if conf.NetworkOptions != nil && conf.NetworkOptions.IsEmpty() {
conf.NetworkOptions = nil
}
if conf.ServiceMeshOptions != nil && conf.ServiceMeshOptions.IstioPilotHost == "" &&
conf.ServiceMeshOptions.ServicemeshPrometheusHost == "" &&
conf.ServiceMeshOptions.JaegerQueryHost == "" {
conf.ServiceMeshOptions = nil
}
if conf.S3Options != nil && conf.S3Options.Endpoint == "" {
conf.S3Options = nil
}
if conf.AlertingOptions != nil && conf.AlertingOptions.Endpoint == "" &&
conf.AlertingOptions.PrometheusEndpoint == "" && conf.AlertingOptions.ThanosRulerEndpoint == "" {
conf.AlertingOptions = nil
}
if conf.LoggingOptions != nil && conf.LoggingOptions.Host == "" {
conf.LoggingOptions = nil
}
if conf.NotificationOptions != nil && conf.NotificationOptions.Endpoint == "" {
conf.NotificationOptions = nil
}
if conf.MultiClusterOptions != nil && !conf.MultiClusterOptions.Enable {
conf.MultiClusterOptions = nil
}
if conf.EventsOptions != nil && conf.EventsOptions.Host == "" {
conf.EventsOptions = nil
}
if conf.AuditingOptions != nil && conf.AuditingOptions.Host == "" {
conf.AuditingOptions = nil
}
if conf.KubeEdgeOptions != nil && conf.KubeEdgeOptions.Endpoint == "" {
conf.KubeEdgeOptions = nil
}
if conf.EdgeRuntimeOptions != nil && conf.EdgeRuntimeOptions.Endpoint == "" {
conf.EdgeRuntimeOptions = nil
}
if conf.GPUOptions != nil && len(conf.GPUOptions.Kinds) == 0 {
conf.GPUOptions = nil
}
}
// GetFromConfigMap returns KubeSphere ruuning config by the given ConfigMap.
func GetFromConfigMap(cm *corev1.ConfigMap) (*Config, error) {
c := &Config{}
value, ok := cm.Data[constants.KubeSphereConfigMapDataKey]
if !ok {
return nil, fmt.Errorf("failed to get configmap kubesphere.yaml value")
}
if err := yaml.Unmarshal([]byte(value), c); err != nil {
return nil, fmt.Errorf("failed to unmarshal value from configmap. err: %s", err)
}
return c, nil
}

299
pkg/apiserver/config/config_test.go
View File

@@ -1,299 +0,0 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
"os"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"gopkg.in/yaml.v2"
networkv1alpha1 "kubesphere.io/api/network/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
"kubesphere.io/kubesphere/pkg/apiserver/authorization"
"kubesphere.io/kubesphere/pkg/models/terminal"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
"kubesphere.io/kubesphere/pkg/simple/client/auditing"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
"kubesphere.io/kubesphere/pkg/simple/client/edgeruntime"
"kubesphere.io/kubesphere/pkg/simple/client/events"
"kubesphere.io/kubesphere/pkg/simple/client/gateway"
"kubesphere.io/kubesphere/pkg/simple/client/gpu"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/kubeedge"
"kubesphere.io/kubesphere/pkg/simple/client/ldap"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"kubesphere.io/kubesphere/pkg/simple/client/metering"
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
"kubesphere.io/kubesphere/pkg/simple/client/network"
"kubesphere.io/kubesphere/pkg/simple/client/notification"
"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
"kubesphere.io/kubesphere/pkg/simple/client/s3"
"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
)
func newTestConfig() (*Config, error) {
var conf = &Config{
DevopsOptions: &jenkins.Options{
Host: "http://ks-devops.kubesphere-devops-system.svc",
Username: "jenkins",
Password: "kubesphere",
MaxConnections: 10,
},
SonarQubeOptions: &sonarqube.Options{
Host: "http://sonarqube.kubesphere-devops-system.svc",
Token: "ABCDEFG",
},
KubernetesOptions: &k8s.KubernetesOptions{
KubeConfig: "/Users/zry/.kube/config",
Master: "https://127.0.0.1:6443",
QPS: 1e6,
Burst: 1e6,
},
ServiceMeshOptions: &servicemesh.Options{
IstioPilotHost: "http://istio-pilot.istio-system.svc:9090",
JaegerQueryHost: "http://jaeger-query.istio-system.svc:80",
ServicemeshPrometheusHost: "http://prometheus-k8s.kubesphere-monitoring-system.svc",
},
LdapOptions: &ldap.Options{
Host: "http://openldap.kubesphere-system.svc",
ManagerDN: "cn=admin,dc=example,dc=org",
ManagerPassword: "P@88w0rd",
UserSearchBase: "ou=Users,dc=example,dc=org",
GroupSearchBase: "ou=Groups,dc=example,dc=org",
InitialCap: 10,
MaxCap: 100,
PoolName: "ldap",
},
CacheOptions: &cache.Options{
Type: "redis",
Options: map[string]interface{}{},
},
S3Options: &s3.Options{
Endpoint: "http://minio.openpitrix-system.svc",
Region: "us-east-1",
DisableSSL: false,
ForcePathStyle: false,
AccessKeyID: "ABCDEFGHIJKLMN",
SecretAccessKey: "OPQRSTUVWXYZ",
SessionToken: "abcdefghijklmn",
Bucket: "ssss",
},
OpenPitrixOptions: &openpitrix.Options{
S3Options: &s3.Options{
Endpoint: "http://minio.openpitrix-system.svc",
Region: "",
DisableSSL: false,
ForcePathStyle: false,
AccessKeyID: "ABCDEFGHIJKLMN",
SecretAccessKey: "OPQRSTUVWXYZ",
SessionToken: "abcdefghijklmn",
Bucket: "app",
},
ReleaseControllerOptions: &openpitrix.ReleaseControllerOptions{
MaxConcurrent: 10,
WaitTime: 30 * time.Second,
},
},
NetworkOptions: &network.Options{
EnableNetworkPolicy: true,
NSNPOptions: network.NSNPOptions{
AllowedIngressNamespaces: []string{},
},
WeaveScopeHost: "weave-scope-app.weave",
IPPoolType: networkv1alpha1.IPPoolTypeNone,
},
MonitoringOptions: &prometheus.Options{
Endpoint: "http://prometheus.kubesphere-monitoring-system.svc",
},
LoggingOptions: &logging.Options{
Host: "http://elasticsearch-logging.kubesphere-logging-system.svc:9200",
IndexPrefix: "elk",
Version: "6",
},
AlertingOptions: &alerting.Options{
Endpoint: "http://alerting-client-server.kubesphere-alerting-system.svc:9200/api",
PrometheusEndpoint: "http://prometheus-operated.kubesphere-monitoring-system.svc",
ThanosRulerEndpoint: "http://thanos-ruler-operated.kubesphere-monitoring-system.svc",
ThanosRuleResourceLabels: "thanosruler=thanos-ruler,role=thanos-alerting-rules",
},
NotificationOptions: &notification.Options{
Endpoint: "http://notification.kubesphere-alerting-system.svc:9200",
},
AuthorizationOptions: authorization.NewOptions(),
AuthenticationOptions: &authentication.Options{
AuthenticateRateLimiterMaxTries: 5,
AuthenticateRateLimiterDuration: 30 * time.Minute,
JwtSecret: "xxxxxx",
LoginHistoryMaximumEntries: 100,
MultipleLogin: false,
OAuthOptions: &oauth.Options{
Issuer: oauth.DefaultIssuer,
IdentityProviders: []oauth.IdentityProviderOptions{},
Clients: []oauth.Client{{
Name: "kubesphere-console-client",
Secret: "xxxxxx-xxxxxx-xxxxxx",
RespondWithChallenges: true,
RedirectURIs: []string{"http://ks-console.kubesphere-system.svc/oauth/token/implicit"},
GrantMethod: oauth.GrantHandlerAuto,
AccessTokenInactivityTimeout: nil,
}},
AccessTokenMaxAge: time.Hour * 24,
AccessTokenInactivityTimeout: 0,
},
},
MultiClusterOptions: multicluster.NewOptions(),
EventsOptions: &events.Options{
Host: "http://elasticsearch-logging-data.kubesphere-logging-system.svc:9200",
IndexPrefix: "ks-logstash-events",
Version: "6",
},
AuditingOptions: &auditing.Options{
Host: "http://elasticsearch-logging-data.kubesphere-logging-system.svc:9200",
IndexPrefix: "ks-logstash-auditing",
Version: "6",
},
KubeEdgeOptions: &kubeedge.Options{
Endpoint: "http://edge-watcher.kubeedge.svc/api/",
},
EdgeRuntimeOptions: &edgeruntime.Options{
Endpoint: "http://edgeservice.kubeedge.svc/api/",
},
MeteringOptions: &metering.Options{
RetentionDay: "7d",
},
GatewayOptions: &gateway.Options{
WatchesPath: "/etc/kubesphere/watches.yaml",
Namespace: "kubesphere-controls-system",
},
GPUOptions: &gpu.Options{
Kinds: []gpu.GPUKind{},
},
TerminalOptions: &terminal.Options{
Image: "alpine:3.15",
Timeout: 600,
},
}
return conf, nil
}
func saveTestConfig(t *testing.T, conf *Config) {
content, err := yaml.Marshal(conf)
if err != nil {
t.Fatalf("error marshal config. %v", err)
}
err = os.WriteFile(fmt.Sprintf("%s.yaml", defaultConfigurationName), content, 0640)
if err != nil {
t.Fatalf("error write configuration file, %v", err)
}
}
func cleanTestConfig(t *testing.T) {
file := fmt.Sprintf("%s.yaml", defaultConfigurationName)
if _, err := os.Stat(file); os.IsNotExist(err) {
t.Log("file not exists, skipping")
return
}
err := os.Remove(file)
if err != nil {
t.Fatalf("remove %s file failed", file)
}
}
func TestGet(t *testing.T) {
conf, err := newTestConfig()
if err != nil {
t.Fatal(err)
}
saveTestConfig(t, conf)
defer cleanTestConfig(t)
conf2, err := TryLoadFromDisk()
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(conf, conf2); diff != "" {
t.Fatal(diff)
}
}
func TestStripEmptyOptions(t *testing.T) {
var config Config
config.CacheOptions = &cache.Options{Type: ""}
config.DevopsOptions = &jenkins.Options{Host: ""}
config.MonitoringOptions = &prometheus.Options{Endpoint: ""}
config.SonarQubeOptions = &sonarqube.Options{Host: ""}
config.LdapOptions = &ldap.Options{Host: ""}
config.NetworkOptions = &network.Options{
EnableNetworkPolicy: false,
WeaveScopeHost: "",
IPPoolType: networkv1alpha1.IPPoolTypeNone,
}
config.ServiceMeshOptions = &servicemesh.Options{
IstioPilotHost: "",
ServicemeshPrometheusHost: "",
JaegerQueryHost: "",
}
config.S3Options = &s3.Options{
Endpoint: "",
}
config.AlertingOptions = &alerting.Options{
Endpoint: "",
PrometheusEndpoint: "",
ThanosRulerEndpoint: "",
}
config.LoggingOptions = &logging.Options{Host: ""}
config.NotificationOptions = &notification.Options{Endpoint: ""}
config.MultiClusterOptions = &multicluster.Options{Enable: false}
config.EventsOptions = &events.Options{Host: ""}
config.AuditingOptions = &auditing.Options{Host: ""}
config.KubeEdgeOptions = &kubeedge.Options{Endpoint: ""}
config.EdgeRuntimeOptions = &edgeruntime.Options{Endpoint: ""}
config.stripEmptyOptions()
if config.CacheOptions != nil ||
config.DevopsOptions != nil ||
config.MonitoringOptions != nil ||
config.SonarQubeOptions != nil ||
config.LdapOptions != nil ||
config.NetworkOptions != nil ||
config.ServiceMeshOptions != nil ||
config.S3Options != nil ||
config.AlertingOptions != nil ||
config.LoggingOptions != nil ||
config.NotificationOptions != nil ||
config.MultiClusterOptions != nil ||
config.EventsOptions != nil ||
config.AuditingOptions != nil ||
config.KubeEdgeOptions != nil ||
config.EdgeRuntimeOptions != nil {
t.Fatal("config stripEmptyOptions failed")
}
}

119
pkg/apiserver/filters/apiservice.go Normal file
View File

@@ -0,0 +1,119 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"encoding/base64"
"fmt"
"net/http"
"net/url"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/client-go/transport"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/cache"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/request"
)
type apiService struct {
next http.Handler
cache cache.Cache
}
func WithAPIService(next http.Handler, cache cache.Cache) http.Handler {
return &apiService{next: next, cache: cache}
}
func (s *apiService) ServeHTTP(w http.ResponseWriter, req *http.Request) {
requestInfo, _ := request.RequestInfoFrom(req.Context())
if requestInfo.IsKubernetesRequest {
s.next.ServeHTTP(w, req)
return
}
if !requestInfo.IsResourceRequest {
s.next.ServeHTTP(w, req)
return
}
var apiServices extensionsv1alpha1.APIServiceList
if err := s.cache.List(req.Context(), &apiServices); err != nil {
reason := fmt.Errorf("failed to list api services")
klog.Errorf("%v: %v", reason, err)
responsewriters.InternalError(w, req, errors.NewInternalError(reason))
return
}
for _, apiService := range apiServices.Items {
if apiService.Spec.Group != requestInfo.APIGroup || apiService.Spec.Version != requestInfo.APIVersion {
continue
}
if apiService.Status.State != extensionsv1alpha1.StateAvailable {
reason := fmt.Sprintf("apiService %s is not available", apiService.Name)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
s.handleProxyRequest(apiService, w, req)
return
}
s.next.ServeHTTP(w, req)
}
func (s *apiService) handleProxyRequest(apiService extensionsv1alpha1.APIService, w http.ResponseWriter, req *http.Request) {
endpoint, err := url.Parse(apiService.Spec.RawURL())
if err != nil {
reason := fmt.Sprintf("apiService %s is not available", apiService.Name)
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
location := &url.URL{}
location.Scheme = endpoint.Scheme
location.Host = endpoint.Host
location.Path = req.URL.Path
location.RawQuery = req.URL.Query().Encode()
newReq := req.WithContext(req.Context())
newReq.Header = utilnet.CloneHeader(req.Header)
newReq.URL = location
newReq.Host = location.Host
tlsConfig := transport.TLSConfig{
Insecure: apiService.Spec.InsecureSkipVerify,
}
if !apiService.Spec.InsecureSkipVerify && len(apiService.Spec.CABundle) > 0 {
caData, err := base64.StdEncoding.DecodeString(string(apiService.Spec.CABundle))
if err != nil {
reason := "failed to base64 decode cabundle"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
tlsConfig.CAData = caData
}
tr, err := transport.New(&transport.Config{
TLS: tlsConfig,
})
if err != nil {
reason := "failed to create transport.TLSConfig"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
user, _ := request.UserFrom(req.Context())
proxyRoundTripper := transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), tr)
upgrade := httpstream.IsUpgradeRequest(req)
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, false, upgrade, &responder{})
handler.ServeHTTP(w, newReq)
}

23
pkg/apiserver/filters/auditing.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
@@ -53,12 +42,6 @@ func (a *auditingFilter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
// Auditing should ignore k8s request when k8s auditing is enabled.
if info.IsKubernetesRequest && a.K8sAuditingEnabled() {
a.next.ServeHTTP(w, req)
return
}
if event := a.LogRequestObject(req, info); event != nil {
resp := auditing.NewResponseCapture(w)
a.next.ServeHTTP(resp, req)

28
pkg/apiserver/filters/authentication.go
View File

@@ -1,24 +1,12 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"errors"
"fmt"
"net/http"
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -72,8 +60,16 @@ func (a *authnFilter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
return
}
if err != nil {
klog.Errorf("Request authentication failed: %v", err)
}
gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}
responsewriters.ErrorNegotiated(apierrors.NewUnauthorized(fmt.Sprintf("Unauthorized: %s", err)), a.serializer, gv, w, req)
if err != nil {
err = apierrors.NewUnauthorized(err.Error())
} else {
err = apierrors.NewUnauthorized("The request cannot be authenticated.")
}
responsewriters.ErrorNegotiated(err, a.serializer, gv, w, req)
return
}

20
pkg/apiserver/filters/authorization.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
@@ -68,7 +57,7 @@ func (a *authzFilter) ServeHTTP(w http.ResponseWriter, req *http.Request) {
return
}
klog.V(4).Infof("Forbidden: %#v, Reason: %q", req.RequestURI, reason)
klog.V(4).Infof("Forbidden: %s %#v, User: %s", req.Method, req.RequestURI, attributes.GetUser().GetName())
responsewriters.Forbidden(ctx, attributes, w, req, reason, a.serializer)
}
@@ -98,7 +87,6 @@ func getAuthorizerAttributes(ctx context.Context) (authorizer.Attributes, error)
attribs.Resource = requestInfo.Resource
attribs.Subresource = requestInfo.Subresource
attribs.Namespace = requestInfo.Namespace
attribs.DevOps = requestInfo.DevOps
attribs.Name = requestInfo.Name
return &attribs, nil

46
pkg/apiserver/filters/dynamicresource.go
View File

@@ -1,3 +1,8 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
@@ -5,8 +10,13 @@ import (
"io"
"net/http"
tenantv1alpha1 "kubesphere.io/api/tenant/v1beta1"
"github.com/emicklei/go-restful/v3"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -80,31 +90,53 @@ func (d *DynamicResourceHandler) HandleServiceError(serviceError restful.Service
api.HandleError(w, req, err)
return
}
object, err = d.CreateObjectFromRawData(gvr, rawData)
if err != nil {
api.HandleError(w, req, err)
return
}
}
if reqInfo.Verb == request.VerbDelete {
object, err = d.GetResource(req.Request.Context(), gvr, reqInfo.Namespace, reqInfo.Name)
if err != nil {
api.HandleError(w, req, err)
return
}
}
var result interface{}
var result runtime.Object
switch reqInfo.Verb {
case request.VerbGet:
result, err = d.GetResource(req.Request.Context(), gvr, reqInfo.Namespace, reqInfo.Name)
obj, ok := result.(metav1.Object)
if reqInfo.Workspace != "" && ok && obj.GetLabels()[tenantv1alpha1.WorkspaceLabel] != reqInfo.Workspace {
err = errors.NewNotFound(gvr.GroupResource(), reqInfo.Name)
}
case request.VerbList:
result, err = d.ListResources(req.Request.Context(), gvr, reqInfo.Namespace, query.ParseQueryParameter(req))
q := query.ParseQueryParameter(req)
if reqInfo.Workspace != "" {
_ = q.AppendLabelSelector(map[string]string{tenantv1alpha1.WorkspaceLabel: reqInfo.Workspace})
}
result, err = d.ListResources(req.Request.Context(), gvr, reqInfo.Namespace, q)
case request.VerbCreate:
obj, ok := object.(metav1.Object)
if reqInfo.Workspace != "" && ok && obj.GetLabels()[tenantv1alpha1.WorkspaceLabel] != reqInfo.Workspace {
labels := obj.GetLabels()
if labels == nil {
labels = make(map[string]string)
}
labels[tenantv1alpha1.WorkspaceLabel] = reqInfo.Workspace
obj.SetLabels(labels)
}
err = d.CreateResource(req.Request.Context(), object)
case request.VerbUpdate:
err = d.UpdateResource(req.Request.Context(), object)
case request.VerbDelete:
err = d.DeleteResource(req.Request.Context(), gvr, reqInfo.Namespace, reqInfo.Name)
err = d.DeleteResource(req.Request.Context(), object)
case request.VerbPatch:
err = d.PatchResource(req.Request.Context(), object)
default:
err = NotSupportedVerbError
err = errors.NewBadRequest(NotSupportedVerbError.Error())
}
if err != nil {
@@ -116,5 +148,5 @@ func (d *DynamicResourceHandler) HandleServiceError(serviceError restful.Service
return
}
w.WriteAsJson(result)
_ = w.WriteAsJson(result)
}

91
pkg/apiserver/filters/filters.go Normal file
View File

@@ -0,0 +1,91 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"bufio"
"fmt"
"net"
"net/http"
"strconv"
"time"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/metrics"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/utils/iputil"
)
type metaResponseWriter struct {
http.ResponseWriter
statusCode int
size int
}
func newMetaResponseWriter(w http.ResponseWriter) *metaResponseWriter {
return &metaResponseWriter{
ResponseWriter: w,
statusCode: http.StatusOK,
}
}
func (r *metaResponseWriter) WriteHeader(code int) {
r.statusCode = code
r.ResponseWriter.WriteHeader(code)
}
func (r *metaResponseWriter) Write(b []byte) (int, error) {
size, err := r.ResponseWriter.Write(b)
r.size += size
return size, err
}
func (r *metaResponseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
hijacker, ok := r.ResponseWriter.(http.Hijacker)
if !ok {
return nil, nil, fmt.Errorf("ResponseWriter doesn't support Hijacker interface")
}
return hijacker.Hijack()
}
func WithGlobalFilter(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
wrapper := newMetaResponseWriter(w)
start := time.Now()
handler.ServeHTTP(wrapper, req)
elapsedTime := time.Since(start)
// Record metrics for each request
reqInfo, exists := request.RequestInfoFrom(req.Context())
if exists && reqInfo.APIGroup != "" {
metrics.RequestCounter.WithLabelValues(
reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource, strconv.Itoa(wrapper.statusCode),
).Inc()
metrics.RequestLatencies.WithLabelValues(
reqInfo.Verb, reqInfo.APIGroup, reqInfo.APIVersion, reqInfo.Resource,
).Observe(elapsedTime.Seconds())
}
// Record log for each request
logWithVerbose := klog.V(4)
// Always log error response
if wrapper.statusCode > http.StatusBadRequest {
logWithVerbose = klog.V(0)
}
logWithVerbose.Infof("%s - \"%s %s %s\" %d %d %dms",
iputil.RemoteIp(req),
req.Method,
req.URL,
req.Proto,
wrapper.statusCode,
wrapper.size,
elapsedTime.Microseconds(),
)
})
}

150
pkg/apiserver/filters/jsbundle.go Normal file
View File

@@ -0,0 +1,150 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"bytes"
"encoding/base64"
"io"
"net/http"
"net/url"
"strings"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/client-go/transport"
"k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/request"
)
type jsBundle struct {
next http.Handler
cache cache.Cache
}
func WithJSBundle(next http.Handler, cache cache.Cache) http.Handler {
return &jsBundle{next: next, cache: cache}
}
func (s *jsBundle) ServeHTTP(w http.ResponseWriter, req *http.Request) {
requestInfo, _ := request.RequestInfoFrom(req.Context())
if requestInfo.IsResourceRequest || requestInfo.IsKubernetesRequest {
s.next.ServeHTTP(w, req)
return
}
if !strings.HasPrefix(requestInfo.Path, extensionsv1alpha1.DistPrefix) {
s.next.ServeHTTP(w, req)
return
}
var jsBundles extensionsv1alpha1.JSBundleList
if err := s.cache.List(req.Context(), &jsBundles, &client.ListOptions{}); err != nil {
reason := "failed to list js bundles"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
for _, jsBundle := range jsBundles.Items {
if jsBundle.Status.State == extensionsv1alpha1.StateAvailable && jsBundle.Status.Link == requestInfo.Path {
w.Header().Set("Content-Type", "application/javascript; charset=utf-8")
if jsBundle.Spec.Raw != nil {
s.rawContent(jsBundle.Spec.Raw, w, req)
return
}
if jsBundle.Spec.RawFrom.ConfigMapKeyRef != nil {
s.rawFromConfigMap(jsBundle.Spec.RawFrom.ConfigMapKeyRef, w, req)
return
}
if jsBundle.Spec.RawFrom.SecretKeyRef != nil {
s.rawFromSecret(jsBundle.Spec.RawFrom.SecretKeyRef, w, req)
return
}
if jsBundle.Spec.RawFrom.URL != nil || jsBundle.Spec.RawFrom.Service != nil {
s.rawFromRemote(jsBundle.Spec.RawFrom.Endpoint, w, req)
return
}
}
}
s.next.ServeHTTP(w, req)
}
func (s *jsBundle) rawFromRemote(endpoint extensionsv1alpha1.Endpoint, w http.ResponseWriter, req *http.Request) {
location, err := url.Parse(endpoint.RawURL())
if err != nil {
reason := "failed to fetch content"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
tr, err := transport.New(&transport.Config{
TLS: transport.TLSConfig{
CAData: endpoint.CABundle,
Insecure: endpoint.InsecureSkipVerify,
},
})
if err != nil {
reason := "failed to create transport.TLSConfig"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
handler := proxy.NewUpgradeAwareHandler(location, tr, false, false, &responder{})
handler.UseLocationHost = true
handler.ServeHTTP(w, req)
}
func (s *jsBundle) rawContent(base64EncodedData []byte, w http.ResponseWriter, _ *http.Request) {
dec := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(base64EncodedData))
if _, err := io.Copy(w, dec); err != nil {
reason := "failed to decode raw content"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
}
}
func (s *jsBundle) rawFromConfigMap(configMapRef *extensionsv1alpha1.ConfigMapKeyRef, w http.ResponseWriter, req *http.Request) {
var cm v1.ConfigMap
ref := types.NamespacedName{
Namespace: configMapRef.Namespace,
Name: configMapRef.Name,
}
if err := s.cache.Get(req.Context(), ref, &cm); err != nil {
reason := "failed to fetch content from configMap"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
if cm.Data != nil {
_, _ = w.Write([]byte(cm.Data[configMapRef.Key]))
} else if cm.BinaryData != nil {
_, _ = w.Write(cm.BinaryData[configMapRef.Key])
}
}
func (s *jsBundle) rawFromSecret(secretRef *extensionsv1alpha1.SecretKeyRef, w http.ResponseWriter, req *http.Request) {
var secret v1.Secret
ref := types.NamespacedName{
Namespace: secretRef.Namespace,
Name: secretRef.Name,
}
if err := s.cache.Get(req.Context(), ref, &secret); err != nil {
reason := "failed to fetch content from secret"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
_, _ = w.Write(secret.Data[secretRef.Key])
}

56
pkg/apiserver/filters/kubeapiserver.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
@@ -21,32 +10,40 @@ import (
"net/http"
"net/url"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/client-go/rest"
"k8s.io/klog/v2"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/config"
)
type kubeAPIProxy struct {
next http.Handler
kubeAPIServer *url.URL
transport http.RoundTripper
options *config.ExperimentalOptions
}
// WithKubeAPIServer proxy request to kubernetes service if requests path starts with /api
func WithKubeAPIServer(next http.Handler, config *rest.Config) http.Handler {
func WithKubeAPIServer(next http.Handler, config *rest.Config, options *config.ExperimentalOptions) http.Handler {
kubeAPIServer, _ := url.Parse(config.Host)
transport, err := rest.TransportFor(config)
if err != nil {
klog.Errorf("Unable to create transport from rest.Config: %v", err)
return next
}
return &kubeAPIProxy{
next: next,
kubeAPIServer: kubeAPIServer,
transport: transport,
options: options,
}
}
@@ -58,15 +55,32 @@ func (k kubeAPIProxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
if info.IsKubernetesRequest {
s := *req.URL
s.Host = k.kubeAPIServer.Host
s.Scheme = k.kubeAPIServer.Scheme
location := &url.URL{}
location.Scheme = k.kubeAPIServer.Scheme
location.Host = k.kubeAPIServer.Host
location.Path = req.URL.Path
if k.options.ValidationDirective != "" &&
!req.URL.Query().Has(string(resource.QueryParamFieldValidation)) &&
(info.Verb == request.VerbCreate || info.Verb == request.VerbUpdate) {
params := req.URL.Query()
params.Set(string(resource.QueryParamFieldValidation), k.options.ValidationDirective)
req.URL.RawQuery = params.Encode()
}
location.RawQuery = req.URL.Query().Encode()
newReq := req.WithContext(req.Context())
newReq.Header = utilnet.CloneHeader(req.Header)
newReq.URL = location
newReq.Host = location.Host
// make sure we don't override kubernetes's authorization
req.Header.Del("Authorization")
httpProxy := proxy.NewUpgradeAwareHandler(&s, k.transport, true, false, &responder{})
newReq.Header.Del("Authorization")
upgrade := httpstream.IsUpgradeRequest(req)
httpProxy := proxy.NewUpgradeAwareHandler(location, k.transport, false, upgrade, &responder{})
httpProxy.UpgradeTransport = proxy.NewUpgradeRequestRoundTripper(k.transport, k.transport)
httpProxy.ServeHTTP(w, req)
httpProxy.ServeHTTP(w, newReq)
return
}

148
pkg/apiserver/filters/multicluster.go
View File

@@ -1,34 +1,32 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/klog/v2"
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
"kubesphere.io/kubesphere/pkg/apiserver/request"
clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
"kubesphere.io/kubesphere/pkg/multicluster"
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
)
@@ -36,26 +34,28 @@ const proxyURLFormat = "/api/v1/namespaces/kubesphere-system/services/:ks-apiser
type multiclusterDispatcher struct {
next http.Handler
clusterclient.ClusterClients
clusterclient.Interface
options *multicluster.Options
}
// WithMulticluster forward request to desired cluster based on request cluster name
// which included in request path clusters/{cluster}
func WithMulticluster(next http.Handler, clusterClient clusterclient.ClusterClients) http.Handler {
func WithMulticluster(next http.Handler, clusterClient clusterclient.Interface, options *multicluster.Options) http.Handler {
if clusterClient == nil {
klog.V(4).Infof("Multicluster dispatcher is disabled")
return next
}
return &multiclusterDispatcher{
next: next,
ClusterClients: clusterClient,
next: next,
Interface: clusterClient,
options: options,
}
}
func (m *multiclusterDispatcher) ServeHTTP(w http.ResponseWriter, req *http.Request) {
info, ok := request.RequestInfoFrom(req.Context())
if !ok {
responsewriters.InternalError(w, req, fmt.Errorf("no RequestInfo found in the context"))
responsewriters.InternalError(w, req, errors.NewInternalError(fmt.Errorf("no RequestInfo found in the context")))
return
}
if info.Cluster == "" {
@@ -63,7 +63,7 @@ func (m *multiclusterDispatcher) ServeHTTP(w http.ResponseWriter, req *http.Requ
return
}
cluster, err := m.Get(info.Cluster)
cluster, err := m.resolveCluster(info.Cluster)
if err != nil {
if errors.IsNotFound(err) {
responsewriters.WriteRawJSON(http.StatusBadRequest, errors.NewBadRequest(fmt.Sprintf("cluster %s not exists", info.Cluster)), w)
@@ -74,38 +74,43 @@ func (m *multiclusterDispatcher) ServeHTTP(w http.ResponseWriter, req *http.Requ
}
// request cluster is host cluster, no need go through agent
if m.IsHostCluster(cluster) {
if clusterutils.IsHostCluster(cluster) {
req.URL.Path = strings.Replace(req.URL.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1)
m.next.ServeHTTP(w, req)
return
}
if !m.IsClusterReady(cluster) {
responsewriters.InternalError(w, req, fmt.Errorf("cluster %s is not ready", cluster.Name))
if !clusterutils.IsClusterReady(cluster) {
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(fmt.Sprintf("cluster %s is not ready", cluster.Name)), w)
return
}
innCluster := m.GetInnerCluster(cluster.Name)
if innCluster == nil {
responsewriters.InternalError(w, req, fmt.Errorf("cluster %s is not ready", cluster.Name))
clusterClient, err := m.GetClusterClient(cluster.Name)
if err != nil {
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(err.Error()), w)
return
}
transport := http.DefaultTransport
location := &url.URL{}
location.Path = strings.Replace(req.URL.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1)
location.RawQuery = req.URL.Query().Encode()
// change request host to actually cluster hosts
u := *req.URL
u.Path = strings.Replace(u.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1)
// WithContext creates a shallow clone of the request with the same context.
newReq := req.WithContext(req.Context())
newReq.Header = utilnet.CloneHeader(req.Header)
newReq.URL = location
newReq.Host = location.Host
var transport http.RoundTripper
// if cluster connection is direct and kubesphere apiserver endpoint is empty
// we use kube-apiserver proxy way
if cluster.Spec.Connection.Type == clusterv1alpha1.ConnectionTypeDirect &&
len(cluster.Spec.Connection.KubeSphereAPIEndpoint) == 0 {
u.Scheme = innCluster.KubernetesURL.Scheme
u.Host = innCluster.KubernetesURL.Host
u.Path = fmt.Sprintf(proxyURLFormat, u.Path)
transport = innCluster.Transport
location.Scheme = clusterClient.KubernetesURL.Scheme
location.Host = clusterClient.KubernetesURL.Host
location.Path = fmt.Sprintf(proxyURLFormat, location.Path)
transport = clusterClient.Transport
// The reason we need this is kube-apiserver doesn't behave like a standard proxy, it will strip
// authorization header of proxy requests. Use custom header to avoid stripping by kube-apiserver.
@@ -113,20 +118,20 @@ func (m *multiclusterDispatcher) ServeHTTP(w http.ResponseWriter, req *http.Requ
// We first copy req.Header['Authorization'] to req.Header['X-KubeSphere-Authorization'] before sending
// designated cluster kube-apiserver, then copy req.Header['X-KubeSphere-Authorization'] to
// req.Header['Authorization'] before authentication.
req.Header.Set("X-KubeSphere-Authorization", req.Header.Get("Authorization"))
newReq.Header.Set("X-KubeSphere-Authorization", req.Header.Get("Authorization"))
// If cluster kubeconfig using token authentication, transport will not override authorization header,
// this will cause requests reject by kube-apiserver since kubesphere authorization header is not
// acceptable. Delete this header is safe since we are using X-KubeSphere-Authorization.
// https://github.com/kubernetes/client-go/blob/master/transport/round_trippers.go#L285
req.Header.Del("Authorization")
newReq.Header.Del("Authorization")
// Dirty trick again. The kube-apiserver apiserver proxy rejects all proxy requests with dryRun parameter
// https://github.com/kubernetes/kubernetes/pull/66083
// Really don't understand why they do this. And here we are, bypass with replacing 'dryRun'
// with dryrun and switch bach before send to kube-apiserver on the other side.
if len(u.Query()["dryRun"]) != 0 {
req.URL.RawQuery = strings.Replace(req.URL.RawQuery, "dryRun", "dryrun", 1)
if len(newReq.URL.Query()["dryRun"]) != 0 {
newReq.URL.RawQuery = strings.Replace(req.URL.RawQuery, "dryRun", "dryrun", 1)
}
// kube-apiserver lost query string when proxy websocket requests, there are several issues opened
@@ -134,15 +139,72 @@ func (m *multiclusterDispatcher) ServeHTTP(w http.ResponseWriter, req *http.Requ
// PR aim to fix this, but it's unlikely it will get merged soon. So here we are again. Put raw query
// string in Header and extract it on member cluster.
if httpstream.IsUpgradeRequest(req) && len(req.URL.RawQuery) != 0 {
req.Header.Set("X-KubeSphere-Rawquery", req.URL.RawQuery)
newReq.Header.Set("X-KubeSphere-Rawquery", req.URL.RawQuery)
}
} else {
// everything else goes to ks-apiserver, since our ks-apiserver has the ability to proxy kube-apiserver requests
u.Host = innCluster.KubesphereURL.Host
u.Scheme = innCluster.KubesphereURL.Scheme
location.Scheme = clusterClient.KubeSphereURL.Scheme
location.Host = clusterClient.KubeSphereURL.Host
transport = http.DefaultTransport
}
httpProxy := proxy.NewUpgradeAwareHandler(&u, transport, false, false, &responder{})
statusCodeChangeTransport := &statusCodeChangeTransport{transport}
upgrade := httpstream.IsUpgradeRequest(req)
httpProxy := proxy.NewUpgradeAwareHandler(location, statusCodeChangeTransport, true, upgrade, &responder{})
httpProxy.UpgradeTransport = proxy.NewUpgradeRequestRoundTripper(transport, transport)
httpProxy.ServeHTTP(w, req)
httpProxy.ServeHTTP(w, newReq)
}
func (m *multiclusterDispatcher) resolveCluster(name string) (*clusterv1alpha1.Cluster, error) {
cluster, err := m.Get(name)
if err != nil {
if errors.IsNotFound(err) {
// Ensure compatibility with hardcoded host cluster name
if name == "host" && m.options.HostClusterName != "" {
return m.Get(m.options.HostClusterName)
}
}
return nil, err
}
return cluster, nil
}
type statusCodeChangeTransport struct {
http.RoundTripper
}
func (rt *statusCodeChangeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
resp, err := rt.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
if resp.StatusCode == http.StatusUnauthorized {
reason, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
klog.Warningf("Request unauthorized, host: %s, reason: %s", req.URL.Host, string(reason))
data, _ := json.Marshal(metav1.Status{
TypeMeta: metav1.TypeMeta{
Kind: "Status",
APIVersion: "v1",
},
Status: metav1.StatusFailure,
Message: "The request could not be authenticated due to a system issue.",
Reason: metav1.StatusReason(http.StatusText(http.StatusNetworkAuthenticationRequired)),
Code: http.StatusNetworkAuthenticationRequired,
})
// replace the response
*resp = http.Response{
StatusCode: http.StatusNetworkAuthenticationRequired,
Status: fmt.Sprintf("%d %s", http.StatusNetworkAuthenticationRequired, http.StatusText(http.StatusNetworkAuthenticationRequired)),
Body: io.NopCloser(bytes.NewReader(data)),
ContentLength: int64(len(data)),
Header: map[string][]string{"Content-Type": {"application/json"}},
}
}
return resp, nil
}

19
pkg/apiserver/filters/requestinfo.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
@@ -65,7 +54,7 @@ func WithRequestInfo(next http.Handler, resolver request.RequestInfoResolver) ht
return
}
req = req.WithContext(request.WithRequestInfo(ctx, info))
*req = *req.WithContext(request.WithRequestInfo(ctx, info))
next.ServeHTTP(w, req)
})
}

22
pkg/apiserver/filters/responder.go
View File

@@ -1,14 +1,32 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"fmt"
"net/http"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/klog/v2"
)
type responder struct{}
func (r *responder) Error(w http.ResponseWriter, req *http.Request, err error) {
klog.Errorf("Error while proxying request: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
reason := fmt.Sprintf("Error while proxying request: %v", err)
klog.Errorln(reason)
statusError := errors.StatusError{
ErrStatus: metav1.Status{
Code: http.StatusBadGateway,
Message: reason,
Reason: metav1.StatusReason(http.StatusText(http.StatusBadGateway)),
},
}
responsewriters.WriteRawJSON(http.StatusBadGateway, statusError, w)
}

289
pkg/apiserver/filters/reverseproxy.go Normal file
View File

@@ -0,0 +1,289 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package filters
import (
"encoding/base64"
"fmt"
"net"
"net/http"
"net/url"
"strings"
"sync"
"time"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/proxy"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/client-go/transport"
"k8s.io/klog/v2"
extensionsv1alpha1 "kubesphere.io/api/extensions/v1alpha1"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/utils/directives"
)
type reverseProxy struct {
next http.Handler
cache cache.Cache
proxyRoundTrippers *sync.Map
}
func WithReverseProxy(next http.Handler, cache cache.Cache) http.Handler {
return &reverseProxy{next: next, cache: cache, proxyRoundTrippers: &sync.Map{}}
}
func (s *reverseProxy) ServeHTTP(w http.ResponseWriter, req *http.Request) {
requestInfo, _ := request.RequestInfoFrom(req.Context())
if requestInfo.IsKubernetesRequest {
s.next.ServeHTTP(w, req)
return
}
if requestInfo.IsResourceRequest {
s.next.ServeHTTP(w, req)
return
}
if !strings.HasPrefix(requestInfo.Path, extensionsv1alpha1.ProxyPrefix) {
s.next.ServeHTTP(w, req)
return
}
var reverseProxies extensionsv1alpha1.ReverseProxyList
// If the target label is not set, it is also handled by ks-apiserver (backward compatibility)
selector, _ := labels.Parse(fmt.Sprintf("%s!=%s", extensionsv1alpha1.ReverseProxyTargetLabel, extensionsv1alpha1.ReverseProxyTargetConsole))
if err := s.cache.List(req.Context(), &reverseProxies, client.MatchingLabelsSelector{Selector: selector}); err != nil {
reason := "failed to list reverse proxies"
klog.Errorf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
for _, reverseProxy := range reverseProxies.Items {
if !s.match(reverseProxy.Spec.Matcher, req) {
continue
}
if reverseProxy.Status.State != extensionsv1alpha1.StateAvailable {
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, fmt.Errorf("upstream %s is not available", reverseProxy.Name), w)
return
}
s.handleProxyRequest(reverseProxy, w, req)
return
}
s.next.ServeHTTP(w, req)
}
func (s *reverseProxy) match(matcher extensionsv1alpha1.Matcher, req *http.Request) bool {
if matcher.Method != req.Method && matcher.Method != "*" {
return false
}
if matcher.Path == req.URL.Path {
return true
}
if strings.HasSuffix(matcher.Path, "*") &&
strings.HasPrefix(req.URL.Path, strings.TrimRight(matcher.Path, "*")) {
return true
}
return false
}
func (s *reverseProxy) handleProxyRequest(reverseProxy extensionsv1alpha1.ReverseProxy, w http.ResponseWriter, req *http.Request) {
endpoint, err := url.Parse(reverseProxy.Spec.Upstream.RawURL())
if err != nil {
reason := fmt.Sprintf("endpoint %s is not available", endpoint)
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
location := &url.URL{}
location.Scheme = endpoint.Scheme
location.Host = endpoint.Host
location.Path = req.URL.Path
location.RawQuery = req.URL.Query().Encode()
newReq := req.WithContext(req.Context())
newReq.Header = utilnet.CloneHeader(req.Header)
newReq.URL = location
newReq.Host = location.Host
if reverseProxy.Spec.Directives.Method != "" {
newReq.Method = reverseProxy.Spec.Directives.Method
}
if reverseProxy.Spec.Directives.StripPathPrefix != "" {
location.Path = strings.TrimPrefix(location.Path, reverseProxy.Spec.Directives.StripPathPrefix)
}
if reverseProxy.Spec.Directives.StripPathSuffix != "" {
location.Path = strings.TrimSuffix(location.Path, reverseProxy.Spec.Directives.StripPathSuffix)
}
if len(reverseProxy.Spec.Directives.HeaderUp) > 0 {
for _, header := range reverseProxy.Spec.Directives.HeaderUp {
if strings.HasPrefix(header, "-") {
removeHeader(newReq.Header, strings.TrimPrefix(header, "-"))
} else if strings.HasPrefix(header, "+") {
addOrReplaceHeader(newReq.Header, strings.TrimPrefix(header, "+"), false)
} else {
addOrReplaceHeader(newReq.Header, header, true)
}
}
}
if err = directives.HandlerRequest(newReq, reverseProxy.Spec.Directives.Rewrite, directives.WithRewriteFilter); err != nil {
reason := "failed to create handler directives Directives.Rewrite"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
if err = directives.HandlerRequest(newReq, reverseProxy.Spec.Directives.Replace, directives.WithReplaceFilter); err != nil {
reason := "failed to create handler directives Directives.Replace"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
if err = directives.HandlerRequest(newReq, reverseProxy.Spec.Directives.PathRegexp, directives.WithPathRegexpFilter); err != nil {
reason := "failed to create handler directives Directives.PathRegexp"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
var proxyRoundTripper http.RoundTripper
if newProxyRoundTripper, ok := s.proxyRoundTrippers.Load(reverseProxy.Name); !ok {
tlsConfig := transport.TLSConfig{
Insecure: reverseProxy.Spec.Upstream.InsecureSkipVerify,
}
if !reverseProxy.Spec.Upstream.InsecureSkipVerify && len(reverseProxy.Spec.Upstream.CABundle) > 0 {
caData, err := base64.StdEncoding.DecodeString(string(reverseProxy.Spec.Upstream.CABundle))
if err != nil {
reason := fmt.Sprintf("failed to decode CA bundle from upstream %s", reverseProxy.Name)
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
tlsConfig.CAData = caData
}
newProxyRoundTripper, err := transport.New(&transport.Config{
TLS: tlsConfig,
WrapTransport: func(rt http.RoundTripper) http.RoundTripper {
return &http.Transport{
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 0,
MaxConnsPerHost: 0,
MaxIdleConnsPerHost: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: rt.(*http.Transport).TLSClientConfig,
}
},
})
if err != nil {
reason := "failed to create transport.TLSConfig"
klog.Warningf("%v: %v\n", reason, err)
responsewriters.WriteRawJSON(http.StatusServiceUnavailable, errors.NewServiceUnavailable(reason), w)
return
}
proxyRoundTripper = newProxyRoundTripper
s.proxyRoundTrippers.Store(reverseProxy.Name, newProxyRoundTripper)
} else {
proxyRoundTripper = newProxyRoundTripper.(http.RoundTripper)
}
if reverseProxy.Spec.Directives.AuthProxy {
user, _ := request.UserFrom(req.Context())
proxyRoundTripper = transport.NewAuthProxyRoundTripper(user.GetName(), user.GetGroups(), user.GetExtra(), proxyRoundTripper)
}
upgrade := httpstream.IsUpgradeRequest(req)
handler := proxy.NewUpgradeAwareHandler(location, proxyRoundTripper, false, upgrade, &responder{})
if reverseProxy.Spec.Directives.WrapTransport {
handler.WrapTransport = true
}
if len(reverseProxy.Spec.Directives.HeaderDown) > 0 {
w = &responseWriterWrapper{
ResponseWriter: w,
HeaderDown: reverseProxy.Spec.Directives.HeaderDown,
}
}
handler.ServeHTTP(w, newReq)
}
func removeHeader(header http.Header, key string) {
if strings.HasSuffix(key, "*") {
prefix := strings.TrimSuffix(key, "*")
for key := range header {
if strings.HasSuffix(key, prefix) {
header.Del(key)
}
}
} else {
header.Del(key)
}
}
func addOrReplaceHeader(header http.Header, keyValues string, replace bool) {
values := strings.SplitN(keyValues, " ", 2)
if len(values) != 2 {
return
}
key := values[0]
value := values[1]
if strings.HasPrefix(value, "\"") && strings.HasSuffix(value, "\"") {
value = strings.TrimSuffix(strings.TrimPrefix(value, "\""), "\"")
}
if replace {
header.Set(key, value)
} else {
header.Add(key, value)
}
}
type responseWriterWrapper struct {
http.ResponseWriter
wroteHeader bool
HeaderDown []string
}
func (rww *responseWriterWrapper) WriteHeader(status int) {
if rww.wroteHeader {
return
}
rww.wroteHeader = true
for _, header := range rww.HeaderDown {
if strings.HasPrefix(header, "-") {
removeHeader(rww.Header(), strings.TrimPrefix(header, "-"))
} else if strings.HasPrefix(header, "+") {
addOrReplaceHeader(rww.Header(), strings.TrimPrefix(header, "+"), false)
} else {
addOrReplaceHeader(rww.Header(), header, true)
}
}
rww.ResponseWriter.WriteHeader(status)
}
func (rww *responseWriterWrapper) Write(d []byte) (int, error) {
if !rww.wroteHeader {
rww.WriteHeader(http.StatusOK)
}
return rww.ResponseWriter.Write(d)
}

56
pkg/apiserver/metric.go
View File

@@ -1,56 +0,0 @@
// Copyright 2022 The KubeSphere Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package apiserver
import (
compbasemetrics "k8s.io/component-base/metrics"
"kubesphere.io/kubesphere/pkg/utils/metrics"
)
var (
RequestCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "ks_server_request_total",
Help: "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "code"},
)
RequestLatencies = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "ks_server_request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, group, version, resource",
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource"},
)
metricsList = []compbasemetrics.Registerable{
RequestCounter,
RequestLatencies,
}
)
func registerMetrics() {
for _, m := range metricsList {
metrics.MustRegister(m)
}
}

89
pkg/apiserver/metrics/metrics.go Normal file
View File

@@ -0,0 +1,89 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package metrics
import (
"sync"
"github.com/emicklei/go-restful/v3"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"github.com/prometheus/client_golang/prometheus/promhttp"
apimachineryversion "k8s.io/apimachinery/pkg/version"
componentbasemetrics "k8s.io/component-base/metrics"
ksVersion "kubesphere.io/kubesphere/pkg/version"
)
var (
registerOnce sync.Once
Registry = componentbasemetrics.NewKubeRegistry()
RequestCounter = componentbasemetrics.NewCounterVec(
&componentbasemetrics.CounterOpts{
Name: "ks_server_request_total",
Help: "Counter of ks_server requests broken out for each verb, group, version, resource and HTTP response code.",
StabilityLevel: componentbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "code"},
)
RequestLatencies = componentbasemetrics.NewHistogramVec(
&componentbasemetrics.HistogramOpts{
Name: "ks_server_request_duration_seconds",
Help: "Response latency distribution in seconds for each verb, group, version, resource",
// This metric is used for verifying api call latencies SLO,
// as well as tracking regressions in this aspects.
// Thus we customize buckets significantly, to empower both usecases.
Buckets: []float64{0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
StabilityLevel: componentbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource"},
)
metricsList = []componentbasemetrics.Registerable{
RequestCounter,
RequestLatencies,
}
)
func init() {
componentbasemetrics.BuildVersion = versionGet
}
func versionGet() apimachineryversion.Info {
info := ksVersion.Get()
return apimachineryversion.Info{
Major: info.GitMajor,
Minor: info.GitMinor,
GitVersion: info.GitVersion,
GitCommit: info.GitCommit,
GitTreeState: info.GitTreeState,
BuildDate: info.BuildDate,
GoVersion: info.GoVersion,
Compiler: info.Compiler,
Platform: info.Platform,
}
}
func registerMetrics() {
Registry.Registerer().MustRegister(collectors.NewGoCollector())
Registry.Registerer().MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
for _, m := range metricsList {
Registry.MustRegister(m)
}
}
func Install(c *restful.Container) {
registerOnce.Do(registerMetrics)
c.Handle(
"/metrics",
promhttp.InstrumentMetricHandler(prometheus.NewRegistry(), promhttp.HandlerFor(Registry, promhttp.HandlerOpts{})),
)
}

32
pkg/apiserver/options/options.go Normal file
View File

@@ -0,0 +1,32 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package options
import (
"kubesphere.io/utils/s3"
"kubesphere.io/kubesphere/pkg/config"
"kubesphere.io/kubesphere/pkg/apiserver/auditing"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authorization"
"kubesphere.io/kubesphere/pkg/models/terminal"
"kubesphere.io/kubesphere/pkg/multicluster"
"kubesphere.io/kubesphere/pkg/simple/client/cache"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
)
type Options struct {
MultiClusterOptions *multicluster.Options `json:"multicluster"`
AuthenticationOptions *authentication.Options `json:"-"`
KubernetesOptions *k8s.Options `json:"-"`
CacheOptions *cache.Options `json:"-"`
AuthorizationOptions *authorization.Options `json:"-"`
AuditingOptions *auditing.Options `json:"-"`
TerminalOptions *terminal.Options `json:"-"`
S3Options *s3.Options `json:"-"`
ExperimentalOptions *config.ExperimentalOptions `json:"-"`
}

40
pkg/apiserver/query/field.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package query
@@ -21,7 +10,6 @@ type Value string
const (
FieldName = "name"
FieldAlias = "alias"
FieldNames = "names"
FieldUID = "uid"
FieldCreationTimeStamp = "creationTimestamp"
@@ -34,26 +22,4 @@ const (
FieldStatus = "status"
FieldOwnerReference = "ownerReference"
FieldOwnerKind = "ownerKind"
FieldType = "type"
)
var SortableFields = []Field{
FieldCreationTimeStamp,
FieldCreateTime,
FieldUpdateTime,
FieldLastUpdateTimestamp,
FieldName,
}
// Field contains all the query field that can be compared
var ComparableFields = []Field{
FieldName,
FieldUID,
FieldLabel,
FieldAnnotation,
FieldNamespace,
FieldStatus,
FieldOwnerReference,
FieldOwnerKind,
}

38
pkg/apiserver/query/types.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package query
@@ -77,8 +66,16 @@ func (q *Query) Selector() labels.Selector {
}
}
func (p *Pagination) GetValidPagination(total int) (startIndex, endIndex int) {
func (q *Query) AppendLabelSelector(ls map[string]string) error {
labelsMap, err := labels.ConvertSelectorToLabelsMap(q.LabelSelector)
if err != nil {
return err
}
q.LabelSelector = labels.Merge(labelsMap, ls).String()
return nil
}
func (p *Pagination) GetValidPagination(total int) (startIndex, endIndex int) {
// no pagination
if p.Limit == NoPagination.Limit {
return 0, total
@@ -109,8 +106,8 @@ func New() *Query {
}
type Filter struct {
Field Field
Value Value
Field Field `json:"field"`
Value Value `json:"value"`
}
func ParseQueryParameter(request *restful.Request) *Query {
@@ -142,10 +139,11 @@ func ParseQueryParameter(request *restful.Request) *Query {
for key, values := range request.Request.URL.Query() {
if !sliceutil.HasString([]string{ParameterPage, ParameterLimit, ParameterOrderBy, ParameterAscending, ParameterLabelSelector}, key) {
// support multiple query condition
for _, value := range values {
query.Filters[Field(key)] = Value(value)
value := ""
if len(values) > 0 {
value = values[0]
}
query.Filters[Field(key)] = Value(value)
}
}

17
pkg/apiserver/query/types_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package query

48
pkg/apiserver/request/context.go
View File

@@ -19,8 +19,6 @@ package request
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/apis/audit"
"k8s.io/apiserver/pkg/authentication/user"
)
@@ -30,48 +28,15 @@ import (
type key int
const (
// namespaceKey is the context key for the request namespace.
namespaceKey key = iota
// userKey is the context key for the request user.
userKey
// auditKey is the context key for the audit event.
auditKey
userKey key = iota
)
// NewContext instantiates a base context object for request flows.
func NewContext() context.Context {
return context.TODO()
}
// NewDefaultContext instantiates a base context object for request flows in the default namespace
func NewDefaultContext() context.Context {
return WithNamespace(NewContext(), metav1.NamespaceDefault)
}
// WithValue returns a copy of parent in which the value associated with key is val.
func WithValue(parent context.Context, key interface{}, val interface{}) context.Context {
return context.WithValue(parent, key, val)
}
// WithNamespace returns a copy of parent in which the namespace value is set
func WithNamespace(parent context.Context, namespace string) context.Context {
return WithValue(parent, namespaceKey, namespace)
}
// NamespaceFrom returns the value of the namespace key on the ctx
func NamespaceFrom(ctx context.Context) (string, bool) {
namespace, ok := ctx.Value(namespaceKey).(string)
return namespace, ok
}
// NamespaceValue returns the value of the namespace key on the ctx, or the empty string if none
func NamespaceValue(ctx context.Context) string {
namespace, _ := NamespaceFrom(ctx)
return namespace
}
// WithUser returns a copy of parent in which the user value is set
func WithUser(parent context.Context, user user.Info) context.Context {
return WithValue(parent, userKey, user)
@@ -82,14 +47,3 @@ func UserFrom(ctx context.Context) (user.Info, bool) {
user, ok := ctx.Value(userKey).(user.Info)
return user, ok
}
// WithAuditEvent returns set audit event struct.
func WithAuditEvent(parent context.Context, ev *audit.Event) context.Context {
return WithValue(parent, auditKey, ev)
}
// AuditEventFrom returns the audit event struct on the ctx
func AuditEventFrom(ctx context.Context) *audit.Event {
ev, _ := ctx.Value(auditKey).(*audit.Event)
return ev
}

23
pkg/apiserver/request/context_test.go
View File

@@ -17,35 +17,17 @@ limitations under the License.
package request
import (
"context"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/user"
)
// Following code copied from k8s.io/apiserver/pkg/endpoints/request to avoid import collision
// TestNamespaceContext validates that a namespace can be get/set on a context object
func TestNamespaceContext(t *testing.T) {
ctx := NewDefaultContext()
result, ok := NamespaceFrom(ctx)
if !ok {
t.Fatalf("Error getting namespace")
}
if metav1.NamespaceDefault != result {
t.Fatalf("Expected: %s, Actual: %s", metav1.NamespaceDefault, result)
}
ctx = NewContext()
_, ok = NamespaceFrom(ctx)
if ok {
t.Fatalf("Should not be ok because there is no namespace on the context")
}
}
// TestUserContext validates that a userinfo can be get/set on a context object
func TestUserContext(t *testing.T) {
ctx := NewContext()
ctx := context.TODO()
_, ok := UserFrom(ctx)
if ok {
t.Fatalf("Should not be ok because there is no user.Info on the context")
@@ -91,5 +73,4 @@ func TestUserContext(t *testing.T) {
} else if actualExtra[expectedExtraKey][0] != expectedExtraValue {
t.Fatalf("Get user extra map value error, Expected: %s, Actual: %s", expectedExtraValue, actualExtra[expectedExtraKey])
}
}

86
pkg/apiserver/request/requestinfo.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
// NOTE: This file is copied from k8s.io/apiserver/pkg/endpoints/request.
// We expanded requestInfo.
@@ -81,9 +70,6 @@ type RequestInfo struct {
// Cluster of requested resource, this is empty in single-cluster environment
Cluster string
// DevOps project of requested resource
DevOps string
// Scope of requested resource.
ResourceScope string
@@ -125,8 +111,8 @@ type RequestInfoFactory struct {
// /kapis/{api-group}/{version}/namespaces/{namespace}/{resource}
// /kapis/{api-group}/{version}/namespaces/{namespace}/{resource}/{resourceName}
// With workspaces:
// /kapis/clusters/{cluster}/{api-group}/{version}/namespaces/{namespace}/{resource}
// /kapis/clusters/{cluster}/{api-group}/{version}/namespaces/{namespace}/{resource}/{resourceName}
// /clusters/{cluster}/kapis/{api-group}/{version}/namespaces/{namespace}/{resource}
// /clusters/{cluster}/kapis/{api-group}/{version}/namespaces/{namespace}/{resource}/{resourceName}
func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, error) {
requestInfo := RequestInfo{
IsKubernetesRequest: false,
@@ -144,7 +130,7 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er
prefix := requestInfo.APIPrefix
if prefix == "" {
currentParts := splitPath(requestInfo.Path)
//Proxy discovery API
// Proxy discovery API
if len(currentParts) > 0 && len(currentParts) < 3 {
prefix = currentParts[0]
}
@@ -159,6 +145,18 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er
return &requestInfo, nil
}
// URL forms: /clusters/{cluster}/*
if currentParts[0] == "clusters" {
if len(currentParts) > 1 {
requestInfo.Cluster = currentParts[1]
// resolve the real path behind the cluster dispatcher
requestInfo.Path = strings.TrimPrefix(requestInfo.Path, fmt.Sprintf("/clusters/%s", requestInfo.Cluster))
}
if len(currentParts) > 2 {
currentParts = currentParts[2:]
}
}
if !r.APIPrefixes.Has(currentParts[0]) {
// return a non-resource request
return &requestInfo, nil
@@ -166,13 +164,19 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er
requestInfo.APIPrefix = currentParts[0]
currentParts = currentParts[1:]
// URL forms: /clusters/{cluster}/*
if currentParts[0] == "clusters" {
if len(currentParts) > 1 {
requestInfo.Cluster = currentParts[1]
}
if len(currentParts) > 2 {
currentParts = currentParts[2:]
// fallback to legacy cluster API
// TODO remove the following codes
if requestInfo.Cluster == "" {
// URL forms: /(kapis|apis|api)/clusters/{cluster}/*
if currentParts[0] == "clusters" {
if len(currentParts) > 1 {
requestInfo.Cluster = currentParts[1]
// resolve the real path behind the cluster dispatcher
requestInfo.Path = strings.Replace(requestInfo.Path, fmt.Sprintf("/clusters/%s", requestInfo.Cluster), "", 1)
}
if len(currentParts) > 2 {
currentParts = currentParts[2:]
}
}
}
@@ -216,7 +220,7 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er
}
// URL forms: /workspaces/{workspace}/*
if currentParts[0] == "workspaces" {
if currentParts[0] == "workspaces" || currentParts[0] == "workspacetemplates" {
if len(currentParts) > 1 {
requestInfo.Workspace = currentParts[1]
}
@@ -230,25 +234,12 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er
if len(currentParts) > 1 {
requestInfo.Namespace = currentParts[1]
// if there is another step after the namespace name and it is not a known namespace subresource
// if there is another step after the namespace name, and it is not a known namespace subresource
// move currentParts to include it as a resource in its own right
if len(currentParts) > 2 && !namespaceSubresources.Has(currentParts[2]) {
currentParts = currentParts[2:]
}
}
} else if currentParts[0] == "devops" {
if len(currentParts) > 1 {
requestInfo.DevOps = currentParts[1]
// if there is another step after the devops name
// move currentParts to include it as a resource in its own right
if len(currentParts) > 2 {
currentParts = currentParts[2:]
}
}
} else {
requestInfo.Namespace = metav1.NamespaceNone
requestInfo.DevOps = metav1.NamespaceNone
}
// parsing successful, so we now know the proper value for .Parts
@@ -327,7 +318,7 @@ type requestInfoKeyType int
const requestInfoKey requestInfoKeyType = iota
func WithRequestInfo(parent context.Context, info *RequestInfo) context.Context {
return k8srequest.WithValue(parent, requestInfoKey, info)
return context.WithValue(parent, requestInfoKey, info)
}
func RequestInfoFrom(ctx context.Context) (*RequestInfo, bool) {
@@ -349,12 +340,15 @@ const (
ClusterScope = "Cluster"
WorkspaceScope = "Workspace"
NamespaceScope = "Namespace"
DevOpsScope = "DevOps"
workspaceSelectorPrefix = constants.WorkspaceLabelKey + "="
)
func (r *RequestInfoFactory) resolveResourceScope(request RequestInfo) string {
if r.isGlobalScopeResource(request.APIGroup, request.Resource) {
// GET /apis/tenant.kubesphere.io/v1beta1/workspaces/{workspace}
if request.Workspace != "" {
return WorkspaceScope
}
return GlobalScope
}
@@ -362,10 +356,6 @@ func (r *RequestInfoFactory) resolveResourceScope(request RequestInfo) string {
return NamespaceScope
}
if request.DevOps != "" {
return DevOpsScope
}
if request.Workspace != "" {
return WorkspaceScope
}

25
pkg/apiserver/request/requestinfo_test.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package request
@@ -59,7 +48,7 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) {
},
{
name: "list clusterRoles of cluster gondor",
url: "/apis/clusters/gondor/rbac.authorization.k8s.io/v1/clusterroles",
url: "/clusters/gondor/apis/rbac.authorization.k8s.io/v1/clusterroles",
method: http.MethodGet,
expectedErr: nil,
expectedVerb: "list",
@@ -81,7 +70,7 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) {
},
{
name: "list nodes of cluster gondor",
url: "/api/clusters/gondor/v1/nodes",
url: "/clusters/gondor/api/v1/nodes",
method: http.MethodGet,
expectedErr: nil,
expectedVerb: "list",
@@ -92,7 +81,7 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) {
},
{
name: "list roles of cluster gondor",
url: "/apis/clusters/gondor/rbac.authorization.k8s.io/v1/namespaces/namespace1/roles",
url: "/clusters/gondor/apis/rbac.authorization.k8s.io/v1/namespaces/namespace1/roles",
method: http.MethodGet,
expectedErr: nil,
expectedVerb: "list",
@@ -128,7 +117,7 @@ func TestRequestInfoFactory_NewRequestInfo(t *testing.T) {
},
{
name: "list namespaces of cluster gondor",
url: "/kapis/clusters/gondor/resources.kubesphere.io/v1alpha3/workspaces/workspace1/namespaces",
url: "/clusters/gondor/kapis/resources.kubesphere.io/v1alpha3/workspaces/workspace1/namespaces",
method: http.MethodGet,
expectedErr: nil,
expectedVerb: "list",

12
pkg/apiserver/rest/interface.go Normal file
View File

@@ -0,0 +1,12 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package rest
import "github.com/emicklei/go-restful/v3"
type Handler interface {
AddToContainer(c *restful.Container) error
}

25
pkg/apiserver/runtime/runtime.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package runtime
@@ -27,7 +16,7 @@ const (
ApiRootPath = "/kapis"
)
// container holds all webservice of apiserver
// Container holds all webservice of apiserver
var Container = restful.NewContainer()
type ContainerBuilder []func(c *restful.Container) error
@@ -64,9 +53,3 @@ func (cb *ContainerBuilder) Register(funcs ...func(*restful.Container) error) {
*cb = append(*cb, f)
}
}
func NewContainerBuilder(funcs ...func(*restful.Container) error) ContainerBuilder {
var cb ContainerBuilder
cb.Register(funcs...)
return cb
}

17
pkg/apiserver/server/handler.go
View File

@@ -1,17 +0,0 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package server