admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: kubesphere 4.0 (#6115)

Browse Source 
* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>
This commit is contained in:
KubeSphere CI Bot
2024-09-06 11:05:52 +08:00
committed by GitHub
parent b5015ec7b9
commit 447a51f08b
8557 changed files with 546695 additions and 1146174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
pkg/models/auth/authenticator.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
@@ -22,21 +9,13 @@ import (
"context"
"fmt"
"net/http"
"net/mail"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
authuser "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
iamv1alpha2listers "kubesphere.io/kubesphere/pkg/client/listers/iam/v1alpha2"
)
var (
@@ -52,7 +31,7 @@ var (
// "k8s.io/apimachinery/pkg/api", and restful.ServerError defined at package
// "github.com/emicklei/go-restful/v3", or the server cannot handle error correctly.
type PasswordAuthenticator interface {
Authenticate(ctx context.Context, provider, username, password string) (authuser.Info, string, error)
Authenticate(ctx context.Context, provider, username, password string) (authuser.Info, error)
}
// OAuthAuthenticator authenticate users by OAuth 2.0 Authorization Framework. Note that implement this
@@ -60,76 +39,32 @@ type PasswordAuthenticator interface {
// "k8s.io/apimachinery/pkg/api", and restful.ServerError defined at package
// "github.com/emicklei/go-restful/v3", or the server cannot handle error correctly.
type OAuthAuthenticator interface {
Authenticate(ctx context.Context, provider string, req *http.Request) (authuser.Info, string, error)
Authenticate(ctx context.Context, provider string, req *http.Request) (authuser.Info, error)
}
type userGetter struct {
userLister iamv1alpha2listers.UserLister
}
func preRegistrationUser(idp string, identity identityprovider.Identity) authuser.Info {
func newRreRegistrationUser(idp string, identity identityprovider.Identity) authuser.Info {
return &authuser.DefaultInfo{
Name: iamv1alpha2.PreRegistrationUser,
Name: iamv1beta1.PreRegistrationUser,
Extra: map[string][]string{
iamv1alpha2.ExtraIdentityProvider: {idp},
iamv1alpha2.ExtraUID: {identity.GetUserID()},
iamv1alpha2.ExtraUsername: {identity.GetUsername()},
iamv1alpha2.ExtraEmail: {identity.GetEmail()},
iamv1beta1.ExtraIdentityProvider: {idp},
iamv1beta1.ExtraUID: {identity.GetUserID()},
iamv1beta1.ExtraUsername: {identity.GetUsername()},
iamv1beta1.ExtraEmail: {identity.GetEmail()},
},
}
}
func mappedUser(idp string, identity identityprovider.Identity) *iamv1alpha2.User {
func newMappedUser(idp string, identity identityprovider.Identity) *iamv1beta1.User {
// username convert
username := strings.ToLower(identity.GetUsername())
return &iamv1alpha2.User{
return &iamv1beta1.User{
ObjectMeta: metav1.ObjectMeta{
Name: username,
Labels: map[string]string{
iamv1alpha2.IdentifyProviderLabel: idp,
iamv1alpha2.OriginUIDLabel: identity.GetUserID(),
iamv1beta1.IdentifyProviderLabel: idp,
iamv1beta1.OriginUIDLabel: identity.GetUserID(),
},
},
Spec: iamv1alpha2.UserSpec{Email: identity.GetEmail()},
Spec: iamv1beta1.UserSpec{Email: identity.GetEmail()},
}
}
// findUser returns the user associated with the username or email
func (u *userGetter) findUser(username string) (*iamv1alpha2.User, error) {
if _, err := mail.ParseAddress(username); err != nil {
return u.userLister.Get(username)
}
users, err := u.userLister.List(labels.Everything())
if err != nil {
klog.Error(err)
return nil, err
}
for _, user := range users {
if user.Spec.Email == username {
return user, nil
}
}
return nil, errors.NewNotFound(iamv1alpha2.Resource("user"), username)
}
// findMappedUser returns the user which mapped to the identity
func (u *userGetter) findMappedUser(idp, uid string) (*iamv1alpha2.User, error) {
selector := labels.SelectorFromSet(labels.Set{
iamv1alpha2.IdentifyProviderLabel: idp,
iamv1alpha2.OriginUIDLabel: uid,
})
users, err := u.userLister.List(selector)
if err != nil {
klog.Error(err)
return nil, err
}
if len(users) != 1 {
return nil, errors.NewNotFound(iamv1alpha2.Resource("user"), uid)
}
return users[0], err
}

55
pkg/models/auth/login_recoder.go
View File

@@ -1,18 +1,7 @@
/*
Copyright 2020 KubeSphere Authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
@@ -23,33 +12,30 @@ import (
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
iamv1alpha2listers "kubesphere.io/kubesphere/pkg/client/listers/iam/v1alpha2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
type LoginRecorder interface {
RecordLogin(username string, loginType iamv1alpha2.LoginType, provider string, sourceIP string, userAgent string, authErr error) error
RecordLogin(ctx context.Context, username string, loginType iamv1beta1.LoginType, provider string, sourceIP string, userAgent string, authErr error) error
}
type loginRecorder struct {
ksClient kubesphere.Interface
userGetter *userGetter
client runtimeclient.Client
userMapper userMapper
}
func NewLoginRecorder(ksClient kubesphere.Interface, userLister iamv1alpha2listers.UserLister) LoginRecorder {
func NewLoginRecorder(cacheClient runtimeclient.Client) LoginRecorder {
return &loginRecorder{
ksClient: ksClient,
userGetter: &userGetter{userLister: userLister},
client: cacheClient,
userMapper: userMapper{cache: cacheClient},
}
}
// RecordLogin Create v1alpha2.LoginRecord for existing accounts
func (l *loginRecorder) RecordLogin(username string, loginType iamv1alpha2.LoginType, provider, sourceIP, userAgent string, authErr error) error {
func (l *loginRecorder) RecordLogin(ctx context.Context, username string, loginType iamv1beta1.LoginType, provider, sourceIP, userAgent string, authErr error) error {
// only for existing accounts, solve the problem of huge entries
user, err := l.userGetter.findUser(username)
user, err := l.userMapper.Find(ctx, username)
if err != nil {
// ignore not found error
if errors.IsNotFound(err) {
@@ -58,30 +44,29 @@ func (l *loginRecorder) RecordLogin(username string, loginType iamv1alpha2.Login
klog.Error(err)
return err
}
loginEntry := &iamv1alpha2.LoginRecord{
record := &iamv1beta1.LoginRecord{
ObjectMeta: metav1.ObjectMeta{
GenerateName: fmt.Sprintf("%s-", user.Name),
Labels: map[string]string{
iamv1alpha2.UserReferenceLabel: user.Name,
iamv1beta1.UserReferenceLabel: user.Name,
},
},
Spec: iamv1alpha2.LoginRecordSpec{
Spec: iamv1beta1.LoginRecordSpec{
Type: loginType,
Provider: provider,
Success: true,
Reason: iamv1alpha2.AuthenticatedSuccessfully,
Reason: iamv1beta1.AuthenticatedSuccessfully,
SourceIP: sourceIP,
UserAgent: userAgent,
},
}
if authErr != nil {
loginEntry.Spec.Success = false
loginEntry.Spec.Reason = authErr.Error()
record.Spec.Success = false
record.Spec.Reason = authErr.Error()
}
_, err = l.ksClient.IamV1alpha2().LoginRecords().Create(context.Background(), loginEntry, metav1.CreateOptions{})
if err != nil {
if err = l.client.Create(context.Background(), record); err != nil {
klog.Error(err)
return err
}

57
pkg/models/auth/mapper.go Normal file
View File

@@ -0,0 +1,57 @@
/*
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
import (
"context"
"net/mail"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
)
type userMapper struct {
cache runtimeclient.Reader
}
// Find returns the user associated with the username or email
func (u *userMapper) Find(ctx context.Context, username string) (*iamv1beta1.User, error) {
user := &iamv1beta1.User{}
if _, err := mail.ParseAddress(username); err != nil {
return user, u.cache.Get(ctx, types.NamespacedName{Name: username}, user)
}
// TODO cache with index
userList := &iamv1beta1.UserList{}
if err := u.cache.List(ctx, userList); err != nil {
return nil, err
}
for _, user := range userList.Items {
if user.Spec.Email == username {
return &user, nil
}
}
return nil, errors.NewNotFound(iamv1beta1.Resource("user"), username)
}
// FindMappedUser returns the user which mapped to the identity
func (u *userMapper) FindMappedUser(ctx context.Context, idp, uid string) (*iamv1beta1.User, error) {
userList := &iamv1beta1.UserList{}
if err := u.cache.List(ctx, userList, runtimeclient.MatchingLabels{
iamv1beta1.IdentifyProviderLabel: idp,
iamv1beta1.OriginUIDLabel: uid,
}); err != nil {
return nil, err
}
if len(userList.Items) != 1 {
return nil, nil
}
return &userList.Items[0], nil
}

114
pkg/models/auth/oauth.go
View File

@@ -1,102 +1,84 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
import (
"context"
"fmt"
"net/http"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"k8s.io/apimachinery/pkg/api/errors"
authuser "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
iamv1alpha2listers "kubesphere.io/kubesphere/pkg/client/listers/iam/v1alpha2"
)
type oauthAuthenticator struct {
ksClient kubesphere.Interface
userGetter *userGetter
options *authentication.Options
client runtimeclient.Client
userGetter *userMapper
idpConfigurationGetter identityprovider.ConfigurationGetter
}
func NewOAuthAuthenticator(ksClient kubesphere.Interface,
userLister iamv1alpha2listers.UserLister,
options *authentication.Options) OAuthAuthenticator {
func NewOAuthAuthenticator(cacheClient runtimeclient.Client) OAuthAuthenticator {
authenticator := &oauthAuthenticator{
ksClient: ksClient,
userGetter: &userGetter{userLister: userLister},
options: options,
client: cacheClient,
userGetter: &userMapper{cache: cacheClient},
idpConfigurationGetter: identityprovider.NewConfigurationGetter(cacheClient),
}
return authenticator
}
func (o *oauthAuthenticator) Authenticate(_ context.Context, provider string, req *http.Request) (authuser.Info, string, error) {
providerOptions, err := o.options.OAuthOptions.IdentityProviderOptions(provider)
func (o *oauthAuthenticator) Authenticate(ctx context.Context, provider string, req *http.Request) (authuser.Info, error) {
providerConfig, err := o.idpConfigurationGetter.GetConfiguration(ctx, provider)
// identity provider not registered
if err != nil {
klog.Error(err)
return nil, "", err
return nil, fmt.Errorf("failed to get identity provider configuration for %s, error: %v", provider, err)
}
oauthIdentityProvider, err := identityprovider.GetOAuthProvider(providerOptions.Name)
oauthIdentityProvider, exist := identityprovider.SharedIdentityProviderController.GetOAuthProvider(provider)
if !exist {
return nil, fmt.Errorf("identity provider %s not exist", provider)
}
identity, err := oauthIdentityProvider.IdentityExchangeCallback(req)
if err != nil {
klog.Error(err)
return nil, "", err
return nil, fmt.Errorf("failed to exchange identity for %s, error: %v", provider, err)
}
authenticated, err := oauthIdentityProvider.IdentityExchangeCallback(req)
mappedUser, err := o.userGetter.FindMappedUser(ctx, providerConfig.Name, identity.GetUserID())
if err != nil {
klog.Error(err)
return nil, "", err
return nil, fmt.Errorf("failed to find mapped user for %s, error: %v", provider, err)
}
user, err := o.userGetter.findMappedUser(providerOptions.Name, authenticated.GetUserID())
if user == nil && providerOptions.MappingMethod == oauth.MappingMethodLookup {
klog.Error(err)
return nil, "", err
}
// the user will automatically create and mapping when login successful.
if user == nil && providerOptions.MappingMethod == oauth.MappingMethodAuto {
if !providerOptions.DisableLoginConfirmation {
return preRegistrationUser(providerOptions.Name, authenticated), providerOptions.Name, nil
if mappedUser == nil {
if providerConfig.MappingMethod == identityprovider.MappingMethodLookup {
return nil, fmt.Errorf("failed to find mapped user: %s", identity.GetUserID())
}
user, err = o.ksClient.IamV1alpha2().Users().Create(context.Background(), mappedUser(providerOptions.Name, authenticated), metav1.CreateOptions{})
if err != nil {
return nil, providerOptions.Name, err
if providerConfig.MappingMethod == identityprovider.MappingMethodManual {
return newRreRegistrationUser(providerConfig.Name, identity), nil
}
if providerConfig.MappingMethod == identityprovider.MappingMethodAuto {
mappedUser = newMappedUser(providerConfig.Name, identity)
if err = o.client.Create(ctx, mappedUser); err != nil {
return nil, err
}
return &authuser.DefaultInfo{Name: mappedUser.GetName()}, nil
}
return nil, fmt.Errorf("invalid mapping method found %s", providerConfig.MappingMethod)
}
if user != nil {
if user.Status.State == iamv1alpha2.UserDisabled {
// state not active
return nil, "", AccountIsNotActiveError
}
return &authuser.DefaultInfo{Name: user.GetName()}, providerOptions.Name, nil
if mappedUser.Status.State == iamv1beta1.UserDisabled {
return nil, AccountIsNotActiveError
}
return nil, "", errors.NewNotFound(iamv1alpha2.Resource("user"), authenticated.GetUsername())
return &authuser.DefaultInfo{Name: mappedUser.GetName()}, nil
}

159
pkg/models/auth/oauth_test.go
View File

@@ -1,18 +1,8 @@
/*
Copyright 2021 The KubeSphere Authors.
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package auth
import (
@@ -22,66 +12,90 @@ import (
"reflect"
"testing"
"kubesphere.io/kubesphere/pkg/server/options"
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/constants"
"github.com/mitchellh/mapstructure"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/user"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
runtimefakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
fakeks "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
"kubesphere.io/kubesphere/pkg/scheme"
"kubesphere.io/kubesphere/pkg/server/options"
)
func Test_oauthAuthenticator_Authenticate(t *testing.T) {
oauthOptions := &authentication.Options{
OAuthOptions: &oauth.Options{
IdentityProviders: []oauth.IdentityProviderOptions{
{
Name: "fake",
MappingMethod: "auto",
Type: "FakeIdentityProvider",
Provider: options.DynamicOptions{
"identities": map[string]interface{}{
"code1": map[string]string{
"uid": "100001",
"email": "user1@kubesphere.io",
"username": "user1",
},
"code2": map[string]string{
"uid": "100002",
"email": "user2@kubesphere.io",
"username": "user2",
},
},
},
fakeIDP := &identityprovider.Configuration{
Name: "fake",
MappingMethod: "auto",
Type: "FakeOAuthProvider",
ProviderOptions: options.DynamicOptions{
"identities": map[string]interface{}{
"code1": map[string]string{
"uid": "100001",
"email": "user1@kubesphere.io",
"username": "user1",
},
"code2": map[string]string{
"uid": "100002",
"email": "user2@kubesphere.io",
"username": "user2",
},
},
},
}
identityprovider.RegisterOAuthProvider(&fakeProviderFactory{})
if err := identityprovider.SetupWithOptions(oauthOptions.OAuthOptions.IdentityProviders); err != nil {
marshal, err := yaml.Marshal(fakeIDP)
if err != nil {
return
}
secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-fake-idp",
Namespace: "kubesphere-system",
Labels: map[string]string{
constants.GenericConfigTypeLabel: identityprovider.ConfigTypeIdentityProvider,
},
},
Data: map[string][]byte{
"configuration.yaml": marshal,
},
Type: identityprovider.SecretTypeIdentityProvider,
}
fakeCache := informertest.FakeInformers{Scheme: scheme.Scheme}
err = fakeCache.Start(context.Background())
if err != nil {
t.Fatal(err)
}
fakeSecretInformer, err := fakeCache.FakeInformerFor(context.Background(), &v1.Secret{})
if err != nil {
t.Fatal(err)
}
ksClient := fakeks.NewSimpleClientset()
ksInformerFactory := ksinformers.NewSharedInformerFactory(ksClient, 0)
if err := ksInformerFactory.Iam().V1alpha2().Users().Informer().GetIndexer().Add(newUser("user1", "100001", "fake")); err != nil {
identityprovider.RegisterOAuthProviderFactory(&fakeProviderFactory{})
identityprovider.SharedIdentityProviderController = identityprovider.NewController()
err = identityprovider.SharedIdentityProviderController.WatchConfigurationChanges(context.Background(), &fakeCache)
if err != nil {
t.Fatal(err)
}
fakeSecretInformer.Add(secret)
blockedUser := newUser("user2", "100002", "fake")
blockedUser.Status = iamv1alpha2.UserStatus{State: iamv1alpha2.UserDisabled}
if err := ksInformerFactory.Iam().V1alpha2().Users().Informer().GetIndexer().Add(blockedUser); err != nil {
t.Fatal(err)
}
blockedUser.Status = iamv1beta1.UserStatus{State: iamv1beta1.UserDisabled}
client := runtimefakeclient.NewClientBuilder().
WithScheme(scheme.Scheme).
WithRuntimeObjects(newUser("user1", "100001", "fake"), secret, blockedUser).
Build()
type args struct {
ctx context.Context
@@ -97,12 +111,8 @@ func Test_oauthAuthenticator_Authenticate(t *testing.T) {
wantErr bool
}{
{
name: "Should successfully",
oauthAuthenticator: NewOAuthAuthenticator(
nil,
ksInformerFactory.Iam().V1alpha2().Users().Lister(),
oauthOptions,
),
name: "Should successfully",
oauthAuthenticator: NewOAuthAuthenticator(client),
args: args{
ctx: context.Background(),
provider: "fake",
@@ -115,12 +125,8 @@ func Test_oauthAuthenticator_Authenticate(t *testing.T) {
wantErr: false,
},
{
name: "Blocked user test",
oauthAuthenticator: NewOAuthAuthenticator(
nil,
ksInformerFactory.Iam().V1alpha2().Users().Lister(),
oauthOptions,
),
name: "Blocked user test",
oauthAuthenticator: NewOAuthAuthenticator(client),
args: args{
ctx: context.Background(),
provider: "fake",
@@ -131,12 +137,8 @@ func Test_oauthAuthenticator_Authenticate(t *testing.T) {
wantErr: true,
},
{
name: "Should successfully",
oauthAuthenticator: NewOAuthAuthenticator(
nil,
ksInformerFactory.Iam().V1alpha2().Users().Lister(),
oauthOptions,
),
name: "Should successfully",
oauthAuthenticator: NewOAuthAuthenticator(client),
args: args{
ctx: context.Background(),
provider: "fake1",
@@ -148,7 +150,7 @@ func Test_oauthAuthenticator_Authenticate(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
userInfo, provider, err := tt.oauthAuthenticator.Authenticate(tt.args.ctx, tt.args.provider, tt.args.req)
userInfo, err := tt.oauthAuthenticator.Authenticate(tt.args.ctx, tt.args.provider, tt.args.req)
if (err != nil) != tt.wantErr {
t.Errorf("Authenticate() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -156,9 +158,6 @@ func Test_oauthAuthenticator_Authenticate(t *testing.T) {
if !reflect.DeepEqual(userInfo, tt.userInfo) {
t.Errorf("Authenticate() got = %v, want %v", userInfo, tt.userInfo)
}
if provider != tt.provider {
t.Errorf("Authenticate() got = %v, want %v", provider, tt.provider)
}
})
}
}
@@ -170,17 +169,17 @@ func must(r *http.Request, err error) *http.Request {
return r
}
func newUser(username string, uid string, idp string) *iamv1alpha2.User {
return &iamv1alpha2.User{
func newUser(username string, uid string, idp string) *iamv1beta1.User {
return &iamv1beta1.User{
TypeMeta: metav1.TypeMeta{
Kind: iamv1alpha2.ResourceKindUser,
APIVersion: iamv1alpha2.SchemeGroupVersion.String(),
Kind: iamv1beta1.ResourceKindUser,
APIVersion: iamv1beta1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: username,
Labels: map[string]string{
iamv1alpha2.IdentifyProviderLabel: idp,
iamv1alpha2.OriginUIDLabel: uid,
iamv1beta1.IdentifyProviderLabel: idp,
iamv1beta1.OriginUIDLabel: uid,
},
},
}
@@ -212,7 +211,7 @@ func (f fakeIdentity) GetEmail() string {
}
func (fakeProviderFactory) Type() string {
return "FakeIdentityProvider"
return "FakeOAuthProvider"
}
func (fakeProviderFactory) Create(dynamicOptions options.DynamicOptions) (identityprovider.OAuthProvider, error) {

184
pkg/models/auth/password.go
View File

@@ -1,159 +1,151 @@
/*
Copyright 2021 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
import (
"context"
"fmt"
"golang.org/x/crypto/bcrypt"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
authuser "k8s.io/apiserver/pkg/authentication/user"
"k8s.io/klog/v2"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
iamv1alpha2listers "kubesphere.io/kubesphere/pkg/client/listers/iam/v1alpha2"
)
type passwordAuthenticator struct {
ksClient kubesphere.Interface
userGetter *userGetter
authOptions *authentication.Options
userGetter *userMapper
client runtimeclient.Client
authOptions *authentication.Options
identityProviderConfigurationGetter identityprovider.ConfigurationGetter
}
func NewPasswordAuthenticator(ksClient kubesphere.Interface,
userLister iamv1alpha2listers.UserLister,
options *authentication.Options) PasswordAuthenticator {
func NewPasswordAuthenticator(cacheClient runtimeclient.Client, options *authentication.Options) PasswordAuthenticator {
passwordAuthenticator := &passwordAuthenticator{
ksClient: ksClient,
userGetter: &userGetter{userLister: userLister},
authOptions: options,
client: cacheClient,
userGetter: &userMapper{cache: cacheClient},
identityProviderConfigurationGetter: identityprovider.NewConfigurationGetter(cacheClient),
authOptions: options,
}
return passwordAuthenticator
}
func (p *passwordAuthenticator) Authenticate(_ context.Context, provider, username, password string) (authuser.Info, string, error) {
func (p *passwordAuthenticator) Authenticate(ctx context.Context, provider, username, password string) (authuser.Info, error) {
// empty username or password are not allowed
if username == "" || password == "" {
return nil, "", IncorrectPasswordError
return nil, IncorrectPasswordError
}
if provider != "" {
return p.authByProvider(provider, username, password)
return p.authByProvider(ctx, provider, username, password)
}
return p.authByKubeSphere(username, password)
return p.authByKubeSphere(ctx, username, password)
}
// authByKubeSphere authenticate by the kubesphere user
func (p *passwordAuthenticator) authByKubeSphere(username, password string) (authuser.Info, string, error) {
user, err := p.userGetter.findUser(username)
func (p *passwordAuthenticator) authByKubeSphere(ctx context.Context, username, password string) (authuser.Info, error) {
user, err := p.userGetter.Find(ctx, username)
if err != nil {
// ignore not found error
if !errors.IsNotFound(err) {
klog.Error(err)
return nil, "", err
if errors.IsNotFound(err) {
return nil, IncorrectPasswordError
}
return nil, fmt.Errorf("failed to find user: %s", err)
}
if user == nil {
return nil, IncorrectPasswordError
}
// check user status
if user != nil && user.Status.State != iamv1alpha2.UserActive {
if user.Status.State == iamv1alpha2.UserAuthLimitExceeded {
klog.Errorf("%s, username: %s", RateLimitExceededError, username)
return nil, "", RateLimitExceededError
if user.Status.State != iamv1beta1.UserActive {
if user.Status.State == iamv1beta1.UserAuthLimitExceeded {
return nil, RateLimitExceededError
} else {
// state not active
klog.Errorf("%s, username: %s", AccountIsNotActiveError, username)
return nil, "", AccountIsNotActiveError
return nil, AccountIsNotActiveError
}
}
// if the password is not empty, means that the password has been reset, even if the user was mapping from IDP
if user != nil && user.Spec.EncryptedPassword != "" {
if err = PasswordVerify(user.Spec.EncryptedPassword, password); err != nil {
klog.Error(err)
return nil, "", err
}
u := &authuser.DefaultInfo{
Name: user.Name,
Groups: user.Spec.Groups,
}
// check if the password is initialized
if uninitialized := user.Annotations[iamv1alpha2.UninitializedAnnotation]; uninitialized != "" {
u.Extra = map[string][]string{
iamv1alpha2.ExtraUninitialized: {uninitialized},
}
}
return u, "", nil
if user.Spec.EncryptedPassword == "" {
return nil, IncorrectPasswordError
}
return nil, "", IncorrectPasswordError
if err = PasswordVerify(user.Spec.EncryptedPassword, password); err != nil {
return nil, err
}
info := &authuser.DefaultInfo{
Name: user.Name,
Groups: user.Spec.Groups,
}
// check if the password is initialized
if uninitialized := user.Annotations[iamv1beta1.UninitializedAnnotation]; uninitialized != "" {
info.Extra = map[string][]string{
iamv1beta1.ExtraUninitialized: {uninitialized},
}
}
return info, nil
}
// authByProvider authenticate by the third-party identity provider user
func (p *passwordAuthenticator) authByProvider(provider, username, password string) (authuser.Info, string, error) {
providerOptions, err := p.authOptions.OAuthOptions.IdentityProviderOptions(provider)
if err != nil {
klog.Error(err)
return nil, "", err
func (p *passwordAuthenticator) authByProvider(ctx context.Context, provider, username, password string) (authuser.Info, error) {
genericProvider, exist := identityprovider.SharedIdentityProviderController.GetGenericProvider(provider)
if !exist {
return nil, fmt.Errorf("generic identity provider %s not found", provider)
}
genericProvider, err := identityprovider.GetGenericProvider(providerOptions.Name)
providerConfig, err := p.identityProviderConfigurationGetter.GetConfiguration(ctx, provider)
if err != nil {
klog.Error(err)
return nil, "", err
return nil, fmt.Errorf("failed to get identity provider configuration: %s", err)
}
authenticated, err := genericProvider.Authenticate(username, password)
identity, err := genericProvider.Authenticate(username, password)
if err != nil {
klog.Error(err)
if errors.IsUnauthorized(err) {
return nil, "", IncorrectPasswordError
return nil, IncorrectPasswordError
}
return nil, "", err
}
linkedAccount, err := p.userGetter.findMappedUser(providerOptions.Name, authenticated.GetUserID())
if err != nil && !errors.IsNotFound(err) {
klog.Error(err)
return nil, "", err
return nil, err
}
if linkedAccount != nil {
return &authuser.DefaultInfo{Name: linkedAccount.Name}, provider, nil
mappedUser, err := p.userGetter.FindMappedUser(ctx, provider, identity.GetUserID())
if err != nil {
return nil, fmt.Errorf("failed to find mapped user: %s", err)
}
// the user will automatically create and mapping when login successful.
if providerOptions.MappingMethod == oauth.MappingMethodAuto {
if !providerOptions.DisableLoginConfirmation {
return preRegistrationUser(providerOptions.Name, authenticated), providerOptions.Name, nil
if mappedUser == nil {
if providerConfig.MappingMethod == identityprovider.MappingMethodLookup {
return nil, fmt.Errorf("failed to find mapped user: %s", identity.GetUserID())
}
linkedAccount, err = p.ksClient.IamV1alpha2().Users().Create(context.Background(), mappedUser(providerOptions.Name, authenticated), metav1.CreateOptions{})
if err != nil {
klog.Error(err)
return nil, "", err
if providerConfig.MappingMethod == identityprovider.MappingMethodManual {
return newRreRegistrationUser(providerConfig.Name, identity), nil
}
return &authuser.DefaultInfo{Name: linkedAccount.Name}, provider, nil
if providerConfig.MappingMethod == identityprovider.MappingMethodAuto {
mappedUser = newMappedUser(providerConfig.Name, identity)
if err = p.client.Create(ctx, mappedUser); err != nil {
return nil, err
}
return &authuser.DefaultInfo{Name: mappedUser.GetName()}, nil
}
return nil, fmt.Errorf("invalid mapping method found %s", providerConfig.MappingMethod)
}
return nil, "", err
if mappedUser.Status.State == iamv1beta1.UserDisabled {
return nil, AccountIsNotActiveError
}
return &authuser.DefaultInfo{Name: mappedUser.GetName()}, nil
}
func PasswordVerify(encryptedPassword, password string) error {

248
pkg/models/auth/password_test.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
@@ -23,20 +10,30 @@ import (
"reflect"
"testing"
"sigs.k8s.io/controller-runtime/pkg/cache/informertest"
"kubesphere.io/kubesphere/pkg/constants"
"gopkg.in/yaml.v3"
runtimefakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
"kubesphere.io/kubesphere/pkg/scheme"
"kubesphere.io/kubesphere/pkg/server/options"
"github.com/mitchellh/mapstructure"
"golang.org/x/crypto/bcrypt"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/user"
authuser "k8s.io/apiserver/pkg/authentication/user"
iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
"kubesphere.io/kubesphere/pkg/apiserver/authentication"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/identityprovider"
"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
fakeks "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
)
func TestEncryptPassword(t *testing.T) {
@@ -56,87 +53,154 @@ func hashPassword(password string) (string, error) {
}
func Test_passwordAuthenticator_Authenticate(t *testing.T) {
identityprovider.RegisterGenericProviderFactory(&fakePasswordProviderFactory{})
oauthOptions := &authentication.Options{
OAuthOptions: &oauth.Options{
IdentityProviders: []oauth.IdentityProviderOptions{
{
Name: "fakepwd",
MappingMethod: "auto",
Type: "fakePasswordProvider",
Provider: options.DynamicOptions{
"identities": map[string]interface{}{
"user1": map[string]string{
"uid": "100001",
"email": "user1@kubesphere.io",
"username": "user1",
"password": "password",
},
"user2": map[string]string{
"uid": "100002",
"email": "user2@kubesphere.io",
"username": "user2",
"password": "password",
},
},
},
Issuer: &oauth.IssuerOptions{},
}
fakepwd1 := &identityprovider.Configuration{
Name: "fakepwd1",
MappingMethod: "manual",
Type: "fakePasswordProvider",
ProviderOptions: options.DynamicOptions{
"identities": map[string]interface{}{
"user1": map[string]string{
"uid": "100001",
"email": "user1@kubesphere.io",
"username": "user1",
"password": "password",
},
{
Name: "fakepwd2",
MappingMethod: "auto",
Type: "fakePasswordProvider",
DisableLoginConfirmation: true,
Provider: options.DynamicOptions{
"identities": map[string]interface{}{
"user5": map[string]string{
"uid": "100005",
"email": "user5@kubesphere.io",
"username": "user5",
"password": "password",
},
},
},
},
{
Name: "fakepwd3",
MappingMethod: "lookup",
Type: "fakePasswordProvider",
DisableLoginConfirmation: true,
Provider: options.DynamicOptions{
"identities": map[string]interface{}{
"user6": map[string]string{
"uid": "100006",
"email": "user6@kubesphere.io",
"username": "user6",
"password": "password",
},
},
},
"user2": map[string]string{
"uid": "100002",
"email": "user2@kubesphere.io",
"username": "user2",
"password": "password",
},
},
},
}
identityprovider.RegisterGenericProvider(&fakePasswordProviderFactory{})
if err := identityprovider.SetupWithOptions(oauthOptions.OAuthOptions.IdentityProviders); err != nil {
t.Fatal(err)
fakepwd2 := &identityprovider.Configuration{
Name: "fakepwd2",
MappingMethod: "auto",
Type: "fakePasswordProvider",
ProviderOptions: options.DynamicOptions{
"identities": map[string]interface{}{
"user5": map[string]string{
"uid": "100005",
"email": "user5@kubesphere.io",
"username": "user5",
"password": "password",
},
},
},
}
ksClient := fakeks.NewSimpleClientset()
ksInformerFactory := ksinformers.NewSharedInformerFactory(ksClient, 0)
err := ksInformerFactory.Iam().V1alpha2().Users().Informer().GetIndexer().Add(newUser("user1", "100001", "fakepwd"))
_ = ksInformerFactory.Iam().V1alpha2().Users().Informer().GetIndexer().Add(newUser("user3", "100003", ""))
_ = ksInformerFactory.Iam().V1alpha2().Users().Informer().GetIndexer().Add(newActiveUser("user4", "password"))
fakepwd3 := &identityprovider.Configuration{
Name: "fakepwd3",
MappingMethod: "lookup",
Type: "fakePasswordProvider",
ProviderOptions: options.DynamicOptions{
"identities": map[string]interface{}{
"user6": map[string]string{
"uid": "100006",
"email": "user6@kubesphere.io",
"username": "user6",
"password": "password",
},
},
},
}
marshal1, err := yaml.Marshal(fakepwd1)
if err != nil {
return
}
fakepwd1Secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-fake-idp",
Namespace: "kubesphere-system",
Labels: map[string]string{
constants.GenericConfigTypeLabel: identityprovider.ConfigTypeIdentityProvider,
},
},
Data: map[string][]byte{
"configuration.yaml": marshal1,
},
Type: identityprovider.SecretTypeIdentityProvider,
}
marshal2, err := yaml.Marshal(fakepwd2)
if err != nil {
return
}
fakepwd2Secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-fake-idp2",
Namespace: "kubesphere-system",
Labels: map[string]string{
constants.GenericConfigTypeLabel: identityprovider.ConfigTypeIdentityProvider,
},
},
Data: map[string][]byte{
"configuration.yaml": marshal2,
},
Type: identityprovider.SecretTypeIdentityProvider,
}
marshal3, err := yaml.Marshal(fakepwd3)
if err != nil {
return
}
fakepwd3Secret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "test-fake-idp3",
Namespace: "kubesphere-system",
Labels: map[string]string{
constants.GenericConfigTypeLabel: identityprovider.ConfigTypeIdentityProvider,
},
},
Data: map[string][]byte{
"configuration.yaml": marshal3,
},
Type: identityprovider.SecretTypeIdentityProvider,
}
client := runtimefakeclient.NewClientBuilder().
WithScheme(scheme.Scheme).
WithRuntimeObjects(
newUser("user1", "100001", "fakepwd1"),
newUser("user3", "100003", ""),
newActiveUser("user4", "password"),
fakepwd1Secret,
fakepwd2Secret,
fakepwd3Secret,
).
Build()
fakeCache := informertest.FakeInformers{Scheme: scheme.Scheme}
err = fakeCache.Start(context.Background())
if err != nil {
t.Fatal(err)
}
fakeSecretInformer, err := fakeCache.FakeInformerFor(context.Background(), &v1.Secret{})
if err != nil {
t.Fatal(err)
}
authenticator := NewPasswordAuthenticator(
ksClient,
ksInformerFactory.Iam().V1alpha2().Users().Lister(),
oauthOptions,
)
identityprovider.RegisterOAuthProviderFactory(&fakeProviderFactory{})
identityprovider.SharedIdentityProviderController = identityprovider.NewController()
err = identityprovider.SharedIdentityProviderController.WatchConfigurationChanges(context.Background(), &fakeCache)
if err != nil {
t.Fatal(err)
}
fakeSecretInformer.Add(fakepwd1Secret)
fakeSecretInformer.Add(fakepwd2Secret)
fakeSecretInformer.Add(fakepwd3Secret)
authenticator := NewPasswordAuthenticator(client, oauthOptions)
type args struct {
ctx context.Context
@@ -159,7 +223,7 @@ func Test_passwordAuthenticator_Authenticate(t *testing.T) {
ctx: context.Background(),
username: "user1",
password: "password",
provider: "fakepwd",
provider: "fakepwd1",
},
want: &user.DefaultInfo{
Name: "user1",
@@ -173,13 +237,13 @@ func Test_passwordAuthenticator_Authenticate(t *testing.T) {
ctx: context.Background(),
username: "user2",
password: "password",
provider: "fakepwd",
provider: "fakepwd1",
},
want: &user.DefaultInfo{
Name: "system:pre-registration",
Extra: map[string][]string{
"email": {"user2@kubesphere.io"},
"idp": {"fakepwd"},
"idp": {"fakepwd1"},
"uid": {"100002"},
"username": {"user2"},
},
@@ -236,7 +300,7 @@ func Test_passwordAuthenticator_Authenticate(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := tt.passwordAuthenticator
got, _, err := p.Authenticate(tt.args.ctx, tt.args.provider, tt.args.username, tt.args.password)
got, err := p.Authenticate(tt.args.ctx, tt.args.provider, tt.args.username, tt.args.password)
if (err != nil) != tt.wantErr {
t.Errorf("passwordAuthenticator.Authenticate() error = %v, wantErr %v", err, tt.wantErr)
return
@@ -298,10 +362,10 @@ func encrypt(password string) (string, error) {
return string(bytes), err
}
func newActiveUser(username string, password string) *iamv1alpha2.User {
func newActiveUser(username string, password string) *iamv1beta1.User {
u := newUser(username, "", "")
password, _ = encrypt(password)
u.Spec.EncryptedPassword = password
u.Status.State = iamv1alpha2.UserActive
u.Status.State = iamv1beta1.UserActive
return u
}

38
pkg/models/auth/token.go
View File

@@ -1,20 +1,7 @@
/*
Copyright 2020 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
* Please refer to the LICENSE file in the root directory of the project.
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
*/
package auth
@@ -51,27 +38,30 @@ type tokenOperator struct {
cache cache.Interface
}
func (t tokenOperator) Revoke(token string) error {
func (t *tokenOperator) Revoke(token string) error {
pattern := fmt.Sprintf("kubesphere:user:*:token:%s", token)
if keys, err := t.cache.Keys(pattern); err != nil {
klog.Error(err)
return err
} else if len(keys) > 0 {
if err := t.cache.Del(keys...); err != nil {
klog.Error(err)
return err
}
}
return nil
}
func NewTokenOperator(cache cache.Interface, issuer token.Issuer, options *authentication.Options) TokenManagementInterface {
func NewTokenOperator(cache cache.Interface, options *authentication.Options) (TokenManagementInterface, error) {
issuer, err := token.NewIssuer(options.Issuer)
if err != nil {
klog.Errorf("Failed to create token issuer: %v", err)
return nil, err
}
operator := &tokenOperator{
issuer: issuer,
options: options,
cache: cache,
}
return operator
return operator, nil
}
func (t *tokenOperator) Verify(tokenStr string) (*token.VerifiedResponse, error) {
@@ -79,7 +69,7 @@ func (t *tokenOperator) Verify(tokenStr string) (*token.VerifiedResponse, error)
if err != nil {
return nil, err
}
if t.options.OAuthOptions.AccessTokenMaxAge == 0 ||
if t.options.Issuer.AccessTokenMaxAge == 0 ||
response.TokenType == token.StaticToken {
return response, nil
}
@@ -92,12 +82,10 @@ func (t *tokenOperator) Verify(tokenStr string) (*token.VerifiedResponse, error)
func (t *tokenOperator) IssueTo(request *token.IssueRequest) (string, error) {
tokenStr, err := t.issuer.IssueTo(request)
if err != nil {
klog.Error(err)
return "", err
}
if request.ExpiresIn > 0 {
if err = t.cacheToken(request.User.GetName(), tokenStr, request.ExpiresIn); err != nil {
klog.Error(err)
return "", err
}
}
@@ -108,11 +96,9 @@ func (t *tokenOperator) IssueTo(request *token.IssueRequest) (string, error) {
func (t *tokenOperator) RevokeAllUserTokens(username string) error {
pattern := fmt.Sprintf("kubesphere:user:%s:token:*", username)
if keys, err := t.cache.Keys(pattern); err != nil {
klog.Error(err)
return err
} else if len(keys) > 0 {
if err := t.cache.Del(keys...); err != nil {
klog.Error(err)
return err
}
}