536 lines
16 KiB
Go
536 lines
16 KiB
Go
/*
|
|
Copyright 2018 The KubeSphere Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package controllers
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/golang/glog"
|
|
"k8s.io/api/core/v1"
|
|
rbac "k8s.io/api/rbac/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/client-go/tools/cache"
|
|
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/kubernetes/pkg/util/slice"
|
|
|
|
"k8s.io/client-go/informers"
|
|
|
|
"k8s.io/kubernetes/pkg/apis/core"
|
|
|
|
utilversion "k8s.io/kubernetes/pkg/util/version"
|
|
|
|
"kubesphere.io/kubesphere/pkg/client"
|
|
"kubesphere.io/kubesphere/pkg/constants"
|
|
"kubesphere.io/kubesphere/pkg/options"
|
|
)
|
|
|
|
const (
|
|
provider = "kubernetes"
|
|
admin = "admin"
|
|
operator = "operator"
|
|
viewer = "viewer"
|
|
kubectlNamespace = constants.KubeSphereControlNamespace
|
|
kubectlConfigKey = "config"
|
|
openPitrixRuntimeAnnotateKey = "openpitrix_runtime"
|
|
creatorAnnotateKey = "creator"
|
|
initTimeAnnotateKey = "kubesphere.io/init-time"
|
|
workspaceLabelKey = "kubesphere.io/workspace"
|
|
)
|
|
|
|
var adminRules = []rbac.PolicyRule{{Verbs: []string{"*"}, APIGroups: []string{"*"}, Resources: []string{"*"}}}
|
|
var editorRules = []rbac.PolicyRule{{Verbs: []string{"*"}, APIGroups: []string{"", "apps", "extensions", "batch", "kubesphere.io", "account.kubesphere.io"}, Resources: []string{"*"}}}
|
|
var viewerRules = []rbac.PolicyRule{{Verbs: []string{"list", "get", "watch"}, APIGroups: []string{"", "apps", "extensions", "batch", "kubesphere.io", "account.kubesphere.io"}, Resources: []string{"*"}}}
|
|
|
|
type runTime struct {
|
|
RuntimeId string `json:"runtime_id"`
|
|
RuntimeUrl string `json:"runtime_url"`
|
|
Name string `json:"name"`
|
|
Provider string `json:"provider"`
|
|
Zone string `json:"zone"`
|
|
RuntimeCredential string `json:"runtime_credential"`
|
|
}
|
|
|
|
type DeleteRunTime struct {
|
|
RuntimeId []string `json:"runtime_id"`
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) getKubeConfig(user string) (string, error) {
|
|
k8sClient := client.NewK8sClient()
|
|
configmap, err := k8sClient.CoreV1().ConfigMaps(kubectlNamespace).Get(user, metaV1.GetOptions{})
|
|
if err != nil {
|
|
glog.Errorln(err)
|
|
return "", err
|
|
}
|
|
return configmap.Data[kubectlConfigKey], nil
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) deleteOpRuntime(item v1.Namespace) {
|
|
|
|
var runtimeId string
|
|
if item.Annotations == nil {
|
|
runtimeId = ""
|
|
} else {
|
|
runtimeId = item.Annotations[openPitrixRuntimeAnnotateKey]
|
|
}
|
|
|
|
if len(runtimeId) == 0 {
|
|
return
|
|
}
|
|
|
|
url := options.ServerOptions.GetOpAddress() + "/v1/runtimes"
|
|
|
|
var deleteRuntime = DeleteRunTime{RuntimeId: []string{runtimeId}}
|
|
|
|
body, err := json.Marshal(deleteRuntime)
|
|
if err != nil {
|
|
glog.Error("runtime release failed:", item.Name, runtimeId, err)
|
|
return
|
|
}
|
|
|
|
glog.Info("runtime release succeeded:", item.Name, runtimeId)
|
|
|
|
// todo: if delete failed, what's to be done?
|
|
makeHttpRequest("DELETE", url, string(body))
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) createOpRuntime(namespace string) ([]byte, error) {
|
|
zone := namespace
|
|
name := namespace
|
|
kubeConfig, err := ctl.getKubeConfig("admin")
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
url := options.ServerOptions.GetOpAddress() + "/v1/runtimes"
|
|
|
|
option := runTime{Name: name, Provider: provider, RuntimeCredential: kubeConfig, Zone: zone}
|
|
body, err := json.Marshal(option)
|
|
if err != nil {
|
|
glog.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
return makeHttpRequest("POST", url, string(body))
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) updateSystemRoleBindings(namespace *v1.Namespace) error {
|
|
|
|
workspace := ""
|
|
|
|
if namespace.Labels != nil {
|
|
workspace = namespace.Labels[workspaceLabelKey]
|
|
}
|
|
|
|
adminBinding, err := ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Get(admin, metaV1.GetOptions{})
|
|
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
adminBinding = new(rbac.RoleBinding)
|
|
adminBinding.Name = admin
|
|
adminBinding.Namespace = namespace.Name
|
|
adminBinding.RoleRef = rbac.RoleRef{Kind: "Role", Name: admin}
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
|
|
adminBinding.Subjects = make([]rbac.Subject, 0)
|
|
|
|
if workspace != "" {
|
|
workspaceAdmin, err := ctl.K8sClient.RbacV1().ClusterRoleBindings().Get(fmt.Sprintf("system:%s:%s", workspace, constants.WorkspaceAdmin), metaV1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
adminBinding.Subjects = append(adminBinding.Subjects, workspaceAdmin.Subjects...)
|
|
}
|
|
|
|
if adminBinding.ResourceVersion == "" {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Create(adminBinding)
|
|
} else {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Update(adminBinding)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
viewerBinding, err := ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Get(viewer, metaV1.GetOptions{})
|
|
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
viewerBinding = new(rbac.RoleBinding)
|
|
viewerBinding.Name = viewer
|
|
viewerBinding.Namespace = namespace.Name
|
|
viewerBinding.RoleRef = rbac.RoleRef{Kind: "Role", Name: viewer}
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
|
|
viewerBinding.Subjects = make([]rbac.Subject, 0)
|
|
|
|
if workspace != "" {
|
|
workspaceViewer, err := ctl.K8sClient.RbacV1().ClusterRoleBindings().Get(fmt.Sprintf("system:%s:%s", workspace, constants.WorkspaceViewer), metaV1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
viewerBinding.Subjects = append(viewerBinding.Subjects, workspaceViewer.Subjects...)
|
|
}
|
|
|
|
if viewerBinding.ResourceVersion == "" {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Create(viewerBinding)
|
|
} else {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Update(viewerBinding)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) createDefaultRoleBinding(namespace *v1.Namespace) error {
|
|
creator := ""
|
|
if namespace.Annotations != nil {
|
|
creator = namespace.Annotations[creatorAnnotateKey]
|
|
}
|
|
// create once
|
|
if creator != "" {
|
|
creatorBindingName := fmt.Sprintf("%s-admin", creator)
|
|
creatorBinding, err := ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Get(creatorBindingName, metaV1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
creatorBinding = new(rbac.RoleBinding)
|
|
creatorBinding.Name = creatorBindingName
|
|
creatorBinding.Namespace = namespace.Name
|
|
creatorBinding.RoleRef = rbac.RoleRef{Kind: "Role", Name: admin}
|
|
} else {
|
|
return err
|
|
}
|
|
}
|
|
|
|
creatorBinding.Subjects = []rbac.Subject{{Kind: rbac.UserKind, Name: creator}}
|
|
|
|
if creatorBinding.ResourceVersion == "" {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Create(creatorBinding)
|
|
} else {
|
|
_, err = ctl.K8sClient.RbacV1().RoleBindings(namespace.Name).Update(creatorBinding)
|
|
}
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) CreateDefaultRoleAndRoleBinding(namespace *v1.Namespace) error {
|
|
adminRole := &rbac.Role{ObjectMeta: metaV1.ObjectMeta{Name: admin, Namespace: namespace.Name, Annotations: map[string]string{creatorAnnotateKey: "system"}}, Rules: adminRules}
|
|
operatorRole := &rbac.Role{ObjectMeta: metaV1.ObjectMeta{Name: operator, Namespace: namespace.Name, Annotations: map[string]string{creatorAnnotateKey: "system"}}, Rules: editorRules}
|
|
viewerRole := &rbac.Role{ObjectMeta: metaV1.ObjectMeta{Name: viewer, Namespace: namespace.Name, Annotations: map[string]string{creatorAnnotateKey: "system"}}, Rules: viewerRules}
|
|
|
|
_, err := ctl.K8sClient.RbacV1().Roles(namespace.Name).Create(adminRole)
|
|
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return err
|
|
} else if err == nil {
|
|
if err := ctl.createDefaultRoleBinding(namespace); err != nil {
|
|
glog.Warning("default role binding create failed", namespace.Name)
|
|
}
|
|
}
|
|
|
|
_, err = ctl.K8sClient.RbacV1().Roles(namespace.Name).Create(operatorRole)
|
|
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return err
|
|
}
|
|
|
|
_, err = ctl.K8sClient.RbacV1().Roles(namespace.Name).Create(viewerRole)
|
|
|
|
if err != nil && !errors.IsAlreadyExists(err) {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) createRoleAndRuntime(namespace *v1.Namespace) {
|
|
runtime := ""
|
|
initTime := ""
|
|
if namespace.Annotations != nil {
|
|
runtime = namespace.Annotations[openPitrixRuntimeAnnotateKey]
|
|
initTime = namespace.Annotations[initTimeAnnotateKey]
|
|
}
|
|
|
|
componentsNamespaces := []string{constants.KubeSystemNamespace, constants.OpenPitrixNamespace, constants.IstioNamespace, constants.KubeSphereNamespace}
|
|
|
|
if runtime == "" && !slice.ContainsString(componentsNamespaces, namespace.Name, nil) {
|
|
_, runtimeCreateError := ctl.createOpRuntime(namespace.Name)
|
|
if runtimeCreateError != nil {
|
|
glog.Error("runtime create error:", runtimeCreateError)
|
|
}
|
|
}
|
|
|
|
if initTime == "" {
|
|
err := ctl.CreateDefaultRoleAndRoleBinding(namespace)
|
|
if err == nil {
|
|
err = ctl.updateSystemRoleBindings(namespace)
|
|
if err != nil {
|
|
glog.Error("role binding update error:", err)
|
|
}
|
|
} else {
|
|
glog.Error("default role create error:", err)
|
|
}
|
|
if err == nil {
|
|
pathJson := fmt.Sprintf(`{"metadata":{"annotations":{"%s":"%s"}}}`, initTimeAnnotateKey, time.Now().UTC().Format("2006-01-02T15:04:05Z"))
|
|
_, err = ctl.K8sClient.CoreV1().Namespaces().Patch(namespace.Name, "application/strategic-merge-patch+json", []byte(pathJson))
|
|
if err != nil {
|
|
glog.Error("annotations patch error init failed:", namespace.Name, err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) createCephSecretAfterNewNs(item v1.Namespace) {
|
|
// Kubernetes version must < 1.11.0
|
|
verInfo, err := ctl.K8sClient.ServerVersion()
|
|
if err != nil {
|
|
glog.Error("consult k8s server error: ", err)
|
|
return
|
|
}
|
|
if !utilversion.MustParseSemantic(verInfo.String()).LessThan(utilversion.MustParseSemantic("v1.11.0")) {
|
|
glog.Infof("disable Ceph secret controller due to k8s version %s >= v1.11.0", verInfo.String())
|
|
return
|
|
}
|
|
|
|
// Create Ceph secret in the new namespace
|
|
newNsName := item.Name
|
|
scList, _ := ctl.K8sClient.StorageV1().StorageClasses().List(metaV1.ListOptions{})
|
|
if scList == nil {
|
|
return
|
|
}
|
|
for _, sc := range scList.Items {
|
|
if sc.Provisioner == rbdPluginName {
|
|
glog.Infof("would create Ceph user secret in storage class %s at namespace %s", sc.GetName(), newNsName)
|
|
if secretName, ok := sc.Parameters[rbdUserSecretNameKey]; ok {
|
|
secret, err := ctl.K8sClient.CoreV1().Secrets(core.NamespaceSystem).Get(secretName, metaV1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
glog.Errorf("cannot find secret in namespace %s, error: %s", core.NamespaceSystem, err.Error())
|
|
continue
|
|
}
|
|
glog.Errorf("failed to find secret in namespace %s, error: %s", core.NamespaceSystem, err.Error())
|
|
continue
|
|
}
|
|
glog.Infof("succeed to find secret %s in namespace %s", secret.GetName(), secret.GetNamespace())
|
|
|
|
newSecret := &v1.Secret{
|
|
TypeMeta: metaV1.TypeMeta{
|
|
Kind: secret.Kind,
|
|
APIVersion: secret.APIVersion,
|
|
},
|
|
ObjectMeta: metaV1.ObjectMeta{
|
|
Name: secret.GetName(),
|
|
Namespace: newNsName,
|
|
Labels: secret.GetLabels(),
|
|
Annotations: secret.GetAnnotations(),
|
|
DeletionGracePeriodSeconds: secret.GetDeletionGracePeriodSeconds(),
|
|
ClusterName: secret.GetClusterName(),
|
|
},
|
|
Data: secret.Data,
|
|
StringData: secret.StringData,
|
|
Type: secret.Type,
|
|
}
|
|
glog.Infof("creating secret %s in namespace %s...", newSecret.GetName(), newSecret.GetNamespace())
|
|
_, err = ctl.K8sClient.CoreV1().Secrets(newSecret.GetNamespace()).Create(newSecret)
|
|
if err != nil {
|
|
glog.Errorf("failed to create secret in namespace %s, error: %v", newSecret.GetNamespace(), err)
|
|
continue
|
|
}
|
|
} else {
|
|
glog.Errorf("failed to find user secret name in storage class %s", sc.GetName())
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) generateObject(item *v1.Namespace) *Namespace {
|
|
var displayName string
|
|
|
|
if item.Annotations != nil && len(item.Annotations[DisplayName]) > 0 {
|
|
displayName = item.Annotations[DisplayName]
|
|
}
|
|
|
|
name := item.Name
|
|
createTime := item.CreationTimestamp.Time
|
|
status := fmt.Sprintf("%v", item.Status.Phase)
|
|
|
|
if createTime.IsZero() {
|
|
createTime = time.Now()
|
|
}
|
|
|
|
object := &Namespace{
|
|
Name: name,
|
|
DisplayName: displayName,
|
|
CreateTime: createTime,
|
|
Status: status,
|
|
Annotation: MapString{item.Annotations},
|
|
}
|
|
|
|
return object
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) Name() string {
|
|
return ctl.CommonAttribute.Name
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) sync(stopChan chan struct{}) {
|
|
db := ctl.DB
|
|
|
|
if db.HasTable(&Namespace{}) {
|
|
db.DropTable(&Namespace{})
|
|
}
|
|
|
|
db = db.CreateTable(&Namespace{})
|
|
|
|
ctl.initListerAndInformer()
|
|
//list, err := ctl.lister.List(labels.Everything())
|
|
//if err != nil {
|
|
// glog.Error(err)
|
|
// return
|
|
//}
|
|
|
|
//for _, item := range list {
|
|
// obj := ctl.generateObject(item)
|
|
// db.Create(obj)
|
|
// ctl.createRoleAndRuntime(item)
|
|
//}
|
|
|
|
ctl.informer.Run(stopChan)
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) total() int {
|
|
list, err := ctl.lister.List(labels.Everything())
|
|
if err != nil {
|
|
glog.Errorf("count %s falied, reason:%s", err, ctl.Name())
|
|
return 0
|
|
}
|
|
return len(list)
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) initListerAndInformer() {
|
|
db := ctl.DB
|
|
|
|
informerFactory := informers.NewSharedInformerFactory(ctl.K8sClient, time.Second*resyncCircle)
|
|
|
|
ctl.lister = informerFactory.Core().V1().Namespaces().Lister()
|
|
|
|
informer := informerFactory.Core().V1().Namespaces().Informer()
|
|
informer.AddEventHandler(cache.ResourceEventHandlerFuncs{
|
|
AddFunc: func(obj interface{}) {
|
|
|
|
object := obj.(*v1.Namespace)
|
|
mysqlObject := ctl.generateObject(object)
|
|
db.Create(mysqlObject)
|
|
ctl.createRoleAndRuntime(object)
|
|
ctl.createCephSecretAfterNewNs(*object)
|
|
},
|
|
UpdateFunc: func(old, new interface{}) {
|
|
object := new.(*v1.Namespace)
|
|
mysqlObject := ctl.generateObject(object)
|
|
db.Save(mysqlObject)
|
|
ctl.createRoleAndRuntime(object)
|
|
},
|
|
DeleteFunc: func(obj interface{}) {
|
|
var item Namespace
|
|
object := obj.(*v1.Namespace)
|
|
db.Where("name=?", object.Name).Find(&item)
|
|
db.Delete(item)
|
|
ctl.deleteOpRuntime(*object)
|
|
},
|
|
})
|
|
|
|
ctl.informer = informer
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) CountWithConditions(conditions string) int {
|
|
var object Namespace
|
|
|
|
return countWithConditions(ctl.DB, conditions, &object)
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) ListWithConditions(conditions string, paging *Paging, order string) (int, interface{}, error) {
|
|
var list []Namespace
|
|
var object Namespace
|
|
var total int
|
|
|
|
if len(order) == 0 {
|
|
order = "createTime desc"
|
|
}
|
|
|
|
listWithConditions(ctl.DB, &total, &object, &list, conditions, paging, order)
|
|
|
|
if paging != nil {
|
|
for index := range list {
|
|
usage, err := ctl.GetNamespaceQuota(list[index].Name)
|
|
if err == nil {
|
|
list[index].Usage = usage
|
|
}
|
|
}
|
|
}
|
|
|
|
return total, list, nil
|
|
}
|
|
|
|
func getUsage(namespace, resource string) int {
|
|
ctl := ResourceControllers.Controllers[resource]
|
|
return ctl.CountWithConditions(fmt.Sprintf("namespace = '%s' ", namespace))
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) GetNamespaceQuota(namespace string) (v1.ResourceList, error) {
|
|
|
|
usage := make(v1.ResourceList)
|
|
|
|
resourceList := []string{Daemonsets, Deployments, Ingresses, Roles, Services, Statefulsets, PersistentVolumeClaim, Pods, Jobs, Cronjobs}
|
|
for _, resourceName := range resourceList {
|
|
used := getUsage(namespace, resourceName)
|
|
var quantity resource.Quantity
|
|
quantity.Set(int64(used))
|
|
usage[v1.ResourceName(resourceName)] = quantity
|
|
}
|
|
|
|
podCtl := ResourceControllers.Controllers[Pods]
|
|
var quantity resource.Quantity
|
|
used := podCtl.CountWithConditions(fmt.Sprintf("status=\"%s\" And namespace=\"%s\"", "Running", namespace))
|
|
quantity.Set(int64(used))
|
|
usage["runningPods"] = quantity
|
|
return usage, nil
|
|
}
|
|
|
|
func (ctl *NamespaceCtl) Lister() interface{} {
|
|
|
|
return ctl.lister
|
|
}
|