114f5a6e79
Signed-off-by: peng wu <2030047311@qq.com>
Signed-off-by: hongming <coder.scala@gmail.com>
(cherry picked from commit 3d40b1905d)
350 lines
14 KiB
Go
350 lines
14 KiB
Go
/*
|
|
* Please refer to the LICENSE file in the root directory of the project.
|
|
* https://github.com/kubesphere/kubesphere/blob/master/LICENSE
|
|
*/
|
|
|
|
package workspacetemplate
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/go-logr/logr"
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/validation"
|
|
"k8s.io/apimachinery/pkg/util/yaml"
|
|
"k8s.io/client-go/tools/record"
|
|
"k8s.io/klog/v2"
|
|
clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
|
|
iamv1beta1 "kubesphere.io/api/iam/v1beta1"
|
|
tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
|
|
ctrl "sigs.k8s.io/controller-runtime"
|
|
"sigs.k8s.io/controller-runtime/pkg/builder"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller"
|
|
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
|
|
"sigs.k8s.io/controller-runtime/pkg/handler"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
|
|
"kubesphere.io/kubesphere/pkg/constants"
|
|
kscontroller "kubesphere.io/kubesphere/pkg/controller"
|
|
"kubesphere.io/kubesphere/pkg/controller/cluster/predicate"
|
|
clusterutils "kubesphere.io/kubesphere/pkg/controller/cluster/utils"
|
|
"kubesphere.io/kubesphere/pkg/controller/workspacetemplate/utils"
|
|
"kubesphere.io/kubesphere/pkg/utils/clusterclient"
|
|
"kubesphere.io/kubesphere/pkg/utils/hashutil"
|
|
)
|
|
|
|
const (
|
|
controllerName = "workspacetemplate"
|
|
)
|
|
|
|
// Reconciler reconciles a WorkspaceRoleBinding object
|
|
type Reconciler struct {
|
|
client.Client
|
|
logger logr.Logger
|
|
recorder record.EventRecorder
|
|
clusterClientSet clusterclient.Interface
|
|
}
|
|
|
|
func (r *Reconciler) Enabled(clusterRole string) bool {
|
|
return strings.EqualFold(clusterRole, string(clusterv1alpha1.ClusterRoleHost))
|
|
}
|
|
|
|
var _ kscontroller.Controller = &Reconciler{}
|
|
var _ reconcile.Reconciler = &Reconciler{}
|
|
|
|
func (r *Reconciler) Name() string {
|
|
return controllerName
|
|
}
|
|
|
|
func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
|
|
r.clusterClientSet = mgr.ClusterClient
|
|
r.Client = mgr.GetClient()
|
|
r.logger = ctrl.Log.WithName("controllers").WithName(controllerName)
|
|
r.recorder = mgr.GetEventRecorderFor(controllerName)
|
|
return ctrl.NewControllerManagedBy(mgr).
|
|
Named(controllerName).
|
|
WithOptions(controller.Options{MaxConcurrentReconciles: 2}).
|
|
For(&tenantv1beta1.WorkspaceTemplate{}).
|
|
Watches(
|
|
&clusterv1alpha1.Cluster{},
|
|
handler.EnqueueRequestsFromMapFunc(r.mapper),
|
|
builder.WithPredicates(predicate.ClusterStatusChangedPredicate{}),
|
|
).
|
|
Complete(r)
|
|
}
|
|
|
|
func (r *Reconciler) mapper(ctx context.Context, o client.Object) []reconcile.Request {
|
|
cluster := o.(*clusterv1alpha1.Cluster)
|
|
if !clusterutils.IsClusterReady(cluster) {
|
|
return []reconcile.Request{}
|
|
}
|
|
workspaceTemplates := &tenantv1beta1.WorkspaceTemplateList{}
|
|
if err := r.List(ctx, workspaceTemplates); err != nil {
|
|
r.logger.Error(err, "failed to list workspace templates")
|
|
return []reconcile.Request{}
|
|
}
|
|
var result []reconcile.Request
|
|
for _, workspaceTemplate := range workspaceTemplates.Items {
|
|
if utils.WorkspaceTemplateMatchTargetCluster(&workspaceTemplate, cluster) {
|
|
result = append(result, reconcile.Request{NamespacedName: types.NamespacedName{Name: workspaceTemplate.Name}})
|
|
}
|
|
}
|
|
return result
|
|
}
|
|
|
|
// +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete
|
|
// +kubebuilder:rbac:groups=tenant.kubesphere.io,resources=workspaces,verbs=get;list;watch;
|
|
|
|
func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
|
|
logger := r.logger.WithValues("workspacetemplate", req.NamespacedName)
|
|
workspaceTemplate := &tenantv1beta1.WorkspaceTemplate{}
|
|
if err := r.Get(ctx, req.NamespacedName, workspaceTemplate); err != nil {
|
|
return ctrl.Result{}, client.IgnoreNotFound(err)
|
|
}
|
|
|
|
ctx = klog.NewContext(ctx, logger)
|
|
if workspaceTemplate.ObjectMeta.DeletionTimestamp.IsZero() {
|
|
// The object is not being deleted, so if it does not have our finalizer,
|
|
// then lets add the finalizer and update the object.
|
|
if !controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) {
|
|
if err := r.initWorkspaceRoles(ctx, workspaceTemplate); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
if err := r.initManagerRoleBinding(ctx, workspaceTemplate); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
updated := workspaceTemplate.DeepCopy()
|
|
// Remove legacy finalizer
|
|
controllerutil.RemoveFinalizer(updated, "finalizers.workspacetemplate.kubesphere.io")
|
|
controllerutil.AddFinalizer(updated, constants.CascadingDeletionFinalizer)
|
|
return ctrl.Result{}, r.Patch(ctx, updated, client.MergeFrom(workspaceTemplate))
|
|
}
|
|
} else {
|
|
// The object is being deleted
|
|
if controllerutil.ContainsFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer) {
|
|
ok, err := r.workspaceTemplateCascadingDeletion(ctx, workspaceTemplate)
|
|
if err != nil {
|
|
return ctrl.Result{}, fmt.Errorf("failed to cascade delete workspacetemplate %s: %s", workspaceTemplate.Name, err)
|
|
}
|
|
if ok {
|
|
// remove our finalizer from the list and update it.
|
|
controllerutil.RemoveFinalizer(workspaceTemplate, constants.CascadingDeletionFinalizer)
|
|
if err := r.Update(ctx, workspaceTemplate); err != nil {
|
|
return ctrl.Result{}, fmt.Errorf("failed to remove finalizer: %s", err)
|
|
}
|
|
}
|
|
}
|
|
// Our finalizer has finished, so the reconciler can do nothing.
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
if err := r.multiClusterSync(ctx, workspaceTemplate); err != nil {
|
|
return ctrl.Result{}, err
|
|
}
|
|
|
|
r.recorder.Event(workspaceTemplate, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
|
|
return ctrl.Result{}, nil
|
|
}
|
|
|
|
func (r *Reconciler) multiClusterSync(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
|
|
clusters, err := r.clusterClientSet.ListClusters(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to list clusters: %s", err)
|
|
}
|
|
var notReadyClusters []string
|
|
for _, cluster := range clusters {
|
|
// skip if cluster is not ready
|
|
if !clusterutils.IsClusterReady(&cluster) {
|
|
notReadyClusters = append(notReadyClusters, cluster.Name)
|
|
continue
|
|
}
|
|
if err := r.syncWorkspaceTemplate(ctx, cluster, workspaceTemplate); err != nil {
|
|
return fmt.Errorf("failed to sync workspace template %s to cluster %s: %s", workspaceTemplate.Name, cluster.Name, err)
|
|
}
|
|
}
|
|
if len(notReadyClusters) > 0 {
|
|
klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
|
|
r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Reconciler) syncWorkspaceTemplate(ctx context.Context, cluster clusterv1alpha1.Cluster, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
|
|
clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get cluster client for %s: %s", cluster.Name, err)
|
|
}
|
|
if utils.WorkspaceTemplateMatchTargetCluster(workspaceTemplate, &cluster) {
|
|
target := &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}}
|
|
op, err := controllerutil.CreateOrUpdate(ctx, clusterClient, target, func() error {
|
|
for k, v := range workspaceTemplate.Spec.Template.Labels {
|
|
if target.Labels == nil {
|
|
target.Labels = make(map[string]string)
|
|
}
|
|
target.Labels[k] = v
|
|
}
|
|
for k, v := range workspaceTemplate.Spec.Template.Annotations {
|
|
if target.Annotations == nil {
|
|
target.Annotations = make(map[string]string)
|
|
}
|
|
target.Annotations[k] = v
|
|
}
|
|
target.Spec = workspaceTemplate.Spec.Template.Spec
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
klog.FromContext(ctx).V(4).Info("workspace successfully synced", "cluster", cluster.Name, "operation", op)
|
|
} else {
|
|
err = clusterClient.Delete(ctx, &tenantv1beta1.Workspace{ObjectMeta: metav1.ObjectMeta{Name: workspaceTemplate.Name}})
|
|
return client.IgnoreNotFound(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Reconciler) initWorkspaceRoles(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
|
|
logger := klog.FromContext(ctx)
|
|
var templates iamv1beta1.BuiltinRoleList
|
|
// scope.iam.kubesphere.io/workspace: ""
|
|
if err := r.List(ctx, &templates, client.MatchingLabels{iamv1beta1.ScopeLabel: iamv1beta1.ScopeWorkspace}); err != nil {
|
|
return err
|
|
}
|
|
for _, template := range templates.Items {
|
|
selector, err := metav1.LabelSelectorAsSelector(&template.TargetSelector)
|
|
if err != nil {
|
|
logger.V(4).Error(err, "failed to pares target selector", "template", template.Name)
|
|
continue
|
|
}
|
|
if !selector.Matches(labels.Set(workspaceTemplate.Labels)) {
|
|
continue
|
|
}
|
|
var builtinWorkspaceRole iamv1beta1.WorkspaceRole
|
|
if err := yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(template.Role.Raw), 1024).Decode(&builtinWorkspaceRole); err == nil &&
|
|
builtinWorkspaceRole.Kind == iamv1beta1.ResourceKindWorkspaceRole {
|
|
target := &iamv1beta1.WorkspaceRole{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: ensureWorkspaceRoleName(workspaceTemplate.Name, builtinWorkspaceRole.Name),
|
|
},
|
|
}
|
|
op, err := controllerutil.CreateOrUpdate(ctx, r.Client, target, func() error {
|
|
target.Labels = builtinWorkspaceRole.Labels
|
|
if target.Labels == nil {
|
|
target.Labels = make(map[string]string)
|
|
}
|
|
target.Labels[tenantv1beta1.WorkspaceLabel] = workspaceTemplate.Name
|
|
target.Annotations = builtinWorkspaceRole.Annotations
|
|
target.AggregationRoleTemplates = builtinWorkspaceRole.AggregationRoleTemplates
|
|
target.Rules = builtinWorkspaceRole.Rules
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
logger.V(4).Info("builtin workspace role successfully updated", "operation", op, "name", target.Name)
|
|
} else if err != nil {
|
|
logger.Error(err, "invalid builtin workspace role found", "name", template.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ensureWorkspaceRoleName(workspace, role string) string {
|
|
workspaceRoleName := fmt.Sprintf("%s-%s", workspace, role)
|
|
if len(workspaceRoleName) <= validation.LabelValueMaxLength {
|
|
return workspaceRoleName
|
|
}
|
|
hashedWorkspaceName := hashutil.FNVString([]byte(workspace))
|
|
return fmt.Sprintf("%s.%s", role, hashedWorkspaceName)
|
|
}
|
|
|
|
func (r *Reconciler) initManagerRoleBinding(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
|
|
manager := workspaceTemplate.Spec.Template.Spec.Manager
|
|
if manager == "" {
|
|
return nil
|
|
}
|
|
workspaceAdminRoleName := ensureWorkspaceRoleName(workspaceTemplate.Name, "admin")
|
|
existWorkspaceRoleBinding := &iamv1beta1.WorkspaceRoleBinding{ObjectMeta: metav1.ObjectMeta{Name: workspaceAdminRoleName}}
|
|
if _, err := ctrl.CreateOrUpdate(ctx, r.Client, existWorkspaceRoleBinding, func() error {
|
|
existWorkspaceRoleBinding.Labels = map[string]string{
|
|
tenantv1beta1.WorkspaceLabel: workspaceTemplate.Name,
|
|
iamv1beta1.UserReferenceLabel: manager,
|
|
iamv1beta1.RoleReferenceLabel: workspaceAdminRoleName,
|
|
}
|
|
existWorkspaceRoleBinding.RoleRef = rbacv1.RoleRef{
|
|
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
|
|
Kind: iamv1beta1.ResourceKindWorkspaceRole,
|
|
Name: workspaceAdminRoleName,
|
|
}
|
|
existWorkspaceRoleBinding.Subjects = []rbacv1.Subject{
|
|
{
|
|
Name: manager,
|
|
Kind: iamv1beta1.ResourceKindUser,
|
|
APIGroup: iamv1beta1.SchemeGroupVersion.Group,
|
|
},
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *Reconciler) workspaceTemplateCascadingDeletion(ctx context.Context, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) (bool, error) {
|
|
clusters, err := r.clusterClientSet.ListClusters(ctx)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to list clusters: %s", err)
|
|
}
|
|
var notReadyClusters []string
|
|
for _, cluster := range clusters {
|
|
// skip if cluster is not ready
|
|
if !clusterutils.IsClusterReady(&cluster) {
|
|
notReadyClusters = append(notReadyClusters, cluster.Name)
|
|
continue
|
|
}
|
|
clusterClient, err := r.clusterClientSet.GetRuntimeClient(cluster.Name)
|
|
if err != nil {
|
|
notReadyClusters = append(notReadyClusters, cluster.Name)
|
|
continue
|
|
}
|
|
if err := r.workspaceCascadingDeletion(ctx, cluster.Name, clusterClient, workspaceTemplate); err != nil {
|
|
return false, fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspaceTemplate.Name, cluster.Name, err)
|
|
}
|
|
}
|
|
if len(notReadyClusters) > 0 {
|
|
klog.FromContext(ctx).V(4).Info("cluster not ready", "clusters", strings.Join(notReadyClusters, ","))
|
|
r.recorder.Event(workspaceTemplate, corev1.EventTypeWarning, kscontroller.SyncFailed, fmt.Sprintf("cluster not ready: %s", strings.Join(notReadyClusters, ",")))
|
|
return false, fmt.Errorf("cluster not ready: %s", strings.Join(notReadyClusters, ","))
|
|
}
|
|
return true, nil
|
|
}
|
|
|
|
func (r *Reconciler) workspaceCascadingDeletion(ctx context.Context, clusterName string, clusterClient client.Client, workspaceTemplate *tenantv1beta1.WorkspaceTemplate) error {
|
|
workspace := &tenantv1beta1.Workspace{}
|
|
if err := clusterClient.Get(ctx, types.NamespacedName{Name: workspaceTemplate.Name}, workspace); err != nil {
|
|
return client.IgnoreNotFound(err)
|
|
}
|
|
if !workspace.DeletionTimestamp.IsZero() {
|
|
return nil
|
|
}
|
|
if workspace.Annotations == nil {
|
|
workspace.Annotations = make(map[string]string)
|
|
}
|
|
workspace.Annotations[constants.DeletionPropagationAnnotation] = workspaceTemplate.Annotations[constants.DeletionPropagationAnnotation]
|
|
if err := clusterClient.Update(ctx, workspace); err != nil {
|
|
return fmt.Errorf("failed to update workspace %s in cluster %s: %s", workspace.Name, clusterName, err)
|
|
}
|
|
if err := clusterClient.Delete(ctx, workspace); err != nil {
|
|
return fmt.Errorf("failed to delete workspace %s in cluster %s: %s", workspace.Name, clusterName, err)
|
|
}
|
|
return nil
|
|
}
|