feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

pkg/controller/workspace/metrics.go

@@ -1,38 +1,28 @@
-// Copyright 2022 The KubeSphere Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/*
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
+
 package workspace
 
 import (
-	compbasemetrics "k8s.io/component-base/metrics"
-
-	"kubesphere.io/kubesphere/pkg/utils/metrics"
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 var (
-	workspaceOperation = compbasemetrics.NewCounterVec(
-		&compbasemetrics.CounterOpts{
+	workspaceOperation = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
 			Name: "ks_controller_manager_workspace_operation",
 			Help: "Counter of ks controller manager workspace operation broken out for each operation, name",
 			// This metric is used for verifying api call latencies SLO,
 			// as well as tracking regressions in this aspects.
 			// Thus we customize buckets significantly, to empower both usecases.
-			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{"operation", "name"},
 	)
 )
 
 func init() {
-	metrics.MustRegister(workspaceOperation)
+	metrics.Registry.MustRegister(workspaceOperation)
 }

pkg/controller/workspace/workspace_controller.go

@@ -1,18 +1,7 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspace
 
@@ -21,53 +10,45 @@ import (
 
 	"github.com/go-logr/logr"
 	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
-	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
-
-	"kubesphere.io/kubesphere/pkg/constants"
-	controllerutils "kubesphere.io/kubesphere/pkg/controller/utils/controller"
-	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
-	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
 )
 
 const (
-	controllerName = "workspace-controller"
+	controllerName = "workspace"
+	finalizer      = "finalizers.tenant.kubesphere.io"
 )
 
+var _ kscontroller.Controller = &Reconciler{}
+var _ reconcile.Reconciler = &Reconciler{}
+
 // Reconciler reconciles a Workspace object
 type Reconciler struct {
 	client.Client
-	Logger                  logr.Logger
-	Recorder                record.EventRecorder
-	MaxConcurrentReconciles int
+	logger   logr.Logger
+	recorder record.EventRecorder
 }
 
-func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if r.Client == nil {
-		r.Client = mgr.GetClient()
-	}
-	if r.Logger.GetSink() == nil {
-		r.Logger = ctrl.Log.WithName("controllers").WithName(controllerName)
-	}
-	if r.Recorder == nil {
-		r.Recorder = mgr.GetEventRecorderFor(controllerName)
-	}
-	if r.MaxConcurrentReconciles <= 0 {
-		r.MaxConcurrentReconciles = 1
-	}
+func (r *Reconciler) Name() string {
+	return controllerName
+}
+
+func (r *Reconciler) SetupWithManager(mgr *kscontroller.Manager) error {
+	r.Client = mgr.GetClient()
+	r.logger = mgr.GetLogger().WithName(controllerName)
+	r.recorder = mgr.GetEventRecorderFor(controllerName)
 	return ctrl.NewControllerManagedBy(mgr).
 		Named(controllerName).
-		WithOptions(controller.Options{
-			MaxConcurrentReconciles: r.MaxConcurrentReconciles,
-		}).
-		For(&tenantv1alpha1.Workspace{}).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 2}).
+		For(&tenantv1beta1.Workspace{}).
 		Complete(r)
 }
 
@@ -77,37 +58,32 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
 // +kubebuilder:rbac:groups=iam.kubesphere.io,resources=rolebases,verbs=get;list;watch
 // +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspaceroles,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=iam.kubesphere.io,resources=workspacerolebindings,verbs=get;list;watch;create;update;patch;delete
+
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	logger := r.Logger.WithValues("workspace", req.NamespacedName)
-	rootCtx := context.Background()
-	workspace := &tenantv1alpha1.Workspace{}
-	if err := r.Get(rootCtx, req.NamespacedName, workspace); err != nil {
+	logger := r.logger.WithValues("workspace", req.NamespacedName)
+	ctx = klog.NewContext(ctx, logger)
+	workspace := &tenantv1beta1.Workspace{}
+	if err := r.Get(ctx, req.NamespacedName, workspace); err != nil {
 		return ctrl.Result{}, client.IgnoreNotFound(err)
 	}
 
-	// name of your custom finalizer
-	finalizer := "finalizers.tenant.kubesphere.io"
-
 	if workspace.ObjectMeta.DeletionTimestamp.IsZero() {
 		// The object is not being deleted, so if it does not have our finalizer,
 		// then lets add the finalizer and update the object.
-		if !sliceutil.HasString(workspace.ObjectMeta.Finalizers, finalizer) {
-			workspace.ObjectMeta.Finalizers = append(workspace.ObjectMeta.Finalizers, finalizer)
-			if err := r.Update(rootCtx, workspace); err != nil {
+		if !controllerutil.ContainsFinalizer(workspace, finalizer) {
+			expected := workspace.DeepCopy()
+			controllerutil.AddFinalizer(expected, finalizer)
+			if err := r.Patch(ctx, expected, client.MergeFrom(workspace)); err != nil {
 				return ctrl.Result{}, err
 			}
 			workspaceOperation.WithLabelValues("create", workspace.Name).Inc()
 		}
 	} else {
 		// The object is being deleted
-		if sliceutil.HasString(workspace.ObjectMeta.Finalizers, finalizer) {
+		if controllerutil.ContainsFinalizer(workspace, finalizer) {
 			// remove our finalizer from the list and update it.
-			workspace.ObjectMeta.Finalizers = sliceutil.RemoveString(workspace.ObjectMeta.Finalizers, func(item string) bool {
-				return item == finalizer
-			})
-			logger.V(4).Info("update workspace")
-			if err := r.Update(rootCtx, workspace); err != nil {
-				logger.Error(err, "update workspace failed")
+			controllerutil.RemoveFinalizer(workspace, finalizer)
+			if err := r.Update(ctx, workspace); err != nil {
 				return ctrl.Result{}, err
 			}
 			workspaceOperation.WithLabelValues("delete", workspace.Name).Inc()
@@ -116,42 +92,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 		return ctrl.Result{}, nil
 	}
 
-	var namespaces corev1.NamespaceList
-	if err := r.List(rootCtx, &namespaces, client.MatchingLabels{tenantv1alpha1.WorkspaceLabel: req.Name}); err != nil {
-		logger.Error(err, "list namespaces failed")
-		return ctrl.Result{}, err
-	} else {
-		for _, namespace := range namespaces.Items {
-			// managed by kubefed-controller-manager
-			kubefedManaged := namespace.Labels[constants.KubefedManagedLabel] == "true"
-			if kubefedManaged {
-				continue
-			}
-			// managed by workspace
-			if err := r.bindWorkspace(rootCtx, logger, &namespace, workspace); err != nil {
-				return ctrl.Result{}, err
-			}
-		}
-	}
-
-	r.Recorder.Event(workspace, corev1.EventTypeNormal, controllerutils.SuccessSynced, controllerutils.MessageResourceSynced)
+	r.recorder.Event(workspace, corev1.EventTypeNormal, kscontroller.Synced, kscontroller.MessageResourceSynced)
 	return ctrl.Result{}, nil
 }
-
-func (r *Reconciler) bindWorkspace(ctx context.Context, logger logr.Logger, namespace *corev1.Namespace, workspace *tenantv1alpha1.Workspace) error {
-	// owner reference not match workspace label
-	if !metav1.IsControlledBy(namespace, workspace) {
-		namespace := namespace.DeepCopy()
-		namespace.OwnerReferences = k8sutil.RemoveWorkspaceOwnerReference(namespace.OwnerReferences)
-		if err := controllerutil.SetControllerReference(workspace, namespace, scheme.Scheme); err != nil {
-			logger.Error(err, "set controller reference failed")
-			return err
-		}
-		logger.V(4).Info("update namespace owner reference", "workspace", workspace.Name)
-		if err := r.Update(ctx, namespace); err != nil {
-			logger.Error(err, "update namespace failed")
-			return err
-		}
-	}
-	return nil
-}

pkg/controller/workspace/workspace_controller_suite_test.go

@@ -1,39 +1,30 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspace
 
 import (
+	"context"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
-	"github.com/onsi/gomega/gexec"
-	"k8s.io/client-go/kubernetes/scheme"
+	"kubesphere.io/kubesphere/pkg/controller/controllertest"
+
+	kscontroller "kubesphere.io/kubesphere/pkg/controller"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-
-	"kubesphere.io/kubesphere/pkg/apis"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"kubesphere.io/kubesphere/pkg/scheme"
 )
 
 // These tests use Ginkgo (BDD-style Go testing framework). Refer to
@@ -42,13 +33,15 @@ import (
 var k8sClient client.Client
 var k8sManager ctrl.Manager
 var testEnv *envtest.Environment
+var ctx context.Context
+var cancel context.CancelFunc
 
 func TestWorkspaceController(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Workspace Controller Test Suite")
 }
 
-var _ = BeforeSuite(func(done Done) {
+var _ = BeforeSuite(func() {
 	logf.SetLogger(klog.NewKlogr())
 
 	By("bootstrapping test environment")
@@ -58,8 +51,10 @@ var _ = BeforeSuite(func(done Done) {
 			UseExistingCluster: &t,
 		}
 	} else {
+		crdDirPaths, err := controllertest.LoadCrdPath()
+		Expect(err).ToNot(HaveOccurred())
 		testEnv = &envtest.Environment{
-			CRDDirectoryPaths:        []string{filepath.Join("..", "..", "..", "config", "ks-core", "crds")},
+			CRDDirectoryPaths:        crdDirPaths,
 			AttachControlPlaneOutput: false,
 		}
 	}
@@ -68,32 +63,32 @@ var _ = BeforeSuite(func(done Done) {
 	Expect(err).ToNot(HaveOccurred())
 	Expect(cfg).ToNot(BeNil())
 
-	err = apis.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
 	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
-		Scheme:             scheme.Scheme,
-		MetricsBindAddress: "0",
+		Scheme: scheme.Scheme,
+		Metrics: metricsserver.Options{
+			BindAddress: "0",
+		},
 	})
 	Expect(err).ToNot(HaveOccurred())
 
-	err = (&Reconciler{}).SetupWithManager(k8sManager)
+	err = (&Reconciler{}).SetupWithManager(&kscontroller.Manager{Manager: k8sManager})
 	Expect(err).ToNot(HaveOccurred())
 
+	ctx, cancel = context.WithCancel(context.Background())
+
 	go func() {
-		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		err = k8sManager.Start(ctx)
 		Expect(err).ToNot(HaveOccurred())
 	}()
 
 	k8sClient = k8sManager.GetClient()
 	Expect(k8sClient).ToNot(BeNil())
-
-	close(done)
-}, 60)
+})
 
 var _ = AfterSuite(func() {
+	cancel()
 	By("tearing down the test environment")
-	gexec.KillAndWait(5 * time.Second)
-	err := testEnv.Stop()
-	Expect(err).ToNot(HaveOccurred())
+	Eventually(func() error {
+		return testEnv.Stop()
+	}, 30*time.Second, 5*time.Second).ShouldNot(HaveOccurred())
 })

pkg/controller/workspace/workspace_controller_test.go

@@ -1,18 +1,7 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
+ * Please refer to the LICENSE file in the root directory of the project.
+ * https://github.com/kubesphere/kubesphere/blob/master/LICENSE
+ */
 
 package workspace
 
@@ -20,12 +9,11 @@ import (
 	"context"
 	"time"
 
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-
-	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
+	tenantv1beta1 "kubesphere.io/api/tenant/v1beta1"
 )
 
 var _ = Describe("Workspace", func() {
@@ -40,7 +28,7 @@ var _ = Describe("Workspace", func() {
 	Context("Workspace Controller", func() {
 		It("Should create successfully", func() {
 
-			workspace := &tenantv1alpha1.Workspace{
+			workspace := &tenantv1beta1.Workspace{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "workspace-test",
 				},
@@ -51,13 +39,13 @@ var _ = Describe("Workspace", func() {
 
 			By("Expecting to create workspace successfully")
 			Eventually(func() bool {
-				f := &tenantv1alpha1.Workspace{}
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
+				f := &tenantv1beta1.Workspace{}
+				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
 				return len(f.Finalizers) > 0
 			}, timeout, interval).Should(BeTrue())
 
 			// Update
-			updated := &tenantv1alpha1.Workspace{}
+			updated := &tenantv1beta1.Workspace{}
 			Expect(k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, updated)).Should(Succeed())
 			updated.Spec.Manager = "admin"
 			Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
@@ -65,21 +53,21 @@ var _ = Describe("Workspace", func() {
 			// List workspace role bindings
 			By("Expecting to update workspace successfully")
 			Eventually(func() bool {
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)
+				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, workspace)
 				return workspace.Spec.Manager == "admin"
 			}, timeout, interval).Should(BeTrue())
 
 			// Delete
 			By("Expecting to delete workspace successfully")
 			Eventually(func() error {
-				f := &tenantv1alpha1.Workspace{}
-				k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
+				f := &tenantv1beta1.Workspace{}
+				_ = k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
 				return k8sClient.Delete(context.Background(), f)
 			}, timeout, interval).Should(Succeed())
 
 			By("Expecting to delete workspace finish")
 			Eventually(func() error {
-				f := &tenantv1alpha1.Workspace{}
+				f := &tenantv1beta1.Workspace{}
 				return k8sClient.Get(context.Background(), types.NamespacedName{Name: workspace.Name}, f)
 			}, timeout, interval).ShouldNot(Succeed())
 		})