create admin ns for devops project

Signed-off-by: runzexia <runzexia@yunify.com>

cmd/controller-manager/app/controllers.go

@@ -86,6 +86,7 @@ func AddControllers(
 		kubesphereInformer.Devops().V1alpha1().S2iRuns())
 	devopsProjectController := devopsproject.NewController(client.Kubernetes(),
 		client.KubeSphere(), devopsClient,
+		informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
 		informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().DevOpsProjects(),
 	)
 

pkg/apis/devops/v1alpha3/devopsproject_types.go

@@ -42,6 +42,7 @@ type DevOpsProjectSpec struct {
 type DevOpsProjectStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
+	AdminNamespace string `json:"adminNamespace,omitempty"`
 }
 
 // +genclient

pkg/constants/constants.go

@@ -46,6 +46,7 @@ const (
 	WorkspacesManager              = "workspaces-manager"
 	DevopsOwner                    = "owner"
 	DevopsReporter                 = "reporter"
+	DevOpsProjectLabelKey          = "kubesphere.io/devopsproject"
 
 	UserNameHeader = "X-Token-Username"
 

pkg/controller/devopsproject/devopsproject_controller.go

@@ -4,19 +4,27 @@ import (
 	"fmt"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	corev1informer "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/kubernetes/scheme"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	corev1lister "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3"
+	"kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+	"kubesphere.io/kubesphere/pkg/constants"
 	devopsClient "kubesphere.io/kubesphere/pkg/simple/client/devops"
+	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
 	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
 	"net/http"
+	"reflect"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"time"
 
 	kubesphereclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
@@ -38,6 +46,9 @@ type Controller struct {
 	devOpsProjectLister devopslisters.DevOpsProjectLister
 	devOpsProjectSynced cache.InformerSynced
 
+	namespaceLister corev1lister.NamespaceLister
+	namespaceSynced cache.InformerSynced
+
 	workqueue workqueue.RateLimitingInterface
 
 	workerLoopPeriod time.Duration
@@ -48,6 +59,7 @@ type Controller struct {
 func NewController(client clientset.Interface,
 	kubesphereClient kubesphereclient.Interface,
 	devopsClinet devopsClient.Interface,
+	namespaceInformer corev1informer.NamespaceInformer,
 	devopsInformer devopsinformers.DevOpsProjectInformer) *Controller {
 
 	broadcaster := record.NewBroadcaster()
@@ -64,6 +76,8 @@ func NewController(client clientset.Interface,
 		workqueue:           workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "devopsproject"),
 		devOpsProjectLister: devopsInformer.Lister(),
 		devOpsProjectSynced: devopsInformer.Informer().HasSynced,
+		namespaceLister:     namespaceInformer.Lister(),
+		namespaceSynced:     namespaceInformer.Informer().HasSynced,
 		workerLoopPeriod:    time.Second,
 	}
 
@@ -175,17 +189,13 @@ func (c *Controller) syncHandler(key string) error {
 		klog.Error(err, fmt.Sprintf("could not get devopsproject %s ", key))
 		return err
 	}
+	copyProject := project.DeepCopy()
 	// DeletionTimestamp.IsZero() means DevOps project has not been deleted.
 	if project.ObjectMeta.DeletionTimestamp.IsZero() {
 		// Use Finalizers to sync DevOps status when DevOps project was deleted
 		// https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers
 		if !sliceutil.HasString(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) {
-			project.ObjectMeta.Finalizers = append(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName)
+			copyProject.ObjectMeta.Finalizers = append(copyProject.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName)
-			_, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(project)
-			if err != nil {
-				klog.Error(err, fmt.Sprintf("failed to update project %s ", key))
-				return err
-			}
 		}
 		// Check project exists, otherwise we will create it.
 		_, err := c.devopsClient.GetDevOpsProject(key)
@@ -199,6 +209,88 @@ func (c *Controller) syncHandler(key string) error {
 				return err
 			}
 		}
+		if project.Status.AdminNamespace != "" {
+			ns, err := c.namespaceLister.Get(project.Status.AdminNamespace)
+			if err != nil && !errors.IsNotFound(err) {
+				klog.Error(err, fmt.Sprintf("faild to get namespace"))
+				return err
+			} else if errors.IsNotFound(err) {
+				// if admin ns is not found, clean project status, rerun reconcile
+				copyProject.Status.AdminNamespace = ""
+				_, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(copyProject)
+				if err != nil {
+					klog.Error(err, fmt.Sprintf("failed to update project %s ", key))
+					return err
+				}
+				c.enqueueDevOpsProject(key)
+				return nil
+			}
+			// If ns exists, but the associated attributes with the project are not set correctly,
+			// then reset the associated attributes
+			if k8sutil.IsControlledBy(ns.OwnerReferences,
+				devopsv1alpha3.ResourceKindDevOpsProject, project.Name) &&
+				ns.Labels[constants.DevOpsProjectLabelKey] == project.Name {
+			} else {
+				copyNs := ns.DeepCopy()
+				err := controllerutil.SetControllerReference(copyProject, copyNs, scheme.Scheme)
+				if err != nil {
+					klog.Error(err, fmt.Sprintf("failed to set ownerreference %s ", key))
+					return err
+				}
+				copyNs.Labels[constants.DevOpsProjectLabelKey] = project.Name
+				_, err = c.client.CoreV1().Namespaces().Update(copyNs)
+				if err != nil {
+					klog.Error(err, fmt.Sprintf("failed to update ns %s ", key))
+					return err
+				}
+			}
+		} else {
+			// list ns by devops project
+			namespaces, err := c.namespaceLister.List(
+				labels.SelectorFromSet(labels.Set{constants.DevOpsProjectLabelKey: project.Name}))
+			if err != nil {
+				klog.Error(err, fmt.Sprintf("failed to list ns %s ", key))
+				return err
+			}
+			// if there is no ns, generate new one
+			if len(namespaces) == 0 {
+				ns := c.generateNewNamespace(project)
+				ns, err := c.client.CoreV1().Namespaces().Create(ns)
+				if err != nil {
+					klog.Error(err, fmt.Sprintf("failed to create ns %s ", key))
+					return err
+				}
+				copyProject.Status.AdminNamespace = ns.Name
+			} else if len(namespaces) != 0 {
+				ns := namespaces[0]
+				// reset ownerReferences
+				if !k8sutil.IsControlledBy(ns.OwnerReferences,
+					devopsv1alpha3.ResourceKindDevOpsProject, project.Name) {
+					copyNs := ns.DeepCopy()
+					err := controllerutil.SetControllerReference(copyProject, copyNs, scheme.Scheme)
+					if err != nil {
+						klog.Error(err, fmt.Sprintf("failed to set ownerreference %s ", key))
+						return err
+					}
+					copyNs.Labels[constants.DevOpsProjectLabelKey] = project.Name
+					_, err = c.client.CoreV1().Namespaces().Update(copyNs)
+					if err != nil {
+						klog.Error(err, fmt.Sprintf("failed to update ns %s ", key))
+						return err
+					}
+				}
+				copyProject.Status.AdminNamespace = ns.Name
+			}
+		}
+
+		if !reflect.DeepEqual(copyProject, project) {
+			_, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(copyProject)
+			if err != nil {
+				klog.Error(err, fmt.Sprintf("failed to update ns %s ", key))
+				return err
+			}
+		}
+
 	} else {
 		// Finalizers processing logic
 		if sliceutil.HasString(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) {
@@ -237,3 +329,18 @@ func (c *Controller) deleteDevOpsProjectInDevOps(project *devopsv1alpha3.DevOpsP
 
 	return nil
 }
+
+func (c *Controller) generateNewNamespace(project *devopsv1alpha3.DevOpsProject) *v1.Namespace {
+	ns := &v1.Namespace{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Namespace",
+			APIVersion: v1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: project.Name,
+			Labels:       map[string]string{constants.DevOpsProjectLabelKey: project.Name},
+		},
+	}
+	controllerutil.SetControllerReference(project, ns, scheme.Scheme)
+	return ns
+}

pkg/controller/devopsproject/devopsproject_controller_test.go

@@ -1,7 +1,9 @@
 package devopsproject
 
 import (
+	v1 "k8s.io/api/core/v1"
 	devopsprojects "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3"
+	"kubesphere.io/kubesphere/pkg/constants"
 	fakeDevOps "kubesphere.io/kubesphere/pkg/simple/client/devops/fake"
 	"reflect"
 	"testing"
@@ -11,6 +13,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/diff"
+	kubeinformers "k8s.io/client-go/informers"
 	k8sfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
@@ -32,7 +35,11 @@ type fixture struct {
 	kubeclient *k8sfake.Clientset
 	// Objects to put in the store.
 	devopsProjectLister []*devops.DevOpsProject
+	namespaceLister     []*v1.Namespace
 	actions             []core.Action
+	kubeactions         []core.Action
+
+	kubeobjects []runtime.Object
 	// Objects from here preloaded into NewSimpleFake.
 	objects []runtime.Object
 	// Objects from here preloaded into devops
@@ -47,14 +54,52 @@ func newFixture(t *testing.T) *fixture {
 	return f
 }
 
-func newDevOpsProject(name string) *devopsprojects.DevOpsProject {
+func newDevOpsProject(name string, nsName string, withFinalizers bool, withStatus bool) *devopsprojects.DevOpsProject {
-	return &devopsprojects.DevOpsProject{
+	project := &devopsprojects.DevOpsProject{
 		TypeMeta: metav1.TypeMeta{APIVersion: devopsprojects.SchemeGroupVersion.String()},
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
 		},
 	}
+	if withFinalizers {
+		project.Finalizers = []string{devopsprojects.DevOpsProjectFinalizerName}
 	}
+	if withStatus {
+		project.Status = devops.DevOpsProjectStatus{AdminNamespace: nsName}
+	}
+	return project
+}
+
+func newNamespace(name string, projectName string, useGenerateName, withOwnerReference bool) *v1.Namespace {
+	ns := &v1.Namespace{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Namespace",
+			APIVersion: v1.SchemeGroupVersion.String(),
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:   name,
+			Labels: map[string]string{constants.DevOpsProjectLabelKey: projectName},
+		},
+	}
+	if useGenerateName {
+		ns.ObjectMeta.Name = ""
+		ns.ObjectMeta.GenerateName = projectName
+	}
+	if withOwnerReference {
+		TRUE := true
+		ns.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
+			{
+				APIVersion:         devops.SchemeGroupVersion.String(),
+				Kind:               devops.ResourceKindDevOpsProject,
+				Name:               projectName,
+				BlockOwnerDeletion: &TRUE,
+				Controller:         &TRUE,
+			},
+		}
+	}
+	return ns
+}
+
 func newDeletingDevOpsProject(name string) *devopsprojects.DevOpsProject {
 	now := metav1.Now()
 	return &devopsprojects.DevOpsProject{
@@ -67,14 +112,16 @@ func newDeletingDevOpsProject(name string) *devopsprojects.DevOpsProject {
 	}
 }
 
-func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, *fakeDevOps.Devops) {
+func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory, *fakeDevOps.Devops) {
 	f.client = fake.NewSimpleClientset(f.objects...)
-	f.kubeclient = k8sfake.NewSimpleClientset()
+	f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)
 
 	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
+	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
 	dI := fakeDevOps.New(f.initDevOpsProject...)
 
-	c := NewController(f.kubeclient, f.client, dI, i.Devops().V1alpha3().DevOpsProjects())
+	c := NewController(f.kubeclient, f.client, dI, k8sI.Core().V1().Namespaces(),
+		i.Devops().V1alpha3().DevOpsProjects())
 
 	c.devOpsProjectSynced = alwaysReady
 	c.eventRecorder = &record.FakeRecorder{}
@@ -83,7 +130,11 @@ func (f *fixture) newController() (*Controller, informers.SharedInformerFactory,
 		i.Devops().V1alpha3().DevOpsProjects().Informer().GetIndexer().Add(f)
 	}
 
-	return c, i, dI
+	for _, d := range f.namespaceLister {
+		k8sI.Core().V1().Namespaces().Informer().GetIndexer().Add(d)
+	}
+
+	return c, i, k8sI, dI
 }
 
 func (f *fixture) run(fooName string) {
@@ -95,11 +146,12 @@ func (f *fixture) runExpectError(fooName string) {
 }
 
 func (f *fixture) runController(projectName string, startInformers bool, expectError bool) {
-	c, i, dI := f.newController()
+	c, i, k8sI, dI := f.newController()
 	if startInformers {
 		stopCh := make(chan struct{})
 		defer close(stopCh)
 		i.Start(stopCh)
+		k8sI.Start(stopCh)
 	}
 
 	err := c.syncHandler(projectName)
@@ -119,6 +171,20 @@ func (f *fixture) runController(projectName string, startInformers bool, expectE
 		expectedAction := f.actions[i]
 		checkAction(expectedAction, action, f.t)
 	}
+	k8sActions := filterInformerActions(f.kubeclient.Actions())
+	for i, action := range k8sActions {
+		if len(f.kubeactions) < i+1 {
+			f.t.Errorf("%d unexpected actions: %+v", len(k8sActions)-len(f.kubeactions), k8sActions[i:])
+			break
+		}
+
+		expectedAction := f.kubeactions[i]
+		checkAction(expectedAction, action, f.t)
+	}
+
+	if len(f.kubeactions) > len(k8sActions) {
+		f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
+	}
 
 	if len(f.actions) > len(actions) {
 		f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):])
@@ -183,7 +249,9 @@ func filterInformerActions(actions []core.Action) []core.Action {
 	for _, action := range actions {
 		if len(action.GetNamespace()) == 0 &&
 			(action.Matches("list", devopsprojects.ResourcePluralDevOpsProject) ||
-				action.Matches("watch", devopsprojects.ResourcePluralDevOpsProject)) {
+				action.Matches("watch", devopsprojects.ResourcePluralDevOpsProject) ||
+				action.Matches("list", "namespaces") ||
+				action.Matches("watch", "namespaces")) {
 			continue
 		}
 		ret = append(ret, action)
@@ -198,6 +266,22 @@ func (f *fixture) expectUpdateDevOpsProjectAction(p *devopsprojects.DevOpsProjec
 	f.actions = append(f.actions, action)
 }
 
+func (f *fixture) expectUpdateNamespaceAction(p *v1.Namespace) {
+	action := core.NewUpdateAction(schema.GroupVersionResource{
+		Version:  "v1",
+		Resource: "namespaces",
+	}, p.Namespace, p)
+	f.kubeactions = append(f.kubeactions, action)
+}
+
+func (f *fixture) expectCreateNamespaceAction(p *v1.Namespace) {
+	action := core.NewCreateAction(schema.GroupVersionResource{
+		Version:  "v1",
+		Resource: "namespaces",
+	}, p.Namespace, p)
+	f.kubeactions = append(f.kubeactions, action)
+}
+
 func getKey(p *devopsprojects.DevOpsProject, t *testing.T) string {
 	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(p)
 	if err != nil {
@@ -209,26 +293,88 @@ func getKey(p *devopsprojects.DevOpsProject, t *testing.T) string {
 
 func TestDoNothing(t *testing.T) {
 	f := newFixture(t)
-	project := newDevOpsProject("test")
+	nsName := "test-123"
+	projectName := "test"
+	project := newDevOpsProject(projectName, nsName, true, true)
+	ns := newNamespace(nsName, projectName, false, true)
 
 	f.devopsProjectLister = append(f.devopsProjectLister, project)
+	f.namespaceLister = append(f.namespaceLister, ns)
 	f.objects = append(f.objects, project)
 	f.initDevOpsProject = []string{project.Name}
 	f.expectDevOpsProject = []string{project.Name}
 
-	f.expectUpdateDevOpsProjectAction(project)
+	f.run(getKey(project, t))
+}
+
+func TestUpdateProjectFinalizers(t *testing.T) {
+	f := newFixture(t)
+	nsName := "test-123"
+	projectName := "test"
+	project := newDevOpsProject(projectName, nsName, false, true)
+	ns := newNamespace(nsName, projectName, false, true)
+
+	f.devopsProjectLister = append(f.devopsProjectLister, project)
+	f.namespaceLister = append(f.namespaceLister, ns)
+	f.objects = append(f.objects, project)
+	f.kubeobjects = append(f.kubeobjects, ns)
+	f.initDevOpsProject = []string{project.Name}
+	f.expectDevOpsProject = []string{project.Name}
+	expectUpdateProject := project.DeepCopy()
+	expectUpdateProject.Finalizers = []string{devops.DevOpsProjectFinalizerName}
+	f.expectUpdateDevOpsProjectAction(expectUpdateProject)
+	f.run(getKey(project, t))
+}
+
+func TestUpdateProjectStatus(t *testing.T) {
+	f := newFixture(t)
+	nsName := "test-123"
+	projectName := "test"
+	project := newDevOpsProject(projectName, nsName, true, false)
+	ns := newNamespace(nsName, projectName, false, true)
+
+	f.devopsProjectLister = append(f.devopsProjectLister, project)
+	f.namespaceLister = append(f.namespaceLister, ns)
+	f.objects = append(f.objects, project)
+	f.kubeobjects = append(f.kubeobjects, ns)
+	f.initDevOpsProject = []string{project.Name}
+	f.expectDevOpsProject = []string{project.Name}
+	expectUpdateProject := project.DeepCopy()
+	expectUpdateProject.Status.AdminNamespace = nsName
+	f.expectUpdateDevOpsProjectAction(expectUpdateProject)
+	f.run(getKey(project, t))
+}
+
+func TestUpdateNsOwnerReference(t *testing.T) {
+	f := newFixture(t)
+	nsName := "test-123"
+	projectName := "test"
+	project := newDevOpsProject(projectName, nsName, true, true)
+	ns := newNamespace(nsName, projectName, false, false)
+
+	f.devopsProjectLister = append(f.devopsProjectLister, project)
+	f.namespaceLister = append(f.namespaceLister, ns)
+	f.objects = append(f.objects, project)
+	f.kubeobjects = append(f.kubeobjects, ns)
+	f.initDevOpsProject = []string{project.Name}
+	f.expectDevOpsProject = []string{project.Name}
+	expectUpdateNs := newNamespace(nsName, projectName, false, true)
+
+	f.expectUpdateNamespaceAction(expectUpdateNs)
 	f.run(getKey(project, t))
 }
 
 func TestCreateDevOpsProjects(t *testing.T) {
 	f := newFixture(t)
-	project := newDevOpsProject("test")
+	project := newDevOpsProject("test", "", true, false)
-
+	ns := newNamespace("test-123", "test", true, true)
 	f.devopsProjectLister = append(f.devopsProjectLister, project)
 	f.objects = append(f.objects, project)
 	f.expectDevOpsProject = []string{project.Name}
 
-	f.expectUpdateDevOpsProjectAction(project)
+	// because generateName not work in fakeClient, so DevOpsProject would not be update
+	// f.expectUpdateDevOpsProjectAction(project)
+	f.expectCreateNamespaceAction(ns)
 	f.run(getKey(project, t))
 }
 
@@ -243,6 +389,7 @@ func TestDeleteDevOpsProjects(t *testing.T) {
 	f.expectUpdateDevOpsProjectAction(project)
 	f.run(getKey(project, t))
 }
+
 func TestDeleteDevOpsProjectsWithNull(t *testing.T) {
 	f := newFixture(t)
 	project := newDeletingDevOpsProject("test")