From e06a1a8335cdaae06175d7dc8a188f917b507356 Mon Sep 17 00:00:00 2001 From: runzexia Date: Mon, 23 Mar 2020 15:56:37 +0800 Subject: [PATCH] create admin ns for devops project Signed-off-by: runzexia --- cmd/controller-manager/app/controllers.go | 1 + .../devops/v1alpha3/devopsproject_types.go | 1 + pkg/constants/constants.go | 1 + .../devopsproject/devopsproject_controller.go | 123 ++++++++++++- .../devopsproject_controller_test.go | 173 ++++++++++++++++-- 5 files changed, 278 insertions(+), 21 deletions(-) diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index b14748a5a..41d3b1d22 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -86,6 +86,7 @@ func AddControllers( kubesphereInformer.Devops().V1alpha1().S2iRuns()) devopsProjectController := devopsproject.NewController(client.Kubernetes(), client.KubeSphere(), devopsClient, + informerFactory.KubernetesSharedInformerFactory().Core().V1().Namespaces(), informerFactory.KubeSphereSharedInformerFactory().Devops().V1alpha3().DevOpsProjects(), ) diff --git a/pkg/apis/devops/v1alpha3/devopsproject_types.go b/pkg/apis/devops/v1alpha3/devopsproject_types.go index 867abf2cf..3a856251f 100644 --- a/pkg/apis/devops/v1alpha3/devopsproject_types.go +++ b/pkg/apis/devops/v1alpha3/devopsproject_types.go @@ -42,6 +42,7 @@ type DevOpsProjectSpec struct { type DevOpsProjectStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster // Important: Run "make" to regenerate code after modifying this file + AdminNamespace string `json:"adminNamespace,omitempty"` } // +genclient diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index a72338ba7..2f45dd696 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -46,6 +46,7 @@ const ( WorkspacesManager = "workspaces-manager" DevopsOwner = "owner" DevopsReporter = "reporter" + DevOpsProjectLabelKey = "kubesphere.io/devopsproject" UserNameHeader = "X-Token-Username" diff --git a/pkg/controller/devopsproject/devopsproject_controller.go b/pkg/controller/devopsproject/devopsproject_controller.go index 0ced43c6b..12bf5bd8b 100644 --- a/pkg/controller/devopsproject/devopsproject_controller.go +++ b/pkg/controller/devopsproject/devopsproject_controller.go @@ -4,19 +4,27 @@ import ( "fmt" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" + corev1informer "k8s.io/client-go/informers/core/v1" clientset "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" + corev1lister "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" "k8s.io/klog" devopsv1alpha3 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3" + "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" + "kubesphere.io/kubesphere/pkg/constants" devopsClient "kubesphere.io/kubesphere/pkg/simple/client/devops" + "kubesphere.io/kubesphere/pkg/utils/k8sutil" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "net/http" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "time" kubesphereclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned" @@ -25,7 +33,7 @@ import ( ) /** -DevOps project controller is used to maintain the state of the DevOps project. + DevOps project controller is used to maintain the state of the DevOps project. */ type Controller struct { @@ -38,6 +46,9 @@ type Controller struct { devOpsProjectLister devopslisters.DevOpsProjectLister devOpsProjectSynced cache.InformerSynced + namespaceLister corev1lister.NamespaceLister + namespaceSynced cache.InformerSynced + workqueue workqueue.RateLimitingInterface workerLoopPeriod time.Duration @@ -48,6 +59,7 @@ type Controller struct { func NewController(client clientset.Interface, kubesphereClient kubesphereclient.Interface, devopsClinet devopsClient.Interface, + namespaceInformer corev1informer.NamespaceInformer, devopsInformer devopsinformers.DevOpsProjectInformer) *Controller { broadcaster := record.NewBroadcaster() @@ -64,6 +76,8 @@ func NewController(client clientset.Interface, workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "devopsproject"), devOpsProjectLister: devopsInformer.Lister(), devOpsProjectSynced: devopsInformer.Informer().HasSynced, + namespaceLister: namespaceInformer.Lister(), + namespaceSynced: namespaceInformer.Informer().HasSynced, workerLoopPeriod: time.Second, } @@ -175,17 +189,13 @@ func (c *Controller) syncHandler(key string) error { klog.Error(err, fmt.Sprintf("could not get devopsproject %s ", key)) return err } + copyProject := project.DeepCopy() // DeletionTimestamp.IsZero() means DevOps project has not been deleted. if project.ObjectMeta.DeletionTimestamp.IsZero() { // Use Finalizers to sync DevOps status when DevOps project was deleted // https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers if !sliceutil.HasString(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) { - project.ObjectMeta.Finalizers = append(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) - _, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(project) - if err != nil { - klog.Error(err, fmt.Sprintf("failed to update project %s ", key)) - return err - } + copyProject.ObjectMeta.Finalizers = append(copyProject.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) } // Check project exists, otherwise we will create it. _, err := c.devopsClient.GetDevOpsProject(key) @@ -199,6 +209,88 @@ func (c *Controller) syncHandler(key string) error { return err } } + if project.Status.AdminNamespace != "" { + ns, err := c.namespaceLister.Get(project.Status.AdminNamespace) + if err != nil && !errors.IsNotFound(err) { + klog.Error(err, fmt.Sprintf("faild to get namespace")) + return err + } else if errors.IsNotFound(err) { + // if admin ns is not found, clean project status, rerun reconcile + copyProject.Status.AdminNamespace = "" + _, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(copyProject) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to update project %s ", key)) + return err + } + c.enqueueDevOpsProject(key) + return nil + } + // If ns exists, but the associated attributes with the project are not set correctly, + // then reset the associated attributes + if k8sutil.IsControlledBy(ns.OwnerReferences, + devopsv1alpha3.ResourceKindDevOpsProject, project.Name) && + ns.Labels[constants.DevOpsProjectLabelKey] == project.Name { + } else { + copyNs := ns.DeepCopy() + err := controllerutil.SetControllerReference(copyProject, copyNs, scheme.Scheme) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to set ownerreference %s ", key)) + return err + } + copyNs.Labels[constants.DevOpsProjectLabelKey] = project.Name + _, err = c.client.CoreV1().Namespaces().Update(copyNs) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to update ns %s ", key)) + return err + } + } + } else { + // list ns by devops project + namespaces, err := c.namespaceLister.List( + labels.SelectorFromSet(labels.Set{constants.DevOpsProjectLabelKey: project.Name})) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to list ns %s ", key)) + return err + } + // if there is no ns, generate new one + if len(namespaces) == 0 { + ns := c.generateNewNamespace(project) + ns, err := c.client.CoreV1().Namespaces().Create(ns) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to create ns %s ", key)) + return err + } + copyProject.Status.AdminNamespace = ns.Name + } else if len(namespaces) != 0 { + ns := namespaces[0] + // reset ownerReferences + if !k8sutil.IsControlledBy(ns.OwnerReferences, + devopsv1alpha3.ResourceKindDevOpsProject, project.Name) { + copyNs := ns.DeepCopy() + err := controllerutil.SetControllerReference(copyProject, copyNs, scheme.Scheme) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to set ownerreference %s ", key)) + return err + } + copyNs.Labels[constants.DevOpsProjectLabelKey] = project.Name + _, err = c.client.CoreV1().Namespaces().Update(copyNs) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to update ns %s ", key)) + return err + } + } + copyProject.Status.AdminNamespace = ns.Name + } + } + + if !reflect.DeepEqual(copyProject, project) { + _, err := c.kubesphereClient.DevopsV1alpha3().DevOpsProjects().Update(copyProject) + if err != nil { + klog.Error(err, fmt.Sprintf("failed to update ns %s ", key)) + return err + } + } + } else { // Finalizers processing logic if sliceutil.HasString(project.ObjectMeta.Finalizers, devopsv1alpha3.DevOpsProjectFinalizerName) { @@ -237,3 +329,18 @@ func (c *Controller) deleteDevOpsProjectInDevOps(project *devopsv1alpha3.DevOpsP return nil } + +func (c *Controller) generateNewNamespace(project *devopsv1alpha3.DevOpsProject) *v1.Namespace { + ns := &v1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + GenerateName: project.Name, + Labels: map[string]string{constants.DevOpsProjectLabelKey: project.Name}, + }, + } + controllerutil.SetControllerReference(project, ns, scheme.Scheme) + return ns +} diff --git a/pkg/controller/devopsproject/devopsproject_controller_test.go b/pkg/controller/devopsproject/devopsproject_controller_test.go index bcfcc6973..fd933e987 100644 --- a/pkg/controller/devopsproject/devopsproject_controller_test.go +++ b/pkg/controller/devopsproject/devopsproject_controller_test.go @@ -1,7 +1,9 @@ package devopsproject import ( + v1 "k8s.io/api/core/v1" devopsprojects "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3" + "kubesphere.io/kubesphere/pkg/constants" fakeDevOps "kubesphere.io/kubesphere/pkg/simple/client/devops/fake" "reflect" "testing" @@ -11,6 +13,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/diff" + kubeinformers "k8s.io/client-go/informers" k8sfake "k8s.io/client-go/kubernetes/fake" core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" @@ -32,7 +35,11 @@ type fixture struct { kubeclient *k8sfake.Clientset // Objects to put in the store. devopsProjectLister []*devops.DevOpsProject + namespaceLister []*v1.Namespace actions []core.Action + kubeactions []core.Action + + kubeobjects []runtime.Object // Objects from here preloaded into NewSimpleFake. objects []runtime.Object // Objects from here preloaded into devops @@ -47,14 +54,52 @@ func newFixture(t *testing.T) *fixture { return f } -func newDevOpsProject(name string) *devopsprojects.DevOpsProject { - return &devopsprojects.DevOpsProject{ +func newDevOpsProject(name string, nsName string, withFinalizers bool, withStatus bool) *devopsprojects.DevOpsProject { + project := &devopsprojects.DevOpsProject{ TypeMeta: metav1.TypeMeta{APIVersion: devopsprojects.SchemeGroupVersion.String()}, ObjectMeta: metav1.ObjectMeta{ Name: name, }, } + if withFinalizers { + project.Finalizers = []string{devopsprojects.DevOpsProjectFinalizerName} + } + if withStatus { + project.Status = devops.DevOpsProjectStatus{AdminNamespace: nsName} + } + return project } + +func newNamespace(name string, projectName string, useGenerateName, withOwnerReference bool) *v1.Namespace { + ns := &v1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: v1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{constants.DevOpsProjectLabelKey: projectName}, + }, + } + if useGenerateName { + ns.ObjectMeta.Name = "" + ns.ObjectMeta.GenerateName = projectName + } + if withOwnerReference { + TRUE := true + ns.ObjectMeta.OwnerReferences = []metav1.OwnerReference{ + { + APIVersion: devops.SchemeGroupVersion.String(), + Kind: devops.ResourceKindDevOpsProject, + Name: projectName, + BlockOwnerDeletion: &TRUE, + Controller: &TRUE, + }, + } + } + return ns +} + func newDeletingDevOpsProject(name string) *devopsprojects.DevOpsProject { now := metav1.Now() return &devopsprojects.DevOpsProject{ @@ -67,14 +112,16 @@ func newDeletingDevOpsProject(name string) *devopsprojects.DevOpsProject { } } -func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, *fakeDevOps.Devops) { +func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory, *fakeDevOps.Devops) { f.client = fake.NewSimpleClientset(f.objects...) - f.kubeclient = k8sfake.NewSimpleClientset() + f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...) i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc()) + k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc()) dI := fakeDevOps.New(f.initDevOpsProject...) - c := NewController(f.kubeclient, f.client, dI, i.Devops().V1alpha3().DevOpsProjects()) + c := NewController(f.kubeclient, f.client, dI, k8sI.Core().V1().Namespaces(), + i.Devops().V1alpha3().DevOpsProjects()) c.devOpsProjectSynced = alwaysReady c.eventRecorder = &record.FakeRecorder{} @@ -83,7 +130,11 @@ func (f *fixture) newController() (*Controller, informers.SharedInformerFactory, i.Devops().V1alpha3().DevOpsProjects().Informer().GetIndexer().Add(f) } - return c, i, dI + for _, d := range f.namespaceLister { + k8sI.Core().V1().Namespaces().Informer().GetIndexer().Add(d) + } + + return c, i, k8sI, dI } func (f *fixture) run(fooName string) { @@ -95,11 +146,12 @@ func (f *fixture) runExpectError(fooName string) { } func (f *fixture) runController(projectName string, startInformers bool, expectError bool) { - c, i, dI := f.newController() + c, i, k8sI, dI := f.newController() if startInformers { stopCh := make(chan struct{}) defer close(stopCh) i.Start(stopCh) + k8sI.Start(stopCh) } err := c.syncHandler(projectName) @@ -119,6 +171,20 @@ func (f *fixture) runController(projectName string, startInformers bool, expectE expectedAction := f.actions[i] checkAction(expectedAction, action, f.t) } + k8sActions := filterInformerActions(f.kubeclient.Actions()) + for i, action := range k8sActions { + if len(f.kubeactions) < i+1 { + f.t.Errorf("%d unexpected actions: %+v", len(k8sActions)-len(f.kubeactions), k8sActions[i:]) + break + } + + expectedAction := f.kubeactions[i] + checkAction(expectedAction, action, f.t) + } + + if len(f.kubeactions) > len(k8sActions) { + f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):]) + } if len(f.actions) > len(actions) { f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):]) @@ -183,7 +249,9 @@ func filterInformerActions(actions []core.Action) []core.Action { for _, action := range actions { if len(action.GetNamespace()) == 0 && (action.Matches("list", devopsprojects.ResourcePluralDevOpsProject) || - action.Matches("watch", devopsprojects.ResourcePluralDevOpsProject)) { + action.Matches("watch", devopsprojects.ResourcePluralDevOpsProject) || + action.Matches("list", "namespaces") || + action.Matches("watch", "namespaces")) { continue } ret = append(ret, action) @@ -198,6 +266,22 @@ func (f *fixture) expectUpdateDevOpsProjectAction(p *devopsprojects.DevOpsProjec f.actions = append(f.actions, action) } +func (f *fixture) expectUpdateNamespaceAction(p *v1.Namespace) { + action := core.NewUpdateAction(schema.GroupVersionResource{ + Version: "v1", + Resource: "namespaces", + }, p.Namespace, p) + f.kubeactions = append(f.kubeactions, action) +} + +func (f *fixture) expectCreateNamespaceAction(p *v1.Namespace) { + action := core.NewCreateAction(schema.GroupVersionResource{ + Version: "v1", + Resource: "namespaces", + }, p.Namespace, p) + f.kubeactions = append(f.kubeactions, action) +} + func getKey(p *devopsprojects.DevOpsProject, t *testing.T) string { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(p) if err != nil { @@ -209,26 +293,88 @@ func getKey(p *devopsprojects.DevOpsProject, t *testing.T) string { func TestDoNothing(t *testing.T) { f := newFixture(t) - project := newDevOpsProject("test") + nsName := "test-123" + projectName := "test" + project := newDevOpsProject(projectName, nsName, true, true) + ns := newNamespace(nsName, projectName, false, true) f.devopsProjectLister = append(f.devopsProjectLister, project) + f.namespaceLister = append(f.namespaceLister, ns) f.objects = append(f.objects, project) f.initDevOpsProject = []string{project.Name} f.expectDevOpsProject = []string{project.Name} - f.expectUpdateDevOpsProjectAction(project) + f.run(getKey(project, t)) +} + +func TestUpdateProjectFinalizers(t *testing.T) { + f := newFixture(t) + nsName := "test-123" + projectName := "test" + project := newDevOpsProject(projectName, nsName, false, true) + ns := newNamespace(nsName, projectName, false, true) + + f.devopsProjectLister = append(f.devopsProjectLister, project) + f.namespaceLister = append(f.namespaceLister, ns) + f.objects = append(f.objects, project) + f.kubeobjects = append(f.kubeobjects, ns) + f.initDevOpsProject = []string{project.Name} + f.expectDevOpsProject = []string{project.Name} + expectUpdateProject := project.DeepCopy() + expectUpdateProject.Finalizers = []string{devops.DevOpsProjectFinalizerName} + f.expectUpdateDevOpsProjectAction(expectUpdateProject) + f.run(getKey(project, t)) +} + +func TestUpdateProjectStatus(t *testing.T) { + f := newFixture(t) + nsName := "test-123" + projectName := "test" + project := newDevOpsProject(projectName, nsName, true, false) + ns := newNamespace(nsName, projectName, false, true) + + f.devopsProjectLister = append(f.devopsProjectLister, project) + f.namespaceLister = append(f.namespaceLister, ns) + f.objects = append(f.objects, project) + f.kubeobjects = append(f.kubeobjects, ns) + f.initDevOpsProject = []string{project.Name} + f.expectDevOpsProject = []string{project.Name} + expectUpdateProject := project.DeepCopy() + expectUpdateProject.Status.AdminNamespace = nsName + f.expectUpdateDevOpsProjectAction(expectUpdateProject) + f.run(getKey(project, t)) +} + +func TestUpdateNsOwnerReference(t *testing.T) { + f := newFixture(t) + nsName := "test-123" + projectName := "test" + project := newDevOpsProject(projectName, nsName, true, true) + ns := newNamespace(nsName, projectName, false, false) + + f.devopsProjectLister = append(f.devopsProjectLister, project) + f.namespaceLister = append(f.namespaceLister, ns) + f.objects = append(f.objects, project) + f.kubeobjects = append(f.kubeobjects, ns) + f.initDevOpsProject = []string{project.Name} + f.expectDevOpsProject = []string{project.Name} + expectUpdateNs := newNamespace(nsName, projectName, false, true) + + f.expectUpdateNamespaceAction(expectUpdateNs) f.run(getKey(project, t)) } func TestCreateDevOpsProjects(t *testing.T) { f := newFixture(t) - project := newDevOpsProject("test") - + project := newDevOpsProject("test", "", true, false) + ns := newNamespace("test-123", "test", true, true) f.devopsProjectLister = append(f.devopsProjectLister, project) f.objects = append(f.objects, project) f.expectDevOpsProject = []string{project.Name} - f.expectUpdateDevOpsProjectAction(project) + // because generateName not work in fakeClient, so DevOpsProject would not be update + // f.expectUpdateDevOpsProjectAction(project) + f.expectCreateNamespaceAction(ns) f.run(getKey(project, t)) } @@ -243,6 +389,7 @@ func TestDeleteDevOpsProjects(t *testing.T) { f.expectUpdateDevOpsProjectAction(project) f.run(getKey(project, t)) } + func TestDeleteDevOpsProjectsWithNull(t *testing.T) { f := newFixture(t) project := newDeletingDevOpsProject("test")