add devops credential controller

Signed-off-by: runzexia <runzexia@yunify.com>
This commit is contained in:
runzexia
2020-03-26 14:53:33 +08:00
parent e8b9d9cdf3
commit af3b87ddca
13 changed files with 961 additions and 641 deletions

View File

@@ -0,0 +1,284 @@
package devopscredential
import (
"fmt"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
corev1informer "k8s.io/client-go/informers/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
corev1lister "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
devopsv1alpha3 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3"
kubesphereclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
"kubesphere.io/kubesphere/pkg/constants"
devopsClient "kubesphere.io/kubesphere/pkg/simple/client/devops"
"kubesphere.io/kubesphere/pkg/utils/k8sutil"
"kubesphere.io/kubesphere/pkg/utils/sliceutil"
"net/http"
"reflect"
"strings"
"time"
)
/**
DevOps project controller is used to maintain the state of the DevOps project.
*/
type Controller struct {
client clientset.Interface
kubesphereClient kubesphereclient.Interface
eventBroadcaster record.EventBroadcaster
eventRecorder record.EventRecorder
secretLister corev1lister.SecretLister
secretSynced cache.InformerSynced
namespaceLister corev1lister.NamespaceLister
namespaceSynced cache.InformerSynced
workqueue workqueue.RateLimitingInterface
workerLoopPeriod time.Duration
devopsClient devopsClient.Interface
}
func NewController(client clientset.Interface,
devopsClinet devopsClient.Interface,
namespaceInformer corev1informer.NamespaceInformer,
secretInformer corev1informer.SecretInformer) *Controller {
broadcaster := record.NewBroadcaster()
broadcaster.StartLogging(func(format string, args ...interface{}) {
klog.Info(fmt.Sprintf(format, args))
})
broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: client.CoreV1().Events("")})
recorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "pipeline-controller"})
v := &Controller{
client: client,
devopsClient: devopsClinet,
workqueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "pipeline"),
secretLister: secretInformer.Lister(),
secretSynced: secretInformer.Informer().HasSynced,
namespaceLister: namespaceInformer.Lister(),
namespaceSynced: namespaceInformer.Informer().HasSynced,
workerLoopPeriod: time.Second,
}
v.eventBroadcaster = broadcaster
v.eventRecorder = recorder
secretInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
secret := obj.(*v1.Secret)
if strings.HasPrefix(string(secret.Type), devopsv1alpha3.DevOpsCredentialPrefix) {
v.enqueueSecret(obj)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
old := oldObj.(*v1.Secret)
new := newObj.(*v1.Secret)
if old.ResourceVersion == new.ResourceVersion {
return
}
if strings.HasPrefix(string(new.Type), devopsv1alpha3.DevOpsCredentialPrefix) {
v.enqueueSecret(newObj)
}
},
DeleteFunc: func(obj interface{}) {
secret := obj.(*v1.Secret)
if strings.HasPrefix(string(secret.Type), devopsv1alpha3.DevOpsCredentialPrefix) {
v.enqueueSecret(obj)
}
},
})
return v
}
// enqueueSecret takes a Foo resource and converts it into a namespace/name
// string which is then put onto the work workqueue. This method should *not* be
// passed resources of any type other than DevOpsProject.
func (c *Controller) enqueueSecret(obj interface{}) {
var key string
var err error
if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
utilruntime.HandleError(err)
return
}
c.workqueue.Add(key)
}
func (c *Controller) processNextWorkItem() bool {
obj, shutdown := c.workqueue.Get()
if shutdown {
return false
}
err := func(obj interface{}) error {
defer c.workqueue.Done(obj)
var key string
var ok bool
if key, ok = obj.(string); !ok {
c.workqueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
return nil
}
if err := c.syncHandler(key); err != nil {
c.workqueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
}
c.workqueue.Forget(obj)
klog.V(5).Infof("Successfully synced '%s'", key)
return nil
}(obj)
if err != nil {
klog.Error(err, "could not reconcile devopsProject")
utilruntime.HandleError(err)
return true
}
return true
}
func (c *Controller) worker() {
for c.processNextWorkItem() {
}
}
func (c *Controller) Start(stopCh <-chan struct{}) error {
return c.Run(1, stopCh)
}
func (c *Controller) Run(workers int, stopCh <-chan struct{}) error {
defer utilruntime.HandleCrash()
defer c.workqueue.ShutDown()
klog.Info("starting pipeline controller")
defer klog.Info("shutting down pipeline controller")
if !cache.WaitForCacheSync(stopCh, c.secretSynced) {
return fmt.Errorf("failed to wait for caches to sync")
}
for i := 0; i < workers; i++ {
go wait.Until(c.worker, c.workerLoopPeriod, stopCh)
}
<-stopCh
return nil
}
// syncHandler compares the actual state with the desired, and attempts to
// converge the two. It then updates the Status block of the pipeline resource
// with the current status of the resource.
func (c *Controller) syncHandler(key string) error {
nsName, name, err := cache.SplitMetaNamespaceKey(key)
if err != nil {
klog.Error(err, fmt.Sprintf("could not split copySecret meta %s ", key))
return nil
}
namespace, err := c.namespaceLister.Get(nsName)
if err != nil {
if errors.IsNotFound(err) {
klog.Info(fmt.Sprintf("namespace '%s' in work queue no longer exists ", key))
return nil
}
klog.Error(err, fmt.Sprintf("could not get namespace %s ", key))
return err
}
if !isDevOpsProjectAdminNamespace(namespace) {
err := fmt.Errorf("cound not create credential in normal namespaces %s", namespace.Name)
klog.Warning(err)
return err
}
secret, err := c.secretLister.Secrets(nsName).Get(name)
if err != nil {
if errors.IsNotFound(err) {
klog.Info(fmt.Sprintf("secret '%s' in work queue no longer exists ", key))
return nil
}
klog.Error(err, fmt.Sprintf("could not get secret %s ", key))
return err
}
copySecret := secret.DeepCopy()
// DeletionTimestamp.IsZero() means copySecret has not been deleted.
if copySecret.ObjectMeta.DeletionTimestamp.IsZero() {
// https://kubernetes.io/docs/tasks/access-kubernetes-api/custom-resources/custom-resource-definitions/#finalizers
if !sliceutil.HasString(copySecret.ObjectMeta.Finalizers, devopsv1alpha3.CredentialFinalizerName) {
copySecret.ObjectMeta.Finalizers = append(copySecret.ObjectMeta.Finalizers, devopsv1alpha3.CredentialFinalizerName)
}
// Check secret config exists, otherwise we will create it.
// if secret exists, update config
_, err := c.devopsClient.GetCredentialInProject(nsName, secret.Name)
if err != nil && devopsClient.GetDevOpsStatusCode(err) != http.StatusNotFound {
klog.Error(err, fmt.Sprintf("failed to get secret %s ", key))
return err
} else if err != nil {
_, err := c.devopsClient.CreateCredentialInProject(nsName, copySecret)
if err != nil {
klog.Error(err, fmt.Sprintf("failed to create secret %s ", key))
return err
}
} else {
if _, ok := copySecret.Annotations[devopsv1alpha3.CredentialAutoSyncAnnoKey]; ok {
_, err := c.devopsClient.UpdateCredentialInProject(nsName, copySecret)
if err != nil {
klog.Error(err, fmt.Sprintf("failed to update secret %s ", key))
return err
}
}
}
} else {
// Finalizers processing logic
if sliceutil.HasString(copySecret.ObjectMeta.Finalizers, devopsv1alpha3.CredentialFinalizerName) {
_, err := c.devopsClient.GetCredentialInProject(nsName, secret.Name)
if err != nil && devopsClient.GetDevOpsStatusCode(err) != http.StatusNotFound {
klog.Error(err, fmt.Sprintf("failed to get secret %s ", key))
return err
} else if err != nil && devopsClient.GetDevOpsStatusCode(err) == http.StatusNotFound {
} else {
if _, err := c.devopsClient.DeleteCredentialInProject(nsName, secret.Name); err != nil {
klog.Error(err, fmt.Sprintf("failed to delete secret %s in devops", key))
return err
}
}
copySecret.ObjectMeta.Finalizers = sliceutil.RemoveString(copySecret.ObjectMeta.Finalizers, func(item string) bool {
return item == devopsv1alpha3.CredentialFinalizerName
})
}
}
if !reflect.DeepEqual(secret, copySecret) {
_, err = c.client.CoreV1().Secrets(nsName).Update(copySecret)
if err != nil {
klog.Error(err, fmt.Sprintf("failed to update secret %s ", key))
return err
}
}
return nil
}
func isDevOpsProjectAdminNamespace(namespace *v1.Namespace) bool {
_, ok := namespace.Labels[constants.DevOpsProjectLabelKey]
return ok && k8sutil.IsControlledBy(namespace.OwnerReferences,
devopsv1alpha3.ResourceKindDevOpsProject, "")
}

View File

@@ -0,0 +1,410 @@
package devopscredential
import (
v1 "k8s.io/api/core/v1"
"kubesphere.io/kubesphere/pkg/constants"
fakeDevOps "kubesphere.io/kubesphere/pkg/simple/client/devops/fake"
"reflect"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/diff"
kubeinformers "k8s.io/client-go/informers"
k8sfake "k8s.io/client-go/kubernetes/fake"
core "k8s.io/client-go/testing"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/tools/record"
devops "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha3"
)
var (
alwaysReady = func() bool { return true }
noResyncPeriodFunc = func() time.Duration { return 0 }
)
type fixture struct {
t *testing.T
kubeclient *k8sfake.Clientset
namespaceLister []*v1.Namespace
secretLister []*v1.Secret
kubeactions []core.Action
kubeobjects []runtime.Object
// Objects from here preloaded into NewSimpleFake.
objects []runtime.Object
// Objects from here preloaded into devops
initDevOpsProject string
initCredential []*v1.Secret
expectCredential []*v1.Secret
}
func newFixture(t *testing.T) *fixture {
f := &fixture{}
f.t = t
f.objects = []runtime.Object{}
return f
}
func newNamespace(name string, projectName string) *v1.Namespace {
ns := &v1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: v1.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{constants.DevOpsProjectLabelKey: projectName},
},
}
TRUE := true
ns.ObjectMeta.OwnerReferences = []metav1.OwnerReference{
{
APIVersion: devops.SchemeGroupVersion.String(),
Kind: devops.ResourceKindDevOpsProject,
Name: projectName,
BlockOwnerDeletion: &TRUE,
Controller: &TRUE,
},
}
return ns
}
func newSecret(namespace, name string, data map[string][]byte, withFinalizers bool, autoSync bool) *v1.Secret {
secret := &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: devops.ResourceKindPipeline,
APIVersion: devops.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
},
Data: data,
Type: devops.DevOpsCredentialPrefix + "test",
}
if withFinalizers {
secret.Finalizers = append(secret.Finalizers, devops.CredentialFinalizerName)
}
if autoSync{
if secret.Annotations == nil{
secret.Annotations = map[string]string{}
}
secret.Annotations[devops.CredentialAutoSyncAnnoKey] = "true"
}
return secret
}
func newDeletingSecret(namespace, name string) *v1.Secret {
now := metav1.Now()
pipeline := &v1.Secret{
TypeMeta: metav1.TypeMeta{
Kind: devops.ResourceKindPipeline,
APIVersion: devops.SchemeGroupVersion.String(),
},
ObjectMeta: metav1.ObjectMeta{
Namespace: namespace,
Name: name,
DeletionTimestamp: &now,
},
Type: devops.DevOpsCredentialPrefix + "test",
}
pipeline.Finalizers = append(pipeline.Finalizers, devops.CredentialFinalizerName)
return pipeline
}
func (f *fixture) newController() (*Controller, kubeinformers.SharedInformerFactory, *fakeDevOps.Devops) {
f.kubeclient = k8sfake.NewSimpleClientset(f.kubeobjects...)
k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
dI := fakeDevOps.NewWithCredentials(f.initDevOpsProject, f.initCredential...)
c := NewController(f.kubeclient, dI, k8sI.Core().V1().Namespaces(),
k8sI.Core().V1().Secrets())
c.secretSynced = alwaysReady
c.eventRecorder = &record.FakeRecorder{}
for _, f := range f.secretLister {
k8sI.Core().V1().Secrets().Informer().GetIndexer().Add(f)
}
for _, d := range f.namespaceLister {
k8sI.Core().V1().Namespaces().Informer().GetIndexer().Add(d)
}
return c, k8sI, dI
}
func (f *fixture) run(fooName string) {
f.runController(fooName, true, false)
}
func (f *fixture) runExpectError(fooName string) {
f.runController(fooName, true, true)
}
func (f *fixture) runController(name string, startInformers bool, expectError bool) {
c, k8sI, dI := f.newController()
if startInformers {
stopCh := make(chan struct{})
defer close(stopCh)
k8sI.Start(stopCh)
}
err := c.syncHandler(name)
if !expectError && err != nil {
f.t.Errorf("error syncing foo: %v", err)
} else if expectError && err == nil {
f.t.Error("expected error syncing foo, got nil")
}
k8sActions := filterInformerActions(f.kubeclient.Actions())
for i, action := range k8sActions {
if len(f.kubeactions) < i+1 {
f.t.Errorf("%d unexpected actions: %+v", len(k8sActions)-len(f.kubeactions), k8sActions[i:])
break
}
expectedAction := f.kubeactions[i]
checkAction(expectedAction, action, f.t)
}
if len(f.kubeactions) > len(k8sActions) {
f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
}
if len(dI.Credentials[f.initDevOpsProject]) != len(f.expectCredential) {
f.t.Errorf(" unexpected objects: %v", dI.Projects)
}
for _, credential := range f.expectCredential {
actualCredential := dI.Credentials[f.initDevOpsProject][credential.Name]
if !reflect.DeepEqual(actualCredential, credential) {
f.t.Errorf(" credential %+v not match \n %+v", credential, actualCredential)
}
}
}
// checkAction verifies that expected and actual actions are equal and both have
// same attached resources
func checkAction(expected, actual core.Action, t *testing.T) {
if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) {
t.Errorf("Expected\n\t%#v\ngot\n\t%#v", expected, actual)
return
}
if reflect.TypeOf(actual) != reflect.TypeOf(expected) {
t.Errorf("Action has wrong type. Expected: %t. Got: %t", expected, actual)
return
}
switch a := actual.(type) {
case core.CreateActionImpl:
e, _ := expected.(core.CreateActionImpl)
expObject := e.GetObject()
object := a.GetObject()
if !reflect.DeepEqual(expObject, object) {
t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintSideBySide(expObject, object))
}
case core.UpdateActionImpl:
e, _ := expected.(core.UpdateActionImpl)
expObject := e.GetObject()
object := a.GetObject()
if !reflect.DeepEqual(expObject, object) {
t.Errorf("Action %s %s has wrong object\nDiff:\n %s",
a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintSideBySide(expObject, object))
}
case core.PatchActionImpl:
e, _ := expected.(core.PatchActionImpl)
expPatch := e.GetPatch()
patch := a.GetPatch()
if !reflect.DeepEqual(expPatch, patch) {
t.Errorf("Action %s %s has wrong patch\nDiff:\n %s",
a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintSideBySide(expPatch, patch))
}
default:
t.Errorf("Uncaptured Action %s %s, you should explicitly add a case to capture it",
actual.GetVerb(), actual.GetResource().Resource)
}
}
// filterInformerActions filters list and watch actions for testing resources.
// Since list and watch don't change resource state we can filter it to lower
// nose level in our tests.
func filterInformerActions(actions []core.Action) []core.Action {
ret := []core.Action{}
for _, action := range actions {
if len(action.GetNamespace()) == 0 &&
(action.Matches("list", "secrets") ||
action.Matches("watch", "secrets") ||
action.Matches("list", "namespaces") ||
action.Matches("watch", "namespaces")) {
continue
}
ret = append(ret, action)
}
return ret
}
func (f *fixture) expectUpdateSecretAction(p *v1.Secret) {
action := core.NewUpdateAction(schema.GroupVersionResource{
Version: "v1",
Resource: "secrets",
}, p.Namespace, p)
f.kubeactions = append(f.kubeactions, action)
}
func getKey(p *v1.Secret, t *testing.T) string {
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(p)
if err != nil {
t.Errorf("Unexpected error getting key for pipeline %v: %v", p.Name, err)
return ""
}
return key
}
func TestDoNothing(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
secret := newSecret(nsName, secretName, nil, true, true)
f.secretLister = append(f.secretLister, secret)
f.namespaceLister = append(f.namespaceLister, ns)
f.objects = append(f.objects, secret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{secret}
f.expectCredential = []*v1.Secret{secret}
f.run(getKey(secret, t))
}
func TestAddCredentialFinalizers(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
secret := newSecret(nsName, secretName, nil, false, true)
expectSecret := newSecret(nsName, secretName, nil, true, true)
f.secretLister = append(f.secretLister, secret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, secret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{secret}
f.expectCredential = []*v1.Secret{expectSecret}
f.expectUpdateSecretAction(expectSecret)
f.run(getKey(secret, t))
}
func TestCreateCredential(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
secret := newSecret(nsName, secretName, nil, true, true)
f.secretLister = append(f.secretLister, secret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, secret)
f.initDevOpsProject = nsName
f.expectCredential = []*v1.Secret{secret}
f.run(getKey(secret, t))
}
func TestDeleteCredential(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
secret := newDeletingSecret(nsName, secretName)
expectSecret := secret.DeepCopy()
expectSecret.Finalizers = []string{}
f.secretLister = append(f.secretLister, secret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, secret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{secret}
f.expectCredential = []*v1.Secret{}
f.expectUpdateSecretAction(expectSecret)
f.run(getKey(secret, t))
}
func TestDeleteNotExistCredential(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
pipelineName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
secret := newDeletingSecret(nsName, pipelineName)
expectSecret := secret.DeepCopy()
expectSecret.Finalizers = []string{}
f.secretLister = append(f.secretLister, secret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, secret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{}
f.expectCredential = []*v1.Secret{}
f.expectUpdateSecretAction(expectSecret)
f.run(getKey(secret, t))
}
func TestUpdateCredential(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
initSecret := newSecret(nsName, secretName, nil, true, true)
expectSecret := newSecret(nsName, secretName, map[string][]byte{"a":[]byte("aa")}, true, true)
f.secretLister = append(f.secretLister, expectSecret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, expectSecret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{initSecret}
f.expectCredential = []*v1.Secret{expectSecret}
f.run(getKey(expectSecret, t))
}
func TestNotUpdateCredential(t *testing.T) {
f := newFixture(t)
nsName := "test-123"
secretName := "test"
projectName := "test_project"
ns := newNamespace(nsName, projectName)
initSecret := newSecret(nsName, secretName, nil, true, false)
expectSecret := newSecret(nsName, secretName, map[string][]byte{"a":[]byte("aa")}, true, false)
f.secretLister = append(f.secretLister, expectSecret)
f.namespaceLister = append(f.namespaceLister, ns)
f.kubeobjects = append(f.kubeobjects, expectSecret)
f.initDevOpsProject = nsName
f.initCredential = []*v1.Secret{initSecret}
f.expectCredential = []*v1.Secret{initSecret}
f.run(getKey(expectSecret, t))
}