446 lines
17 KiB
Go
446 lines
17 KiB
Go
/*
|
|
|
|
Copyright 2020 The KubeSphere Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
package capability
|
|
|
|
import (
|
|
"math/rand"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
|
|
//"github.com/google/go-cmp/cmp"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
snapbeta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1"
|
|
snapfake "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/fake"
|
|
snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions"
|
|
storagev1 "k8s.io/api/storage/v1"
|
|
storagev1beta1 "k8s.io/api/storage/v1beta1"
|
|
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/util/diff"
|
|
k8sinformers "k8s.io/client-go/informers"
|
|
k8sfake "k8s.io/client-go/kubernetes/fake"
|
|
core "k8s.io/client-go/testing"
|
|
"k8s.io/client-go/tools/cache"
|
|
|
|
ksv1alpha1 "kubesphere.io/api/storage/v1alpha1"
|
|
|
|
ksfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
|
|
ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
|
|
)
|
|
|
|
var (
|
|
noReSyncPeriodFunc = func() time.Duration { return 0 }
|
|
)
|
|
|
|
type fixture struct {
|
|
t *testing.T
|
|
snapshotSupported bool
|
|
// Clients
|
|
k8sClient *k8sfake.Clientset
|
|
snapshotClassClient *snapfake.Clientset
|
|
ksClient *ksfake.Clientset
|
|
// Objects from here preload into NewSimpleFake.
|
|
storageObjects []runtime.Object // include StorageClass and CSIDriver
|
|
snapshotClassObjects []runtime.Object
|
|
capabilityObjects []runtime.Object // include StorageClassCapability and ProvisionerCapability
|
|
// Objects to put in the store.
|
|
storageClassLister []*storagev1.StorageClass
|
|
snapshotClassLister []*snapbeta1.VolumeSnapshotClass
|
|
storageClassCapabilityLister []*ksv1alpha1.StorageClassCapability
|
|
provisionerCapabilityLister []*ksv1alpha1.ProvisionerCapability
|
|
csiDriverLister []*storagev1beta1.CSIDriver
|
|
// Actions expected to happen on the client.
|
|
actions []core.Action
|
|
// CSI server
|
|
runCSIServer bool
|
|
fakeCSIServer *fakeCSIServer
|
|
}
|
|
|
|
func newFixture(t *testing.T, snapshotSupported bool, runCSIServer bool) *fixture {
|
|
return &fixture{
|
|
t: t,
|
|
snapshotSupported: snapshotSupported,
|
|
runCSIServer: runCSIServer,
|
|
}
|
|
}
|
|
|
|
func (f *fixture) newController() (*StorageCapabilityController,
|
|
k8sinformers.SharedInformerFactory,
|
|
ksinformers.SharedInformerFactory,
|
|
snapinformers.SharedInformerFactory) {
|
|
|
|
f.k8sClient = k8sfake.NewSimpleClientset(f.storageObjects...)
|
|
f.ksClient = ksfake.NewSimpleClientset(f.capabilityObjects...)
|
|
f.snapshotClassClient = snapfake.NewSimpleClientset(f.snapshotClassObjects...)
|
|
|
|
k8sInformers := k8sinformers.NewSharedInformerFactory(f.k8sClient, noReSyncPeriodFunc())
|
|
ksInformers := ksinformers.NewSharedInformerFactory(f.ksClient, noReSyncPeriodFunc())
|
|
snapshotInformers := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc())
|
|
|
|
c := NewController(
|
|
f.ksClient.StorageV1alpha1().StorageClassCapabilities(),
|
|
ksInformers.Storage().V1alpha1(),
|
|
f.k8sClient.StorageV1().StorageClasses(),
|
|
k8sInformers.Storage().V1().StorageClasses(),
|
|
f.snapshotSupported,
|
|
f.snapshotClassClient.SnapshotV1beta1().VolumeSnapshotClasses(),
|
|
snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses(),
|
|
k8sInformers.Storage().V1beta1().CSIDrivers(),
|
|
)
|
|
if f.runCSIServer {
|
|
port := 30000 + rand.Intn(100)
|
|
fakeCSIServer, address := newTestCSIServer(port)
|
|
f.fakeCSIServer = fakeCSIServer
|
|
c.setCSIAddressGetter(func(storageClassProvisioner string) string { return address })
|
|
}
|
|
|
|
for _, storageClass := range f.storageClassLister {
|
|
_ = k8sInformers.Storage().V1().StorageClasses().Informer().GetIndexer().Add(storageClass)
|
|
}
|
|
for _, csiDriver := range f.csiDriverLister {
|
|
_ = k8sInformers.Storage().V1beta1().CSIDrivers().Informer().GetIndexer().Add(csiDriver)
|
|
}
|
|
for _, snapshotClass := range f.snapshotClassLister {
|
|
_ = snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass)
|
|
}
|
|
for _, storageClassCapability := range f.storageClassCapabilityLister {
|
|
_ = ksInformers.Storage().V1alpha1().StorageClassCapabilities().Informer().GetIndexer().Add(storageClassCapability)
|
|
}
|
|
for _, provisionerCapability := range f.provisionerCapabilityLister {
|
|
_ = ksInformers.Storage().V1alpha1().ProvisionerCapabilities().Informer().GetIndexer().Add(provisionerCapability)
|
|
}
|
|
|
|
return c, k8sInformers, ksInformers, snapshotInformers
|
|
}
|
|
|
|
func (f *fixture) runController(scName string, startInformers bool, expectError bool) {
|
|
c, k8sI, crdI, snapI := f.newController()
|
|
|
|
if f.runCSIServer {
|
|
f.fakeCSIServer.run()
|
|
defer f.fakeCSIServer.stop()
|
|
}
|
|
|
|
if startInformers {
|
|
stopCh := make(chan struct{})
|
|
defer close(stopCh)
|
|
k8sI.Start(stopCh)
|
|
crdI.Start(stopCh)
|
|
snapI.Start(stopCh)
|
|
}
|
|
|
|
err := c.syncHandler(scName)
|
|
if !expectError && err != nil {
|
|
f.t.Errorf("error syncing: %v", err)
|
|
} else if expectError && err == nil {
|
|
f.t.Error("expected error syncing, got nil")
|
|
}
|
|
|
|
var actions []core.Action
|
|
actions = append(actions, f.snapshotClassClient.Actions()...)
|
|
actions = append(actions, f.k8sClient.Actions()...)
|
|
actions = append(actions, f.ksClient.Actions()...)
|
|
filerActions := filterInformerActions(actions)
|
|
if len(filerActions) != len(f.actions) {
|
|
f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions))
|
|
return
|
|
}
|
|
for i, action := range filerActions {
|
|
expectedAction := f.actions[i]
|
|
checkAction(expectedAction, action, f.t)
|
|
}
|
|
}
|
|
|
|
func (f *fixture) run(scName string) {
|
|
f.runController(scName, true, false)
|
|
}
|
|
|
|
func (f *fixture) expectCreateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
|
|
f.actions = append(f.actions, core.NewCreateAction(
|
|
schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability))
|
|
}
|
|
|
|
func (f *fixture) expectUpdateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
|
|
f.actions = append(f.actions, core.NewUpdateAction(
|
|
schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability))
|
|
}
|
|
|
|
func (f *fixture) expectDeleteStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
|
|
f.actions = append(f.actions, core.NewDeleteAction(
|
|
schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability.Name))
|
|
}
|
|
|
|
func (f *fixture) expectUpdateStorageClassAction(storageClass *storagev1.StorageClass) {
|
|
f.actions = append(f.actions, core.NewUpdateAction(
|
|
schema.GroupVersionResource{Resource: "storageclasses"}, storageClass.Namespace, storageClass))
|
|
}
|
|
|
|
func (f *fixture) expectCreateSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) {
|
|
f.actions = append(f.actions, core.NewCreateAction(
|
|
schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass))
|
|
}
|
|
|
|
func (f *fixture) expectDeleteSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) {
|
|
f.actions = append(f.actions, core.NewDeleteAction(
|
|
schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass.Name))
|
|
}
|
|
|
|
// filterInformerActions filters list and watch actions for testing resources.
|
|
// Since list and watch don't change resource state we can filter it to lower
|
|
// nose level in our tests.
|
|
func filterInformerActions(actions []core.Action) []core.Action {
|
|
var ret []core.Action
|
|
for _, action := range actions {
|
|
if action.GetVerb() == "list" || action.GetVerb() == "watch" {
|
|
continue
|
|
}
|
|
ret = append(ret, action)
|
|
}
|
|
return ret
|
|
}
|
|
|
|
// checkAction verifies that expected and actual actions are equal and both have
|
|
// same attached resources
|
|
func checkAction(expected, actual core.Action, t *testing.T) {
|
|
if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) {
|
|
t.Errorf("\nExpected\n\t%#v\ngot\n\t%#v", expected, actual)
|
|
return
|
|
}
|
|
|
|
if reflect.TypeOf(actual) != reflect.TypeOf(expected) {
|
|
t.Errorf("Action has wrong type. Expected: %t. Got: %t", expected, actual)
|
|
return
|
|
}
|
|
|
|
switch a := actual.(type) {
|
|
case core.CreateActionImpl:
|
|
e, _ := expected.(core.CreateActionImpl)
|
|
expObject := e.GetObject()
|
|
object := a.GetObject()
|
|
if difference := cmp.Diff(object, expObject); len(difference) > 0 {
|
|
t.Errorf("[CreateAction] %T differ (-got, +want): %s", expObject, difference)
|
|
}
|
|
case core.UpdateActionImpl:
|
|
e, _ := expected.(core.UpdateActionImpl)
|
|
expObject := e.GetObject()
|
|
object := a.GetObject()
|
|
if difference := cmp.Diff(object, expObject); len(difference) > 0 {
|
|
t.Errorf("[UpdateAction] %T differ (-got, +want): %s", expObject, difference)
|
|
}
|
|
case core.PatchActionImpl:
|
|
e, _ := expected.(core.PatchActionImpl)
|
|
expPatch := e.GetPatch()
|
|
patch := a.GetPatch()
|
|
if !reflect.DeepEqual(expPatch, patch) {
|
|
t.Errorf("Action %s %s has wrong patch\nDiff:\n %s",
|
|
a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintSideBySide(expPatch, patch))
|
|
}
|
|
case core.DeleteActionImpl:
|
|
e, _ := expected.(core.DeleteActionImpl)
|
|
if difference := cmp.Diff(e.Name, a.Name); len(difference) > 0 {
|
|
t.Errorf("[UpdateAction] %T differ (-got, +want): %s", e.Name, difference)
|
|
}
|
|
default:
|
|
t.Errorf("Uncaptured Action %s %s, you should explicitly add a case to capture it",
|
|
actual.GetVerb(), actual.GetResource().Resource)
|
|
}
|
|
}
|
|
|
|
func newStorageClass(name string, provisioner string) *storagev1.StorageClass {
|
|
isExpansion := true
|
|
return &storagev1.StorageClass{
|
|
ObjectMeta: v1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Provisioner: provisioner,
|
|
AllowVolumeExpansion: &isExpansion,
|
|
}
|
|
}
|
|
|
|
func newStorageClassCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.StorageClassCapability {
|
|
storageClassCapability := &ksv1alpha1.StorageClassCapability{}
|
|
storageClassCapability.Name = storageClass.Name
|
|
storageClassCapability.Spec = *newStorageClassCapabilitySpec()
|
|
storageClassCapability.Spec.Provisioner = storageClass.Provisioner
|
|
return storageClassCapability
|
|
}
|
|
|
|
func newProvisionerCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.ProvisionerCapability {
|
|
provisionerCapability := &ksv1alpha1.ProvisionerCapability{}
|
|
provisionerCapability.Name = getProvisionerCapabilityName(storageClass.Provisioner)
|
|
provisionerCapability.Spec.PluginInfo.Name = storageClass.Provisioner
|
|
provisionerCapability.Spec.Features = newStorageClassCapabilitySpec().Features
|
|
// ProvisionerCapability snapshot is always false
|
|
provisionerCapability.Spec.Features.Snapshot.Create = false
|
|
return provisionerCapability
|
|
}
|
|
|
|
func newCSIDriver(storageClass *storagev1.StorageClass) *storagev1beta1.CSIDriver {
|
|
csiDriver := &storagev1beta1.CSIDriver{}
|
|
csiDriver.Name = storageClass.Provisioner
|
|
return csiDriver
|
|
}
|
|
|
|
func newSnapshotClass(storageClass *storagev1.StorageClass) *snapbeta1.VolumeSnapshotClass {
|
|
return &snapbeta1.VolumeSnapshotClass{
|
|
ObjectMeta: v1.ObjectMeta{
|
|
Name: storageClass.Name,
|
|
},
|
|
Driver: storageClass.Provisioner,
|
|
DeletionPolicy: snapbeta1.VolumeSnapshotContentDelete,
|
|
}
|
|
}
|
|
|
|
func getKey(sc *storagev1.StorageClass, t *testing.T) string {
|
|
key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(sc)
|
|
if err != nil {
|
|
t.Errorf("Unexpected error getting key for %v: %v", sc.Name, err)
|
|
return ""
|
|
}
|
|
return key
|
|
}
|
|
|
|
func TestCreateStorageClass(t *testing.T) {
|
|
fixture := newFixture(t, true, true)
|
|
storageClass := newStorageClass("csi-example", "csi.example.com")
|
|
storageClassUpdate := storageClass.DeepCopy()
|
|
storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "true"}
|
|
snapshotClass := newSnapshotClass(storageClass)
|
|
storageClassCapability := newStorageClassCapability(storageClass)
|
|
csiDriver := newCSIDriver(storageClass)
|
|
|
|
// Objects exist
|
|
fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
|
|
fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
|
|
fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
|
|
|
|
// Action expected
|
|
fixture.expectCreateSnapshotClassAction(snapshotClass)
|
|
fixture.expectUpdateStorageClassAction(storageClassUpdate)
|
|
fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
|
|
|
|
// Run test
|
|
fixture.run(getKey(storageClass, t))
|
|
}
|
|
|
|
func TestUpdateStorageClass(t *testing.T) {
|
|
storageClass := newStorageClass("csi-example", "csi.example.com")
|
|
storageClass.Annotations = map[string]string{annotationSupportSnapshot: "true"}
|
|
snapshotClass := newSnapshotClass(storageClass)
|
|
storageClassCapabilityUpdate := newStorageClassCapability(storageClass)
|
|
storageClassCapability := newStorageClassCapability(storageClass)
|
|
//old and new should have deference
|
|
storageClassCapability.Spec.Features.Volume.Create = !storageClassCapability.Spec.Features.Volume.Create
|
|
csiDriver := newCSIDriver(storageClass)
|
|
|
|
fixture := newFixture(t, true, true)
|
|
// Object exist
|
|
fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
|
|
fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
|
|
fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
|
|
fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass)
|
|
fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass)
|
|
fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability)
|
|
fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability)
|
|
|
|
// Action expected
|
|
fixture.expectUpdateStorageClassCapabilitiesAction(storageClassCapabilityUpdate)
|
|
|
|
// Run test
|
|
fixture.run(getKey(storageClass, t))
|
|
}
|
|
|
|
func TestDeleteStorageClass(t *testing.T) {
|
|
storageClass := newStorageClass("csi-example", "csi.example.com")
|
|
snapshotClass := newSnapshotClass(storageClass)
|
|
storageClassCapability := newStorageClassCapability(storageClass)
|
|
|
|
csiDriver := newCSIDriver(storageClass)
|
|
|
|
fixture := newFixture(t, true, true)
|
|
// Object exist
|
|
fixture.storageObjects = append(fixture.storageObjects, csiDriver)
|
|
fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
|
|
fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass)
|
|
fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass)
|
|
fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability)
|
|
fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability)
|
|
|
|
// Action expected
|
|
fixture.expectDeleteSnapshotClassAction(snapshotClass)
|
|
fixture.expectDeleteStorageClassCapabilitiesAction(storageClassCapability)
|
|
|
|
// Run test
|
|
fixture.run(getKey(storageClass, t))
|
|
}
|
|
|
|
func TestCreateStorageClassNotSupportSnapshot(t *testing.T) {
|
|
fixture := newFixture(t, false, true)
|
|
storageClass := newStorageClass("csi-example", "csi.example.com")
|
|
storageClassUpdate := storageClass.DeepCopy()
|
|
storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"}
|
|
storageClassCapability := newStorageClassCapability(storageClass)
|
|
storageClassCapability.Spec.Features.Snapshot.Create = false
|
|
storageClassCapability.Spec.Features.Snapshot.List = false
|
|
provisionerCapability := newProvisionerCapability(storageClass)
|
|
csiDriver := newCSIDriver(storageClass)
|
|
|
|
// Objects exist
|
|
fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
|
|
fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
|
|
fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
|
|
fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability)
|
|
fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability)
|
|
|
|
// Action expected
|
|
fixture.expectUpdateStorageClassAction(storageClassUpdate)
|
|
fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
|
|
|
|
// Run test
|
|
fixture.run(getKey(storageClass, t))
|
|
}
|
|
|
|
func TestCreateStorageClassInTree(t *testing.T) {
|
|
// InTree Storage has no snapshot capability
|
|
fixture := newFixture(t, true, true)
|
|
storageClass := newStorageClass("csi-example", "csi.example.com")
|
|
storageClassUpdate := storageClass.DeepCopy()
|
|
storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"}
|
|
storageClassCapability := newStorageClassCapability(storageClass)
|
|
storageClassCapability.Spec.Features.Snapshot.Create = false
|
|
provisionerCapability := newProvisionerCapability(storageClass)
|
|
|
|
// Objects exist
|
|
fixture.storageObjects = append(fixture.storageObjects, storageClass)
|
|
fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
|
|
fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability)
|
|
fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability)
|
|
|
|
// Action expected
|
|
fixture.expectUpdateStorageClassAction(storageClassUpdate)
|
|
fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
|
|
|
|
// Run test
|
|
fixture.run(getKey(storageClass, t))
|
|
}
|