From 574eb221ab37ce05d8b6bcb126b69e24fea08649 Mon Sep 17 00:00:00 2001 From: f10atin9 Date: Wed, 18 Aug 2021 01:19:45 +0800 Subject: [PATCH 1/4] remove capability CRDs and update controller Signed-off-by: f10atin9 --- cmd/controller-manager/app/controllers.go | 3 +- .../fake/fake_provisionercapability.go | 122 -------- .../v1alpha1/fake/fake_storage_client.go | 9 - .../fake/fake_storageclasscapability.go | 122 -------- .../storage/v1alpha1/provisionercapability.go | 168 ----------- .../typed/storage/v1alpha1/storage_client.go | 10 - .../v1alpha1/storageclasscapability.go | 168 ----------- .../informers/externalversions/generic.go | 7 - .../storage/v1alpha1/interface.go | 14 - .../storage/v1alpha1/provisionercapability.go | 89 ------ .../v1alpha1/storageclasscapability.go | 89 ------ .../capability/capability_controller.go | 271 +++++++----------- .../capability/capability_controller_test.go | 225 +++++---------- 13 files changed, 165 insertions(+), 1132 deletions(-) delete mode 100644 pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go delete mode 100644 pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go delete mode 100644 pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go delete mode 100644 pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go delete mode 100644 pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go delete mode 100644 pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index b710730a9..0e250a023 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -96,10 +96,9 @@ func addControllers( jobController := job.NewJobController(kubernetesInformer.Batch().V1().Jobs(), client.Kubernetes()) storageCapabilityController := capability.NewController( - client.KubeSphere().StorageV1alpha1().StorageClassCapabilities(), - kubesphereInformer.Storage().V1alpha1(), client.Kubernetes().StorageV1().StorageClasses(), kubernetesInformer.Storage().V1().StorageClasses(), + kubernetesInformer.Storage().V1beta1().CSIDrivers(), capability.SnapshotSupported(client.Kubernetes().Discovery()), client.Snapshot().SnapshotV1beta1().VolumeSnapshotClasses(), informerFactory.SnapshotSharedInformerFactory().Snapshot().V1beta1().VolumeSnapshotClasses(), diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go deleted file mode 100644 index 0bbdc207f..000000000 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" - v1alpha1 "kubesphere.io/api/storage/v1alpha1" -) - -// FakeProvisionerCapabilities implements ProvisionerCapabilityInterface -type FakeProvisionerCapabilities struct { - Fake *FakeStorageV1alpha1 -} - -var provisionercapabilitiesResource = schema.GroupVersionResource{Group: "storage.kubesphere.io", Version: "v1alpha1", Resource: "provisionercapabilities"} - -var provisionercapabilitiesKind = schema.GroupVersionKind{Group: "storage.kubesphere.io", Version: "v1alpha1", Kind: "ProvisionerCapability"} - -// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any. -func (c *FakeProvisionerCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootGetAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.ProvisionerCapability), err -} - -// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors. -func (c *FakeProvisionerCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootListAction(provisionercapabilitiesResource, provisionercapabilitiesKind, opts), &v1alpha1.ProvisionerCapabilityList{}) - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.ProvisionerCapabilityList{ListMeta: obj.(*v1alpha1.ProvisionerCapabilityList).ListMeta} - for _, item := range obj.(*v1alpha1.ProvisionerCapabilityList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested provisionerCapabilities. -func (c *FakeProvisionerCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewRootWatchAction(provisionercapabilitiesResource, opts)) -} - -// Create takes the representation of a provisionerCapability and creates it. Returns the server's representation of the provisionerCapability, and an error, if there is any. -func (c *FakeProvisionerCapabilities) Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (result *v1alpha1.ProvisionerCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootCreateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.ProvisionerCapability), err -} - -// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any. -func (c *FakeProvisionerCapabilities) Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (result *v1alpha1.ProvisionerCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootUpdateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.ProvisionerCapability), err -} - -// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs. -func (c *FakeProvisionerCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewRootDeleteAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{}) - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeProvisionerCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewRootDeleteCollectionAction(provisionercapabilitiesResource, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.ProvisionerCapabilityList{}) - return err -} - -// Patch applies the patch and returns the patched provisionerCapability. -func (c *FakeProvisionerCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootPatchSubresourceAction(provisionercapabilitiesResource, name, pt, data, subresources...), &v1alpha1.ProvisionerCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.ProvisionerCapability), err -} diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go index 8e3916bd2..7b37af71f 100644 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go +++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go @@ -21,21 +21,12 @@ package fake import ( rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" - v1alpha1 "kubesphere.io/kubesphere/pkg/client/clientset/versioned/typed/storage/v1alpha1" ) type FakeStorageV1alpha1 struct { *testing.Fake } -func (c *FakeStorageV1alpha1) ProvisionerCapabilities() v1alpha1.ProvisionerCapabilityInterface { - return &FakeProvisionerCapabilities{c} -} - -func (c *FakeStorageV1alpha1) StorageClassCapabilities() v1alpha1.StorageClassCapabilityInterface { - return &FakeStorageClassCapabilities{c} -} - // RESTClient returns a RESTClient that is used to communicate // with API server by this client implementation. func (c *FakeStorageV1alpha1) RESTClient() rest.Interface { diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go deleted file mode 100644 index e4781fb00..000000000 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go +++ /dev/null @@ -1,122 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package fake - -import ( - "context" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - labels "k8s.io/apimachinery/pkg/labels" - schema "k8s.io/apimachinery/pkg/runtime/schema" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - testing "k8s.io/client-go/testing" - v1alpha1 "kubesphere.io/api/storage/v1alpha1" -) - -// FakeStorageClassCapabilities implements StorageClassCapabilityInterface -type FakeStorageClassCapabilities struct { - Fake *FakeStorageV1alpha1 -} - -var storageclasscapabilitiesResource = schema.GroupVersionResource{Group: "storage.kubesphere.io", Version: "v1alpha1", Resource: "storageclasscapabilities"} - -var storageclasscapabilitiesKind = schema.GroupVersionKind{Group: "storage.kubesphere.io", Version: "v1alpha1", Kind: "StorageClassCapability"} - -// Get takes name of the storageClassCapability, and returns the corresponding storageClassCapability object, and an error if there is any. -func (c *FakeStorageClassCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.StorageClassCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootGetAction(storageclasscapabilitiesResource, name), &v1alpha1.StorageClassCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.StorageClassCapability), err -} - -// List takes label and field selectors, and returns the list of StorageClassCapabilities that match those selectors. -func (c *FakeStorageClassCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.StorageClassCapabilityList, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootListAction(storageclasscapabilitiesResource, storageclasscapabilitiesKind, opts), &v1alpha1.StorageClassCapabilityList{}) - if obj == nil { - return nil, err - } - - label, _, _ := testing.ExtractFromListOptions(opts) - if label == nil { - label = labels.Everything() - } - list := &v1alpha1.StorageClassCapabilityList{ListMeta: obj.(*v1alpha1.StorageClassCapabilityList).ListMeta} - for _, item := range obj.(*v1alpha1.StorageClassCapabilityList).Items { - if label.Matches(labels.Set(item.Labels)) { - list.Items = append(list.Items, item) - } - } - return list, err -} - -// Watch returns a watch.Interface that watches the requested storageClassCapabilities. -func (c *FakeStorageClassCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - return c.Fake. - InvokesWatch(testing.NewRootWatchAction(storageclasscapabilitiesResource, opts)) -} - -// Create takes the representation of a storageClassCapability and creates it. Returns the server's representation of the storageClassCapability, and an error, if there is any. -func (c *FakeStorageClassCapabilities) Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (result *v1alpha1.StorageClassCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootCreateAction(storageclasscapabilitiesResource, storageClassCapability), &v1alpha1.StorageClassCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.StorageClassCapability), err -} - -// Update takes the representation of a storageClassCapability and updates it. Returns the server's representation of the storageClassCapability, and an error, if there is any. -func (c *FakeStorageClassCapabilities) Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (result *v1alpha1.StorageClassCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootUpdateAction(storageclasscapabilitiesResource, storageClassCapability), &v1alpha1.StorageClassCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.StorageClassCapability), err -} - -// Delete takes name of the storageClassCapability and deletes it. Returns an error if one occurs. -func (c *FakeStorageClassCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - _, err := c.Fake. - Invokes(testing.NewRootDeleteAction(storageclasscapabilitiesResource, name), &v1alpha1.StorageClassCapability{}) - return err -} - -// DeleteCollection deletes a collection of objects. -func (c *FakeStorageClassCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - action := testing.NewRootDeleteCollectionAction(storageclasscapabilitiesResource, listOpts) - - _, err := c.Fake.Invokes(action, &v1alpha1.StorageClassCapabilityList{}) - return err -} - -// Patch applies the patch and returns the patched storageClassCapability. -func (c *FakeStorageClassCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error) { - obj, err := c.Fake. - Invokes(testing.NewRootPatchSubresourceAction(storageclasscapabilitiesResource, name, pt, data, subresources...), &v1alpha1.StorageClassCapability{}) - if obj == nil { - return nil, err - } - return obj.(*v1alpha1.StorageClassCapability), err -} diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go deleted file mode 100644 index 13de7af73..000000000 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" - v1alpha1 "kubesphere.io/api/storage/v1alpha1" - scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" -) - -// ProvisionerCapabilitiesGetter has a method to return a ProvisionerCapabilityInterface. -// A group's client should implement this interface. -type ProvisionerCapabilitiesGetter interface { - ProvisionerCapabilities() ProvisionerCapabilityInterface -} - -// ProvisionerCapabilityInterface has methods to work with ProvisionerCapability resources. -type ProvisionerCapabilityInterface interface { - Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (*v1alpha1.ProvisionerCapability, error) - Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (*v1alpha1.ProvisionerCapability, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.ProvisionerCapability, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.ProvisionerCapabilityList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) - ProvisionerCapabilityExpansion -} - -// provisionerCapabilities implements ProvisionerCapabilityInterface -type provisionerCapabilities struct { - client rest.Interface -} - -// newProvisionerCapabilities returns a ProvisionerCapabilities -func newProvisionerCapabilities(c *StorageV1alpha1Client) *provisionerCapabilities { - return &provisionerCapabilities{ - client: c.RESTClient(), - } -} - -// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any. -func (c *provisionerCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) { - result = &v1alpha1.ProvisionerCapability{} - err = c.client.Get(). - Resource("provisionercapabilities"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors. -func (c *provisionerCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1alpha1.ProvisionerCapabilityList{} - err = c.client.Get(). - Resource("provisionercapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested provisionerCapabilities. -func (c *provisionerCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Resource("provisionercapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a provisionerCapability and creates it. Returns the server's representation of the provisionerCapability, and an error, if there is any. -func (c *provisionerCapabilities) Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (result *v1alpha1.ProvisionerCapability, err error) { - result = &v1alpha1.ProvisionerCapability{} - err = c.client.Post(). - Resource("provisionercapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(provisionerCapability). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any. -func (c *provisionerCapabilities) Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (result *v1alpha1.ProvisionerCapability, err error) { - result = &v1alpha1.ProvisionerCapability{} - err = c.client.Put(). - Resource("provisionercapabilities"). - Name(provisionerCapability.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(provisionerCapability). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs. -func (c *provisionerCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Resource("provisionercapabilities"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *provisionerCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Resource("provisionercapabilities"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched provisionerCapability. -func (c *provisionerCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) { - result = &v1alpha1.ProvisionerCapability{} - err = c.client.Patch(pt). - Resource("provisionercapabilities"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go index d1500585e..39a5c05ab 100644 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go +++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go @@ -26,8 +26,6 @@ import ( type StorageV1alpha1Interface interface { RESTClient() rest.Interface - ProvisionerCapabilitiesGetter - StorageClassCapabilitiesGetter } // StorageV1alpha1Client is used to interact with features provided by the storage.kubesphere.io group. @@ -35,14 +33,6 @@ type StorageV1alpha1Client struct { restClient rest.Interface } -func (c *StorageV1alpha1Client) ProvisionerCapabilities() ProvisionerCapabilityInterface { - return newProvisionerCapabilities(c) -} - -func (c *StorageV1alpha1Client) StorageClassCapabilities() StorageClassCapabilityInterface { - return newStorageClassCapabilities(c) -} - // NewForConfig creates a new StorageV1alpha1Client for the given config. func NewForConfig(c *rest.Config) (*StorageV1alpha1Client, error) { config := *c diff --git a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go b/pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go deleted file mode 100644 index b9f5ca087..000000000 --- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by client-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - types "k8s.io/apimachinery/pkg/types" - watch "k8s.io/apimachinery/pkg/watch" - rest "k8s.io/client-go/rest" - v1alpha1 "kubesphere.io/api/storage/v1alpha1" - scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" -) - -// StorageClassCapabilitiesGetter has a method to return a StorageClassCapabilityInterface. -// A group's client should implement this interface. -type StorageClassCapabilitiesGetter interface { - StorageClassCapabilities() StorageClassCapabilityInterface -} - -// StorageClassCapabilityInterface has methods to work with StorageClassCapability resources. -type StorageClassCapabilityInterface interface { - Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (*v1alpha1.StorageClassCapability, error) - Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (*v1alpha1.StorageClassCapability, error) - Delete(ctx context.Context, name string, opts v1.DeleteOptions) error - DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error - Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.StorageClassCapability, error) - List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.StorageClassCapabilityList, error) - Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) - Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error) - StorageClassCapabilityExpansion -} - -// storageClassCapabilities implements StorageClassCapabilityInterface -type storageClassCapabilities struct { - client rest.Interface -} - -// newStorageClassCapabilities returns a StorageClassCapabilities -func newStorageClassCapabilities(c *StorageV1alpha1Client) *storageClassCapabilities { - return &storageClassCapabilities{ - client: c.RESTClient(), - } -} - -// Get takes name of the storageClassCapability, and returns the corresponding storageClassCapability object, and an error if there is any. -func (c *storageClassCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.StorageClassCapability, err error) { - result = &v1alpha1.StorageClassCapability{} - err = c.client.Get(). - Resource("storageclasscapabilities"). - Name(name). - VersionedParams(&options, scheme.ParameterCodec). - Do(ctx). - Into(result) - return -} - -// List takes label and field selectors, and returns the list of StorageClassCapabilities that match those selectors. -func (c *storageClassCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.StorageClassCapabilityList, err error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - result = &v1alpha1.StorageClassCapabilityList{} - err = c.client.Get(). - Resource("storageclasscapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Do(ctx). - Into(result) - return -} - -// Watch returns a watch.Interface that watches the requested storageClassCapabilities. -func (c *storageClassCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) { - var timeout time.Duration - if opts.TimeoutSeconds != nil { - timeout = time.Duration(*opts.TimeoutSeconds) * time.Second - } - opts.Watch = true - return c.client.Get(). - Resource("storageclasscapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Timeout(timeout). - Watch(ctx) -} - -// Create takes the representation of a storageClassCapability and creates it. Returns the server's representation of the storageClassCapability, and an error, if there is any. -func (c *storageClassCapabilities) Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (result *v1alpha1.StorageClassCapability, err error) { - result = &v1alpha1.StorageClassCapability{} - err = c.client.Post(). - Resource("storageclasscapabilities"). - VersionedParams(&opts, scheme.ParameterCodec). - Body(storageClassCapability). - Do(ctx). - Into(result) - return -} - -// Update takes the representation of a storageClassCapability and updates it. Returns the server's representation of the storageClassCapability, and an error, if there is any. -func (c *storageClassCapabilities) Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (result *v1alpha1.StorageClassCapability, err error) { - result = &v1alpha1.StorageClassCapability{} - err = c.client.Put(). - Resource("storageclasscapabilities"). - Name(storageClassCapability.Name). - VersionedParams(&opts, scheme.ParameterCodec). - Body(storageClassCapability). - Do(ctx). - Into(result) - return -} - -// Delete takes name of the storageClassCapability and deletes it. Returns an error if one occurs. -func (c *storageClassCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error { - return c.client.Delete(). - Resource("storageclasscapabilities"). - Name(name). - Body(&opts). - Do(ctx). - Error() -} - -// DeleteCollection deletes a collection of objects. -func (c *storageClassCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error { - var timeout time.Duration - if listOpts.TimeoutSeconds != nil { - timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second - } - return c.client.Delete(). - Resource("storageclasscapabilities"). - VersionedParams(&listOpts, scheme.ParameterCodec). - Timeout(timeout). - Body(&opts). - Do(ctx). - Error() -} - -// Patch applies the patch and returns the patched storageClassCapability. -func (c *storageClassCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error) { - result = &v1alpha1.StorageClassCapability{} - err = c.client.Patch(pt). - Resource("storageclasscapabilities"). - Name(name). - SubResource(subresources...). - VersionedParams(&opts, scheme.ParameterCodec). - Body(data). - Do(ctx). - Into(result) - return -} diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index 3d08ef2c9..4b5edaa74 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -33,7 +33,6 @@ import ( v2beta1 "kubesphere.io/api/notification/v2beta1" quotav1alpha2 "kubesphere.io/api/quota/v1alpha2" servicemeshv1alpha2 "kubesphere.io/api/servicemesh/v1alpha2" - storagev1alpha1 "kubesphere.io/api/storage/v1alpha1" tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1" tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2" v1beta1 "kubesphere.io/api/types/v1beta1" @@ -149,12 +148,6 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case servicemeshv1alpha2.SchemeGroupVersion.WithResource("strategies"): return &genericInformer{resource: resource.GroupResource(), informer: f.Servicemesh().V1alpha2().Strategies().Informer()}, nil - // Group=storage.kubesphere.io, Version=v1alpha1 - case storagev1alpha1.SchemeGroupVersion.WithResource("provisionercapabilities"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().ProvisionerCapabilities().Informer()}, nil - case storagev1alpha1.SchemeGroupVersion.WithResource("storageclasscapabilities"): - return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().StorageClassCapabilities().Informer()}, nil - // Group=tenant.kubesphere.io, Version=v1alpha1 case tenantv1alpha1.SchemeGroupVersion.WithResource("workspaces"): return &genericInformer{resource: resource.GroupResource(), informer: f.Tenant().V1alpha1().Workspaces().Informer()}, nil diff --git a/pkg/client/informers/externalversions/storage/v1alpha1/interface.go b/pkg/client/informers/externalversions/storage/v1alpha1/interface.go index 01c74e93c..c68e2a439 100644 --- a/pkg/client/informers/externalversions/storage/v1alpha1/interface.go +++ b/pkg/client/informers/externalversions/storage/v1alpha1/interface.go @@ -24,10 +24,6 @@ import ( // Interface provides access to all the informers in this group version. type Interface interface { - // ProvisionerCapabilities returns a ProvisionerCapabilityInformer. - ProvisionerCapabilities() ProvisionerCapabilityInformer - // StorageClassCapabilities returns a StorageClassCapabilityInformer. - StorageClassCapabilities() StorageClassCapabilityInformer } type version struct { @@ -40,13 +36,3 @@ type version struct { func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} } - -// ProvisionerCapabilities returns a ProvisionerCapabilityInformer. -func (v *version) ProvisionerCapabilities() ProvisionerCapabilityInformer { - return &provisionerCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} -} - -// StorageClassCapabilities returns a StorageClassCapabilityInformer. -func (v *version) StorageClassCapabilities() StorageClassCapabilityInformer { - return &storageClassCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions} -} diff --git a/pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go b/pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go deleted file mode 100644 index 905f5904c..000000000 --- a/pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - storagev1alpha1 "kubesphere.io/api/storage/v1alpha1" - versioned "kubesphere.io/kubesphere/pkg/client/clientset/versioned" - internalinterfaces "kubesphere.io/kubesphere/pkg/client/informers/externalversions/internalinterfaces" - v1alpha1 "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1" -) - -// ProvisionerCapabilityInformer provides access to a shared informer and lister for -// ProvisionerCapabilities. -type ProvisionerCapabilityInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1alpha1.ProvisionerCapabilityLister -} - -type provisionerCapabilityInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// NewProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, indexers, nil) -} - -// NewFilteredProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.StorageV1alpha1().ProvisionerCapabilities().List(context.TODO(), options) - }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.StorageV1alpha1().ProvisionerCapabilities().Watch(context.TODO(), options) - }, - }, - &storagev1alpha1.ProvisionerCapability{}, - resyncPeriod, - indexers, - ) -} - -func (f *provisionerCapabilityInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *provisionerCapabilityInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&storagev1alpha1.ProvisionerCapability{}, f.defaultInformer) -} - -func (f *provisionerCapabilityInformer) Lister() v1alpha1.ProvisionerCapabilityLister { - return v1alpha1.NewProvisionerCapabilityLister(f.Informer().GetIndexer()) -} diff --git a/pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go b/pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go deleted file mode 100644 index 147955c4b..000000000 --- a/pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2020 The KubeSphere Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by informer-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - "context" - time "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - runtime "k8s.io/apimachinery/pkg/runtime" - watch "k8s.io/apimachinery/pkg/watch" - cache "k8s.io/client-go/tools/cache" - storagev1alpha1 "kubesphere.io/api/storage/v1alpha1" - versioned "kubesphere.io/kubesphere/pkg/client/clientset/versioned" - internalinterfaces "kubesphere.io/kubesphere/pkg/client/informers/externalversions/internalinterfaces" - v1alpha1 "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1" -) - -// StorageClassCapabilityInformer provides access to a shared informer and lister for -// StorageClassCapabilities. -type StorageClassCapabilityInformer interface { - Informer() cache.SharedIndexInformer - Lister() v1alpha1.StorageClassCapabilityLister -} - -type storageClassCapabilityInformer struct { - factory internalinterfaces.SharedInformerFactory - tweakListOptions internalinterfaces.TweakListOptionsFunc -} - -// NewStorageClassCapabilityInformer constructs a new informer for StorageClassCapability type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewStorageClassCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { - return NewFilteredStorageClassCapabilityInformer(client, resyncPeriod, indexers, nil) -} - -// NewFilteredStorageClassCapabilityInformer constructs a new informer for StorageClassCapability type. -// Always prefer using an informer factory to get a shared informer instead of getting an independent -// one. This reduces memory footprint and number of connections to the server. -func NewFilteredStorageClassCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { - return cache.NewSharedIndexInformer( - &cache.ListWatch{ - ListFunc: func(options v1.ListOptions) (runtime.Object, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.StorageV1alpha1().StorageClassCapabilities().List(context.TODO(), options) - }, - WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { - if tweakListOptions != nil { - tweakListOptions(&options) - } - return client.StorageV1alpha1().StorageClassCapabilities().Watch(context.TODO(), options) - }, - }, - &storagev1alpha1.StorageClassCapability{}, - resyncPeriod, - indexers, - ) -} - -func (f *storageClassCapabilityInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { - return NewFilteredStorageClassCapabilityInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) -} - -func (f *storageClassCapabilityInformer) Informer() cache.SharedIndexInformer { - return f.factory.InformerFor(&storagev1alpha1.StorageClassCapability{}, f.defaultInformer) -} - -func (f *storageClassCapabilityInformer) Lister() v1alpha1.StorageClassCapabilityLister { - return v1alpha1.NewStorageClassCapabilityLister(f.Informer().GetIndexer()) -} diff --git a/pkg/controller/storage/capability/capability_controller.go b/pkg/controller/storage/capability/capability_controller.go index 06ac81d72..6bbcfab7b 100644 --- a/pkg/controller/storage/capability/capability_controller.go +++ b/pkg/controller/storage/capability/capability_controller.go @@ -21,18 +21,18 @@ package capability import ( "context" "fmt" - "reflect" "strconv" - "strings" "time" + snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/volumesnapshot/v1beta1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/version" "k8s.io/client-go/discovery" + storageinformersv1beta1 "k8s.io/client-go/informers/storage/v1beta1" + storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/typed/volumesnapshot/v1beta1" - snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/volumesnapshot/v1beta1" snapshotlisters "github.com/kubernetes-csi/external-snapshotter/client/v3/listers/volumesnapshot/v1beta1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -47,46 +47,38 @@ import ( "k8s.io/client-go/util/workqueue" "k8s.io/klog" - ksstorage "kubesphere.io/api/storage/v1alpha1" - crdscheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" - ksstorageclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned/typed/storage/v1alpha1" - ksstorageinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/storage/v1alpha1" - ksstoragelisters "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1" ) const ( minSnapshotSupportedVersion = "v1.17.0" - annotationSupportSnapshot = "storageclass.kubesphere.io/support-snapshot" + annotationSupportSnapshot = "storageclass.kubesphere.io/allow-snapshot" + annotationSupportClone = "storageclass.kubesphere.io/allow-clone" ) type StorageCapabilityController struct { - storageClassCapabilityClient ksstorageclient.StorageClassCapabilityInterface - storageClassCapabilityLister ksstoragelisters.StorageClassCapabilityLister - storageClassCapabilitySynced cache.InformerSynced - - provisionerCapabilityLister ksstoragelisters.ProvisionerCapabilityLister - provisionerCapabilitySynced cache.InformerSynced - storageClassClient storageclient.StorageClassInterface storageClassLister storagelistersv1.StorageClassLister storageClassSynced cache.InformerSynced + csiDriverLister storagelistersv1beta1.CSIDriverLister + csiDriverSynced cache.InformerSynced + snapshotSupported bool snapshotClassClient snapshotclient.VolumeSnapshotClassInterface snapshotClassLister snapshotlisters.VolumeSnapshotClassLister snapshotClassSynced cache.InformerSynced - workQueue workqueue.RateLimitingInterface + workQueue workqueue.RateLimitingInterface + csiWorkQueue workqueue.RateLimitingInterface } // This controller is responsible to watch StorageClass/ProvisionerCapability. // And then update StorageClassCapability CRD resource object to the newest status. func NewController( - storageClassCapabilityClient ksstorageclient.StorageClassCapabilityInterface, - ksStorageInformer ksstorageinformers.Interface, storageClassClient storageclient.StorageClassInterface, storageClassInformer storageinformersv1.StorageClassInformer, + csiDriverInformer storageinformersv1beta1.CSIDriverInformer, snapshotSupported bool, snapshotClassClient snapshotclient.VolumeSnapshotClassInterface, snapshotClassInformer snapinformers.VolumeSnapshotClassInformer, @@ -95,16 +87,14 @@ func NewController( utilruntime.Must(crdscheme.AddToScheme(scheme.Scheme)) controller := &StorageCapabilityController{ - storageClassCapabilityClient: storageClassCapabilityClient, - storageClassCapabilityLister: ksStorageInformer.StorageClassCapabilities().Lister(), - storageClassCapabilitySynced: ksStorageInformer.StorageClassCapabilities().Informer().HasSynced, - provisionerCapabilityLister: ksStorageInformer.ProvisionerCapabilities().Lister(), - provisionerCapabilitySynced: ksStorageInformer.ProvisionerCapabilities().Informer().HasSynced, - storageClassClient: storageClassClient, - storageClassLister: storageClassInformer.Lister(), - storageClassSynced: storageClassInformer.Informer().HasSynced, - snapshotSupported: snapshotSupported, - workQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"), + storageClassClient: storageClassClient, + storageClassLister: storageClassInformer.Lister(), + storageClassSynced: storageClassInformer.Informer().HasSynced, + csiDriverLister: csiDriverInformer.Lister(), + csiDriverSynced: csiDriverInformer.Informer().HasSynced, + snapshotSupported: snapshotSupported, + workQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"), + csiWorkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csiDriver"), } if snapshotSupported { @@ -126,24 +116,10 @@ func NewController( DeleteFunc: controller.enqueueStorageClass, }) - // ProvisionerCapability acts as a value source of its relevant StorageClassCapabilities - // so when a PC is created/updated, the corresponding SCCs should be created(if not exists)/updated - // we achieve this by simply enqueueing the StorageClasses of the same provisioner - // but don't overdo by cascade deleting the SCCs when a PC is deleted - // since the role of PCs is more like a template rather than owner to SCCs - - // This is a backward compatible fix to remove the useless auto detection of SCCs - // in the future, we will only keep ProvisionerCapability and remove the StorageClassCapability CRD entirely - ksStorageInformer.ProvisionerCapabilities().Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.handleProvisionerCapability, - UpdateFunc: func(oldObj, newObj interface{}) { - newPC := newObj.(*ksstorage.ProvisionerCapability) - oldPC := oldObj.(*ksstorage.ProvisionerCapability) - if newPC.ResourceVersion == oldPC.ResourceVersion { - return - } - controller.handleProvisionerCapability(newObj) - }, + csiDriverInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueStorageClassByCSI, + UpdateFunc: nil, + DeleteFunc: controller.enqueueStorageClassByCSI, }) return controller @@ -160,13 +136,8 @@ func (c *StorageCapabilityController) Run(threadCnt int, stopCh <-chan struct{}) // Wait for the caches to be synced before starting workers klog.Info("Waiting for informer caches to sync") cacheSyncs := []cache.InformerSynced{ - c.storageClassCapabilitySynced, - c.provisionerCapabilitySynced, c.storageClassSynced, - } - - if c.snapshotAllowed() { - cacheSyncs = append(cacheSyncs, c.snapshotClassSynced) + c.csiDriverSynced, } if ok := cache.WaitForCacheSync(stopCh, cacheSyncs...); !ok { @@ -182,21 +153,6 @@ func (c *StorageCapabilityController) Run(threadCnt int, stopCh <-chan struct{}) return nil } -func (c *StorageCapabilityController) handleProvisionerCapability(obj interface{}) { - provisionerCapability := obj.(*ksstorage.ProvisionerCapability) - storageClasses, err := c.storageClassLister.List(labels.Everything()) - if err != nil { - klog.Error("list StorageClass error when handle provisionerCapability", err) - return - } - for _, storageClass := range storageClasses { - if getProvisionerCapabilityName(storageClass.Provisioner) == provisionerCapability.Name { - klog.V(4).Infof("enqueue StorageClass %s while handling provisionerCapability", storageClass.Name) - c.enqueueStorageClass(storageClass) - } - } -} - func (c *StorageCapabilityController) enqueueStorageClass(obj interface{}) { var key string var err error @@ -207,6 +163,27 @@ func (c *StorageCapabilityController) enqueueStorageClass(obj interface{}) { c.workQueue.Add(key) } +func (c *StorageCapabilityController) enqueueStorageClassByCSI(csi interface{}) { + var objs []*storagev1.StorageClass + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(csi); err != nil { + utilruntime.HandleError(err) + return + } + objs, err = c.storageClassLister.List(labels.NewSelector()) + if err != nil { + utilruntime.HandleError(err) + return + } + for _, obj := range objs { + if obj.Provisioner == key { + c.workQueue.Add(obj.Name) + } + } + return +} + func (c *StorageCapabilityController) runWorker() { for c.processNextWorkItem() { } @@ -257,40 +234,20 @@ func (c *StorageCapabilityController) syncHandler(key string) error { storageClass, err := c.storageClassLister.Get(name) if err != nil { // StorageClass has been deleted, delete StorageClassCapability and VolumeSnapshotClass - if errors.IsNotFound(err) { - if c.snapshotAllowed() { - err = c.deleteSnapshotClass(name) - if err != nil { - return err - } + if errors.IsNotFound(err) && c.snapshotAllowed() { + err = c.deleteSnapshotClass(name) + if err != nil { + return err } - return c.deleteStorageCapability(name) } return err } - // Get capability spec - capabilitySpec, err := c.getCapabilitySpec(storageClass) - if err != nil { - return err - } - // The corresponding ProvisionerCapability Object does not exist - if capabilitySpec == nil { - klog.Infof("Can't get StorageClass %s's capability", name) - err = c.updateStorageClassSnapshotSupported(storageClass, false) - if err != nil { - return err - } - // Don't delete the already created SCC - // as it might be created manually by user - return nil - } - klog.Infof("StorageClass %s has capability %v", name, capabilitySpec) - + //Cloning and volumeSnapshot support only available for CSI drivers. + withCapability := c.supportCapability(storageClass) // Handle VolumeSnapshotClass with same name of StorageClass // annotate "support-snapshot" of StorageClass - withSnapshotCapability := false - if c.snapshotAllowed() && capabilitySpec.Features.Snapshot.Create { + if c.snapshotAllowed() && withCapability { _, err = c.snapshotClassLister.Get(name) if err != nil { // If VolumeSnapshotClass not exist, create it @@ -306,63 +263,71 @@ func (c *StorageCapabilityController) syncHandler(key string) error { } } } - withSnapshotCapability = true - } - err = c.updateStorageClassSnapshotSupported(storageClass, withSnapshotCapability) - if err != nil { - return err } - // Handle StorageClassCapability with the same name of StorageClass - storageClassCapabilityExist, err := c.storageClassCapabilityLister.Get(storageClass.Name) + err = c.addStorageClassSnapshotAnnotation(storageClass, withCapability) if err != nil { - if errors.IsNotFound(err) { - // If StorageClassCapability doesn't exist, create it - storageClassCapabilityCreate := &ksstorage.StorageClassCapability{ObjectMeta: metav1.ObjectMeta{Name: storageClass.Name}} - storageClassCapabilityCreate.Spec = *capabilitySpec - klog.Info("Create StorageClassCapability: ", storageClassCapabilityCreate) - _, err = c.storageClassCapabilityClient.Create(context.Background(), storageClassCapabilityCreate, metav1.CreateOptions{}) - return err - } return err } - // If StorageClassCapability exist, update it. - storageClassCapabilityUpdate := storageClassCapabilityExist.DeepCopy() - storageClassCapabilityUpdate.Spec = *capabilitySpec - if !reflect.DeepEqual(storageClassCapabilityExist, storageClassCapabilityUpdate) { - klog.Info("Update StorageClassCapability: ", storageClassCapabilityUpdate) - _, err = c.storageClassCapabilityClient.Update(context.Background(), storageClassCapabilityUpdate, metav1.UpdateOptions{}) + err = c.addCloneVolumeAnnotation(storageClass, withCapability) + if err != nil { + return nil + } + _, err = c.storageClassClient.Update(context.Background(), storageClass, metav1.UpdateOptions{}) + if err != nil { return err } return nil } -func (c *StorageCapabilityController) updateStorageClassSnapshotSupported(storageClass *storagev1.StorageClass, snapshotSupported bool) error { - if storageClass.Annotations == nil { - storageClass.Annotations = make(map[string]string) +func (c *StorageCapabilityController) supportCapability(storageClass *storagev1.StorageClass) bool { + driver := storageClass.Provisioner + if driver != "" { + if _, err := c.csiDriverLister.Get(driver); err != nil { + return false + } + return true } - snapshotSupportedAnnotated, err := strconv.ParseBool(storageClass.Annotations[annotationSupportSnapshot]) - // err != nil means annotationSupportSnapshot is not illegal, include empty - if err != nil || snapshotSupported != snapshotSupportedAnnotated { - storageClass.Annotations[annotationSupportSnapshot] = strconv.FormatBool(snapshotSupported) - _, err = c.storageClassClient.Update(context.Background(), storageClass, metav1.UpdateOptions{}) + return false +} + +func (c *StorageCapabilityController) addStorageClassSnapshotAnnotation(storageClass *storagev1.StorageClass, snapshotSupported bool) error { + if snapshotSupported || !c.snapshotSupported { + if storageClass.Annotations == nil { + storageClass.Annotations = make(map[string]string) + } + _, err := strconv.ParseBool(storageClass.Annotations[annotationSupportSnapshot]) + // err != nil means annotationSupportSnapshot is not illegal, include empty if err != nil { - return err + storageClass.Annotations[annotationSupportSnapshot] = strconv.FormatBool(c.snapshotSupported) + } + } else { + if storageClass.Annotations != nil && c.snapshotSupported { + if _, ok := storageClass.Annotations[annotationSupportSnapshot]; ok { + delete(storageClass.Annotations, annotationSupportSnapshot) + } } } return nil } -func (c *StorageCapabilityController) deleteStorageCapability(name string) error { - _, err := c.storageClassCapabilityLister.Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil +func (c *StorageCapabilityController) addCloneVolumeAnnotation(storageClass *storagev1.StorageClass, cloneSupported bool) error { + if cloneSupported { + if storageClass.Annotations == nil { + storageClass.Annotations = make(map[string]string) + } + _, err := strconv.ParseBool(storageClass.Annotations[annotationSupportClone]) + if err != nil { + storageClass.Annotations[annotationSupportClone] = strconv.FormatBool(cloneSupported) + } + } else { + if storageClass.Annotations != nil { + if _, ok := storageClass.Annotations[annotationSupportClone]; ok { + delete(storageClass.Annotations, annotationSupportClone) + } } - return err } - klog.Infof("Delete StorageClassCapability %s", name) - return c.storageClassCapabilityClient.Delete(context.Background(), name, metav1.DeleteOptions{}) + return nil } func (c *StorageCapabilityController) deleteSnapshotClass(name string) error { @@ -380,42 +345,6 @@ func (c *StorageCapabilityController) deleteSnapshotClass(name string) error { return c.snapshotClassClient.Delete(context.Background(), name, metav1.DeleteOptions{}) } -func (c *StorageCapabilityController) capabilityFromProvisioner(provisioner string) (*ksstorage.StorageClassCapabilitySpec, error) { - provisionerCapability, err := c.provisionerCapabilityLister.Get(getProvisionerCapabilityName(provisioner)) - if err != nil { - if errors.IsNotFound(err) { - return nil, nil - } - return nil, err - } - klog.V(4).Infof("get provisioner capability:%s %s", provisioner, provisionerCapability.Name) - capabilitySpec := &ksstorage.StorageClassCapabilitySpec{ - Features: provisionerCapability.Spec.Features, - } - return capabilitySpec, nil -} - -func (c *StorageCapabilityController) getCapabilitySpec(storageClass *storagev1.StorageClass) (*ksstorage.StorageClassCapabilitySpec, error) { - // get from provisioner capability first - klog.V(4).Info("get cap ", storageClass.Provisioner) - capabilitySpec, err := c.capabilityFromProvisioner(storageClass.Provisioner) - if err != nil { - return nil, err - } - - if capabilitySpec != nil { - capabilitySpec.Provisioner = storageClass.Provisioner - if storageClass.AllowVolumeExpansion == nil || !*storageClass.AllowVolumeExpansion { - capabilitySpec.Features.Volume.Expand = ksstorage.ExpandModeUnknown - } - if !c.snapshotSupported { - capabilitySpec.Features.Snapshot.Create = false - capabilitySpec.Features.Snapshot.List = false - } - } - return capabilitySpec, nil -} - func (c *StorageCapabilityController) snapshotAllowed() bool { return c.snapshotSupported && c.snapshotClassClient != nil && c.snapshotClassLister != nil && c.snapshotClassSynced != nil } @@ -432,7 +361,3 @@ func SnapshotSupported(discoveryInterface discovery.DiscoveryInterface) bool { } return ver.AtLeast(minVer) } - -func getProvisionerCapabilityName(provisioner string) string { - return strings.NewReplacer(".", "-", "/", "-").Replace(provisioner) -} diff --git a/pkg/controller/storage/capability/capability_controller_test.go b/pkg/controller/storage/capability/capability_controller_test.go index 233739e87..ef7292e59 100644 --- a/pkg/controller/storage/capability/capability_controller_test.go +++ b/pkg/controller/storage/capability/capability_controller_test.go @@ -20,6 +20,7 @@ package capability import ( "github.com/google/go-cmp/cmp" + "k8s.io/api/storage/v1beta1" "reflect" "testing" @@ -38,8 +39,6 @@ import ( core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" - ksv1alpha1 "kubesphere.io/api/storage/v1alpha1" - ksfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake" ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" ) @@ -60,10 +59,9 @@ type fixture struct { snapshotClassObjects []runtime.Object capabilityObjects []runtime.Object // include StorageClassCapability and ProvisionerCapability // Objects to put in the store. - storageClassLister []*storagev1.StorageClass - snapshotClassLister []*snapbeta1.VolumeSnapshotClass - storageClassCapabilityLister []*ksv1alpha1.StorageClassCapability - provisionerCapabilityLister []*ksv1alpha1.ProvisionerCapability + storageClassLister []*storagev1.StorageClass + snapshotClassLister []*snapbeta1.VolumeSnapshotClass + csiDriverLister []*v1beta1.CSIDriver // Actions expected to happen on the client. actions []core.Action } @@ -89,10 +87,9 @@ func (f *fixture) newController() (*StorageCapabilityController, snapshotInformers := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc()) c := NewController( - f.ksClient.StorageV1alpha1().StorageClassCapabilities(), - ksInformers.Storage().V1alpha1(), f.k8sClient.StorageV1().StorageClasses(), k8sInformers.Storage().V1().StorageClasses(), + k8sInformers.Storage().V1beta1().CSIDrivers(), f.snapshotSupported, f.snapshotClassClient.SnapshotV1beta1().VolumeSnapshotClasses(), snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses(), @@ -104,11 +101,8 @@ func (f *fixture) newController() (*StorageCapabilityController, for _, snapshotClass := range f.snapshotClassLister { _ = snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass) } - for _, storageClassCapability := range f.storageClassCapabilityLister { - _ = ksInformers.Storage().V1alpha1().StorageClassCapabilities().Informer().GetIndexer().Add(storageClassCapability) - } - for _, provisionerCapability := range f.provisionerCapabilityLister { - _ = ksInformers.Storage().V1alpha1().ProvisionerCapabilities().Informer().GetIndexer().Add(provisionerCapability) + for _, csiDriver := range f.csiDriverLister { + _ = k8sInformers.Storage().V1beta1().CSIDrivers().Informer().GetIndexer().Add(csiDriver) } return c, k8sInformers, ksInformers, snapshotInformers @@ -151,21 +145,6 @@ func (f *fixture) run(scName string) { f.runController(scName, true, false) } -func (f *fixture) expectCreateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) { - f.actions = append(f.actions, core.NewCreateAction( - schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability)) -} - -func (f *fixture) expectUpdateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) { - f.actions = append(f.actions, core.NewUpdateAction( - schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability)) -} - -func (f *fixture) expectDeleteStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) { - f.actions = append(f.actions, core.NewDeleteAction( - schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability.Name)) -} - func (f *fixture) expectUpdateStorageClassAction(storageClass *storagev1.StorageClass) { f.actions = append(f.actions, core.NewUpdateAction( schema.GroupVersionResource{Resource: "storageclasses"}, storageClass.Namespace, storageClass)) @@ -253,42 +232,14 @@ func newStorageClass(name string, provisioner string) *storagev1.StorageClass { } } -func newStorageClassCapabilitySpec() *ksv1alpha1.StorageClassCapabilitySpec { - return &ksv1alpha1.StorageClassCapabilitySpec{ - Features: ksv1alpha1.CapabilityFeatures{ - Topology: false, - Volume: ksv1alpha1.VolumeFeature{ - Create: true, - Attach: false, - List: false, - Clone: true, - Stats: true, - Expand: ksv1alpha1.ExpandModeOffline, - }, - Snapshot: ksv1alpha1.SnapshotFeature{ - Create: true, - List: false, - }, +func newCSIDriver(name string) *v1beta1.CSIDriver { + return &v1beta1.CSIDriver{ + ObjectMeta: v1.ObjectMeta{ + Name: name, }, } } -func newStorageClassCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.StorageClassCapability { - storageClassCapability := &ksv1alpha1.StorageClassCapability{} - storageClassCapability.Name = storageClass.Name - storageClassCapability.Spec = *newStorageClassCapabilitySpec() - storageClassCapability.Spec.Provisioner = storageClass.Provisioner - return storageClassCapability -} - -func newProvisionerCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.ProvisionerCapability { - provisionerCapability := &ksv1alpha1.ProvisionerCapability{} - provisionerCapability.Name = getProvisionerCapabilityName(storageClass.Provisioner) - provisionerCapability.Spec.PluginInfo.Name = storageClass.Provisioner - provisionerCapability.Spec.Features = newStorageClassCapabilitySpec().Features - return provisionerCapability -} - func newSnapshotClass(storageClass *storagev1.StorageClass) *snapbeta1.VolumeSnapshotClass { return &snapbeta1.VolumeSnapshotClass{ ObjectMeta: v1.ObjectMeta{ @@ -312,88 +263,17 @@ func TestCreateStorageClass(t *testing.T) { fixture := newFixture(t, true) storageClass := newStorageClass("csi-example", "csi.example.com") storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "true"} - provisionerCapability := newProvisionerCapability(storageClass) + storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "true", annotationSupportClone: "true"} snapshotClass := newSnapshotClass(storageClass) - storageClassCapability := newStorageClassCapability(storageClass) + csiDriver := newCSIDriver("csi.example.com") // Objects exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability) - fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability) + fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) // Action expected fixture.expectCreateSnapshotClassAction(snapshotClass) - fixture.expectUpdateStorageClassAction(storageClassUpdate) - fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability) - - // Run test - fixture.run(getKey(storageClass, t)) -} - -func TestCreateStorageClassWithoutProvisionerCapability(t *testing.T) { - fixture := newFixture(t, true) - storageClass := newStorageClass("csi-example", "csi.example.com") - - // Objects exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - - storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"} - fixture.expectUpdateStorageClassAction(storageClassUpdate) - - // Run test - fixture.run(getKey(storageClass, t)) -} - -func TestUpdateStorageClass(t *testing.T) { - storageClass := newStorageClass("csi-example", "csi.example.com") - storageClass.Annotations = map[string]string{annotationSupportSnapshot: "true"} - snapshotClass := newSnapshotClass(storageClass) - storageClassCapabilityUpdate := newStorageClassCapability(storageClass) - storageClassCapability := newStorageClassCapability(storageClass) - provisionerCapability := newProvisionerCapability(storageClass) - //old and new should have deference - storageClassCapability.Spec.Features.Volume.Create = !storageClassCapability.Spec.Features.Volume.Create - - fixture := newFixture(t, true) - // Object exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass) - fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability, provisionerCapability) - fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability) - fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability) - - // Action expected - fixture.expectUpdateStorageClassCapabilitiesAction(storageClassCapabilityUpdate) - - // Run test - fixture.run(getKey(storageClass, t)) -} - -func TestUpdateStorageClassWithoutProvisionerCapability(t *testing.T) { - storageClass := newStorageClass("csi-example", "csi.example.com") - storageClass.Annotations = map[string]string{annotationSupportSnapshot: "true"} - storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations[annotationSupportSnapshot] = "false" - snapshotClass := newSnapshotClass(storageClass) - storageClassCapability := newStorageClassCapability(storageClass) - //old and new should have deference - storageClassCapability.Spec.Features.Volume.Create = !storageClassCapability.Spec.Features.Volume.Create - - fixture := newFixture(t, true) - // Object exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass) - fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability) - fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability) - fixture.expectUpdateStorageClassAction(storageClassUpdate) // Run test @@ -403,19 +283,15 @@ func TestUpdateStorageClassWithoutProvisionerCapability(t *testing.T) { func TestDeleteStorageClass(t *testing.T) { storageClass := newStorageClass("csi-example", "csi.example.com") snapshotClass := newSnapshotClass(storageClass) - storageClassCapability := newStorageClassCapability(storageClass) fixture := newFixture(t, true) // Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass) fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability) - fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability) // Action expected fixture.expectDeleteSnapshotClassAction(snapshotClass) - fixture.expectDeleteStorageClassCapabilitiesAction(storageClassCapability) // Run test fixture.run(getKey(storageClass, t)) @@ -426,47 +302,78 @@ func TestCreateStorageClassNotSupportSnapshot(t *testing.T) { fixture := newFixture(t, false) storageClass := newStorageClass("csi-example", "csi.example.com") storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"} - storageClassCapability := newStorageClassCapability(storageClass) - storageClassCapability.Spec.Features.Snapshot.Create = false - storageClassCapability.Spec.Features.Snapshot.List = false - provisionerCapability := newProvisionerCapability(storageClass) + storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "true"} + csiDriver := newCSIDriver("csi.example.com") // Objects exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability) - fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability) + fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) // Action expected fixture.expectUpdateStorageClassAction(storageClassUpdate) - fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability) // Run test fixture.run(getKey(storageClass, t)) } -func TestCreateStorageClassNotHaveSnapshotCap(t *testing.T) { - // Storage has no snapshot capability +func TestStorageClassHadAnnotation(t *testing.T) { fixture := newFixture(t, true) storageClass := newStorageClass("csi-example", "csi.example.com") + storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "false"} storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"} - storageClassCapability := newStorageClassCapability(storageClass) - storageClassCapability.Spec.Features.Snapshot.Create = false - provisionerCapability := newProvisionerCapability(storageClass) - provisionerCapability.Spec.Features.Snapshot.Create = false + csiDriver := newCSIDriver("csi.example.com") + snapshotClass := newSnapshotClass(storageClass) - // Objects exist + //Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability) - fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability) + fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) - // Action expected + //Action expected + fixture.expectCreateSnapshotClassAction(snapshotClass) fixture.expectUpdateStorageClassAction(storageClassUpdate) - fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability) - // Run test + //Run test + fixture.run(getKey(storageClass, t)) +} + +func TestStorageClassHadOneAnnotation(t *testing.T) { + fixture := newFixture(t, true) + storageClass := newStorageClass("csi-example", "csi.example.com") + storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false"} + storageClassUpdate := storageClass.DeepCopy() + storageClassUpdate.Annotations[annotationSupportClone] = "true" + csiDriver := newCSIDriver("csi.example.com") + snapshotClass := newSnapshotClass(storageClass) + + //object exist + fixture.storageObjects = append(fixture.storageObjects, storageClass) + fixture.storageClassLister = append(fixture.storageClassLister, storageClass) + fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) + + //Action expected + fixture.expectCreateSnapshotClassAction(snapshotClass) + fixture.expectUpdateStorageClassAction(storageClassUpdate) + + //Run test + fixture.run(getKey(storageClass, t)) +} + +func TestDeleteCSIDriver(t *testing.T) { + fixture := newFixture(t, true) + storageClass := newStorageClass("csi-example", "csi.example.com") + storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "false"} + storageClassUpdate := storageClass.DeepCopy() + storageClassUpdate.Annotations = map[string]string{} + + //object exist + fixture.storageObjects = append(fixture.storageObjects, storageClass) + fixture.storageClassLister = append(fixture.storageClassLister, storageClass) + + //Action expected + fixture.expectUpdateStorageClassAction(storageClassUpdate) + + //Run test fixture.run(getKey(storageClass, t)) } From 5e9679941b8efb89d26b3c98be55693eae99dfc6 Mon Sep 17 00:00:00 2001 From: f10atin9 Date: Mon, 23 Aug 2021 10:47:27 +0800 Subject: [PATCH 2/4] split snapshot controller and update capability. Signed-off-by: f10atin9 --- cmd/controller-manager/app/controllers.go | 8 +- .../capability/capability_controller.go | 173 +++--------- .../capability/capability_controller_test.go | 133 +-------- .../storage/snapshot/snapshot_controller.go | 195 +++++++++++++ .../snapshot/snapshot_controller_test.go | 258 ++++++++++++++++++ 5 files changed, 511 insertions(+), 256 deletions(-) create mode 100644 pkg/controller/storage/snapshot/snapshot_controller.go create mode 100644 pkg/controller/storage/snapshot/snapshot_controller_test.go diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index 0e250a023..54ffd9ffb 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -23,6 +23,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/kubefed/pkg/controller/util" + "kubesphere.io/kubesphere/pkg/controller/storage/snapshot" + iamv1alpha2 "kubesphere.io/api/iam/v1alpha2" authoptions "kubesphere.io/kubesphere/pkg/apiserver/authentication/options" @@ -99,7 +101,10 @@ func addControllers( client.Kubernetes().StorageV1().StorageClasses(), kubernetesInformer.Storage().V1().StorageClasses(), kubernetesInformer.Storage().V1beta1().CSIDrivers(), - capability.SnapshotSupported(client.Kubernetes().Discovery()), + ) + + volumeSnapshotController := snapshot.NewController( + kubernetesInformer.Storage().V1().StorageClasses(), client.Snapshot().SnapshotV1beta1().VolumeSnapshotClasses(), informerFactory.SnapshotSharedInformerFactory().Snapshot().V1beta1().VolumeSnapshotClasses(), ) @@ -215,6 +220,7 @@ func addControllers( "destinationrule-controller": drController, "job-controller": jobController, "storagecapability-controller": storageCapabilityController, + "volumesnapshot-controller": volumeSnapshotController, "user-controller": userController, "loginrecord-controller": loginRecordController, "cluster-controller": clusterController, diff --git a/pkg/controller/storage/capability/capability_controller.go b/pkg/controller/storage/capability/capability_controller.go index 6bbcfab7b..9acd35bdf 100644 --- a/pkg/controller/storage/capability/capability_controller.go +++ b/pkg/controller/storage/capability/capability_controller.go @@ -24,18 +24,11 @@ import ( "strconv" "time" - snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/volumesnapshot/v1beta1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/version" - "k8s.io/client-go/discovery" storageinformersv1beta1 "k8s.io/client-go/informers/storage/v1beta1" storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1" - snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" - snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/typed/volumesnapshot/v1beta1" - snapshotlisters "github.com/kubernetes-csi/external-snapshotter/client/v3/listers/volumesnapshot/v1beta1" storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -51,9 +44,8 @@ import ( ) const ( - minSnapshotSupportedVersion = "v1.17.0" - annotationSupportSnapshot = "storageclass.kubesphere.io/allow-snapshot" - annotationSupportClone = "storageclass.kubesphere.io/allow-clone" + annotationAllowSnapshot = "storageclass.kubesphere.io/allow-snapshot" + annotationAllowClone = "storageclass.kubesphere.io/allow-clone" ) type StorageCapabilityController struct { @@ -64,43 +56,26 @@ type StorageCapabilityController struct { csiDriverLister storagelistersv1beta1.CSIDriverLister csiDriverSynced cache.InformerSynced - snapshotSupported bool - snapshotClassClient snapshotclient.VolumeSnapshotClassInterface - snapshotClassLister snapshotlisters.VolumeSnapshotClassLister - snapshotClassSynced cache.InformerSynced - - workQueue workqueue.RateLimitingInterface - csiWorkQueue workqueue.RateLimitingInterface + storageClassWorkQueue workqueue.RateLimitingInterface } -// This controller is responsible to watch StorageClass/ProvisionerCapability. -// And then update StorageClassCapability CRD resource object to the newest status. +// This controller is responsible to watch StorageClass and CSIDriver. +// And then update StorageClass CRD resource object to the newest status. func NewController( storageClassClient storageclient.StorageClassInterface, storageClassInformer storageinformersv1.StorageClassInformer, csiDriverInformer storageinformersv1beta1.CSIDriverInformer, - snapshotSupported bool, - snapshotClassClient snapshotclient.VolumeSnapshotClassInterface, - snapshotClassInformer snapinformers.VolumeSnapshotClassInformer, ) *StorageCapabilityController { utilruntime.Must(crdscheme.AddToScheme(scheme.Scheme)) controller := &StorageCapabilityController{ - storageClassClient: storageClassClient, - storageClassLister: storageClassInformer.Lister(), - storageClassSynced: storageClassInformer.Informer().HasSynced, - csiDriverLister: csiDriverInformer.Lister(), - csiDriverSynced: csiDriverInformer.Informer().HasSynced, - snapshotSupported: snapshotSupported, - workQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"), - csiWorkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "csiDriver"), - } - - if snapshotSupported { - controller.snapshotClassClient = snapshotClassClient - controller.snapshotClassLister = snapshotClassInformer.Lister() - controller.snapshotClassSynced = snapshotClassInformer.Informer().HasSynced + storageClassClient: storageClassClient, + storageClassLister: storageClassInformer.Lister(), + storageClassSynced: storageClassInformer.Informer().HasSynced, + csiDriverLister: csiDriverInformer.Lister(), + csiDriverSynced: csiDriverInformer.Informer().HasSynced, + storageClassWorkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"), } storageClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ @@ -113,13 +88,10 @@ func NewController( } controller.enqueueStorageClass(newStorageClass) }, - DeleteFunc: controller.enqueueStorageClass, }) csiDriverInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.enqueueStorageClassByCSI, - UpdateFunc: nil, - DeleteFunc: controller.enqueueStorageClassByCSI, + AddFunc: controller.enqueueStorageClassByCSI, }) return controller @@ -131,7 +103,7 @@ func (c *StorageCapabilityController) Start(ctx context.Context) error { func (c *StorageCapabilityController) Run(threadCnt int, stopCh <-chan struct{}) error { defer utilruntime.HandleCrash() - defer c.workQueue.ShutDown() + defer c.storageClassWorkQueue.ShutDown() // Wait for the caches to be synced before starting workers klog.Info("Waiting for informer caches to sync") @@ -160,7 +132,7 @@ func (c *StorageCapabilityController) enqueueStorageClass(obj interface{}) { utilruntime.HandleError(err) return } - c.workQueue.Add(key) + c.storageClassWorkQueue.Add(key) } func (c *StorageCapabilityController) enqueueStorageClassByCSI(csi interface{}) { @@ -178,7 +150,7 @@ func (c *StorageCapabilityController) enqueueStorageClassByCSI(csi interface{}) } for _, obj := range objs { if obj.Provisioner == key { - c.workQueue.Add(obj.Name) + c.enqueueStorageClass(obj) } } return @@ -190,25 +162,25 @@ func (c *StorageCapabilityController) runWorker() { } func (c *StorageCapabilityController) processNextWorkItem() bool { - obj, shutdown := c.workQueue.Get() + obj, shutdown := c.storageClassWorkQueue.Get() if shutdown { return false } err := func(obj interface{}) error { - defer c.workQueue.Done(obj) + defer c.storageClassWorkQueue.Done(obj) var key string var ok bool if key, ok = obj.(string); !ok { - c.workQueue.Forget(obj) + c.storageClassWorkQueue.Forget(obj) utilruntime.HandleError(fmt.Errorf("expected string in workQueue but got %#v", obj)) return nil } if err := c.syncHandler(key); err != nil { - c.workQueue.AddRateLimited(key) + c.storageClassWorkQueue.AddRateLimited(key) return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) } - c.workQueue.Forget(obj) + c.storageClassWorkQueue.Forget(obj) klog.Infof("Successfully synced '%s'", key) return nil }(obj) @@ -232,55 +204,28 @@ func (c *StorageCapabilityController) syncHandler(key string) error { // Get StorageClass storageClass, err := c.storageClassLister.Get(name) - if err != nil { - // StorageClass has been deleted, delete StorageClassCapability and VolumeSnapshotClass - if errors.IsNotFound(err) && c.snapshotAllowed() { - err = c.deleteSnapshotClass(name) - if err != nil { - return err - } - } - return err - } //Cloning and volumeSnapshot support only available for CSI drivers. - withCapability := c.supportCapability(storageClass) - // Handle VolumeSnapshotClass with same name of StorageClass - // annotate "support-snapshot" of StorageClass - if c.snapshotAllowed() && withCapability { - _, err = c.snapshotClassLister.Get(name) - if err != nil { - // If VolumeSnapshotClass not exist, create it - if errors.IsNotFound(err) { - volumeSnapshotClassCreate := &snapshotv1beta1.VolumeSnapshotClass{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Driver: storageClass.Provisioner, - DeletionPolicy: snapshotv1beta1.VolumeSnapshotContentDelete, - } - _, err = c.snapshotClassClient.Create(context.Background(), volumeSnapshotClassCreate, metav1.CreateOptions{}) - if err != nil { - return err - } - } - } - } + isCSIStorage := c.hasCSIDriver(storageClass) - err = c.addStorageClassSnapshotAnnotation(storageClass, withCapability) + //Annotate storageClass + storageClassUpdated := storageClass.DeepCopy() + err = c.addStorageClassSnapshotAnnotation(storageClassUpdated, isCSIStorage) if err != nil { return err } - err = c.addCloneVolumeAnnotation(storageClass, withCapability) + err = c.addCloneVolumeAnnotation(storageClassUpdated, isCSIStorage) if err != nil { - return nil + return err } - _, err = c.storageClassClient.Update(context.Background(), storageClass, metav1.UpdateOptions{}) + _, err = c.storageClassClient.Update(context.Background(), storageClassUpdated, metav1.UpdateOptions{}) if err != nil { return err } return nil } -func (c *StorageCapabilityController) supportCapability(storageClass *storagev1.StorageClass) bool { +func (c *StorageCapabilityController) hasCSIDriver(storageClass *storagev1.StorageClass) bool { driver := storageClass.Provisioner if driver != "" { if _, err := c.csiDriverLister.Get(driver); err != nil { @@ -291,73 +236,29 @@ func (c *StorageCapabilityController) supportCapability(storageClass *storagev1. return false } -func (c *StorageCapabilityController) addStorageClassSnapshotAnnotation(storageClass *storagev1.StorageClass, snapshotSupported bool) error { - if snapshotSupported || !c.snapshotSupported { +func (c *StorageCapabilityController) addStorageClassSnapshotAnnotation(storageClass *storagev1.StorageClass, snapshotAllow bool) error { + if snapshotAllow { if storageClass.Annotations == nil { storageClass.Annotations = make(map[string]string) } - _, err := strconv.ParseBool(storageClass.Annotations[annotationSupportSnapshot]) - // err != nil means annotationSupportSnapshot is not illegal, include empty + _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowSnapshot]) + // err != nil means annotationAllowSnapshot is not illegal, include empty if err != nil { - storageClass.Annotations[annotationSupportSnapshot] = strconv.FormatBool(c.snapshotSupported) - } - } else { - if storageClass.Annotations != nil && c.snapshotSupported { - if _, ok := storageClass.Annotations[annotationSupportSnapshot]; ok { - delete(storageClass.Annotations, annotationSupportSnapshot) - } + storageClass.Annotations[annotationAllowSnapshot] = strconv.FormatBool(snapshotAllow) } } return nil } -func (c *StorageCapabilityController) addCloneVolumeAnnotation(storageClass *storagev1.StorageClass, cloneSupported bool) error { - if cloneSupported { +func (c *StorageCapabilityController) addCloneVolumeAnnotation(storageClass *storagev1.StorageClass, cloneAllow bool) error { + if cloneAllow { if storageClass.Annotations == nil { storageClass.Annotations = make(map[string]string) } - _, err := strconv.ParseBool(storageClass.Annotations[annotationSupportClone]) + _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowClone]) if err != nil { - storageClass.Annotations[annotationSupportClone] = strconv.FormatBool(cloneSupported) - } - } else { - if storageClass.Annotations != nil { - if _, ok := storageClass.Annotations[annotationSupportClone]; ok { - delete(storageClass.Annotations, annotationSupportClone) - } + storageClass.Annotations[annotationAllowClone] = strconv.FormatBool(cloneAllow) } } return nil } - -func (c *StorageCapabilityController) deleteSnapshotClass(name string) error { - if !c.snapshotAllowed() { - return nil - } - _, err := c.snapshotClassLister.Get(name) - if err != nil { - if errors.IsNotFound(err) { - return nil - } - return err - } - klog.Infof("Delete SnapshotClass %s", name) - return c.snapshotClassClient.Delete(context.Background(), name, metav1.DeleteOptions{}) -} - -func (c *StorageCapabilityController) snapshotAllowed() bool { - return c.snapshotSupported && c.snapshotClassClient != nil && c.snapshotClassLister != nil && c.snapshotClassSynced != nil -} - -func SnapshotSupported(discoveryInterface discovery.DiscoveryInterface) bool { - minVer := version.MustParseGeneric(minSnapshotSupportedVersion) - rawVer, err := discoveryInterface.ServerVersion() - if err != nil { - return false - } - ver, err := version.ParseSemantic(rawVer.String()) - if err != nil { - return false - } - return ver.AtLeast(minVer) -} diff --git a/pkg/controller/storage/capability/capability_controller_test.go b/pkg/controller/storage/capability/capability_controller_test.go index ef7292e59..d54ab3d66 100644 --- a/pkg/controller/storage/capability/capability_controller_test.go +++ b/pkg/controller/storage/capability/capability_controller_test.go @@ -26,9 +26,6 @@ import ( "testing" "time" - snapbeta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" - snapfake "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/fake" - snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions" storagev1 "k8s.io/api/storage/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -40,7 +37,6 @@ import ( "k8s.io/client-go/tools/cache" ksfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake" - ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" ) var ( @@ -51,17 +47,13 @@ type fixture struct { t *testing.T snapshotSupported bool // Clients - k8sClient *k8sfake.Clientset - snapshotClassClient *snapfake.Clientset - ksClient *ksfake.Clientset + k8sClient *k8sfake.Clientset + ksClient *ksfake.Clientset // Objects from here preload into NewSimpleFake. - storageObjects []runtime.Object // include StorageClass - snapshotClassObjects []runtime.Object - capabilityObjects []runtime.Object // include StorageClassCapability and ProvisionerCapability + storageObjects []runtime.Object // include StorageClass // Objects to put in the store. - storageClassLister []*storagev1.StorageClass - snapshotClassLister []*snapbeta1.VolumeSnapshotClass - csiDriverLister []*v1beta1.CSIDriver + storageClassLister []*storagev1.StorageClass + csiDriverLister []*v1beta1.CSIDriver // Actions expected to happen on the client. actions []core.Action } @@ -74,49 +66,34 @@ func newFixture(t *testing.T, snapshotSupported bool) *fixture { } func (f *fixture) newController() (*StorageCapabilityController, - k8sinformers.SharedInformerFactory, - ksinformers.SharedInformerFactory, - snapinformers.SharedInformerFactory) { + k8sinformers.SharedInformerFactory) { f.k8sClient = k8sfake.NewSimpleClientset(f.storageObjects...) - f.ksClient = ksfake.NewSimpleClientset(f.capabilityObjects...) - f.snapshotClassClient = snapfake.NewSimpleClientset(f.snapshotClassObjects...) k8sInformers := k8sinformers.NewSharedInformerFactory(f.k8sClient, noReSyncPeriodFunc()) - ksInformers := ksinformers.NewSharedInformerFactory(f.ksClient, noReSyncPeriodFunc()) - snapshotInformers := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc()) - c := NewController( f.k8sClient.StorageV1().StorageClasses(), k8sInformers.Storage().V1().StorageClasses(), k8sInformers.Storage().V1beta1().CSIDrivers(), - f.snapshotSupported, - f.snapshotClassClient.SnapshotV1beta1().VolumeSnapshotClasses(), - snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses(), ) for _, storageClass := range f.storageClassLister { _ = k8sInformers.Storage().V1().StorageClasses().Informer().GetIndexer().Add(storageClass) } - for _, snapshotClass := range f.snapshotClassLister { - _ = snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass) - } for _, csiDriver := range f.csiDriverLister { _ = k8sInformers.Storage().V1beta1().CSIDrivers().Informer().GetIndexer().Add(csiDriver) } - return c, k8sInformers, ksInformers, snapshotInformers + return c, k8sInformers } func (f *fixture) runController(scName string, startInformers bool, expectError bool) { - c, k8sI, crdI, snapI := f.newController() + c, k8sI := f.newController() if startInformers { stopCh := make(chan struct{}) defer close(stopCh) k8sI.Start(stopCh) - crdI.Start(stopCh) - snapI.Start(stopCh) } err := c.syncHandler(scName) @@ -127,9 +104,8 @@ func (f *fixture) runController(scName string, startInformers bool, expectError } var actions []core.Action - actions = append(actions, f.snapshotClassClient.Actions()...) actions = append(actions, f.k8sClient.Actions()...) - actions = append(actions, f.ksClient.Actions()...) + //actions = append(actions, f.ksClient.Actions()...) filerActions := filterInformerActions(actions) if len(filerActions) != len(f.actions) { f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions)) @@ -150,16 +126,6 @@ func (f *fixture) expectUpdateStorageClassAction(storageClass *storagev1.Storage schema.GroupVersionResource{Resource: "storageclasses"}, storageClass.Namespace, storageClass)) } -func (f *fixture) expectCreateSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) { - f.actions = append(f.actions, core.NewCreateAction( - schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass)) -} - -func (f *fixture) expectDeleteSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) { - f.actions = append(f.actions, core.NewDeleteAction( - schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass.Name)) -} - // filterInformerActions filters list and watch actions for testing resources. // Since list and watch don't change resource state we can filter it to lower // nose level in our tests. @@ -240,16 +206,6 @@ func newCSIDriver(name string) *v1beta1.CSIDriver { } } -func newSnapshotClass(storageClass *storagev1.StorageClass) *snapbeta1.VolumeSnapshotClass { - return &snapbeta1.VolumeSnapshotClass{ - ObjectMeta: v1.ObjectMeta{ - Name: storageClass.Name, - }, - Driver: storageClass.Provisioner, - DeletionPolicy: snapbeta1.VolumeSnapshotContentDelete, - } -} - func getKey(sc *storagev1.StorageClass, t *testing.T) string { key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(sc) if err != nil { @@ -263,46 +219,7 @@ func TestCreateStorageClass(t *testing.T) { fixture := newFixture(t, true) storageClass := newStorageClass("csi-example", "csi.example.com") storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "true", annotationSupportClone: "true"} - snapshotClass := newSnapshotClass(storageClass) - csiDriver := newCSIDriver("csi.example.com") - - // Objects exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) - - // Action expected - fixture.expectCreateSnapshotClassAction(snapshotClass) - fixture.expectUpdateStorageClassAction(storageClassUpdate) - - // Run test - fixture.run(getKey(storageClass, t)) -} - -func TestDeleteStorageClass(t *testing.T) { - storageClass := newStorageClass("csi-example", "csi.example.com") - snapshotClass := newSnapshotClass(storageClass) - - fixture := newFixture(t, true) - // Object exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass) - fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass) - - // Action expected - fixture.expectDeleteSnapshotClassAction(snapshotClass) - - // Run test - fixture.run(getKey(storageClass, t)) -} - -func TestCreateStorageClassNotSupportSnapshot(t *testing.T) { - // K8S version < 1.17.0 - fixture := newFixture(t, false) - storageClass := newStorageClass("csi-example", "csi.example.com") - storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "true"} + storageClassUpdate.Annotations = map[string]string{annotationAllowSnapshot: "true", annotationAllowClone: "true"} csiDriver := newCSIDriver("csi.example.com") // Objects exist @@ -320,10 +237,9 @@ func TestCreateStorageClassNotSupportSnapshot(t *testing.T) { func TestStorageClassHadAnnotation(t *testing.T) { fixture := newFixture(t, true) storageClass := newStorageClass("csi-example", "csi.example.com") - storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "false"} + storageClass.Annotations = map[string]string{annotationAllowSnapshot: "false", annotationAllowClone: "false"} storageClassUpdate := storageClass.DeepCopy() csiDriver := newCSIDriver("csi.example.com") - snapshotClass := newSnapshotClass(storageClass) //Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) @@ -331,7 +247,6 @@ func TestStorageClassHadAnnotation(t *testing.T) { fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) //Action expected - fixture.expectCreateSnapshotClassAction(snapshotClass) fixture.expectUpdateStorageClassAction(storageClassUpdate) //Run test @@ -341,36 +256,16 @@ func TestStorageClassHadAnnotation(t *testing.T) { func TestStorageClassHadOneAnnotation(t *testing.T) { fixture := newFixture(t, true) storageClass := newStorageClass("csi-example", "csi.example.com") - storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false"} + storageClass.Annotations = map[string]string{annotationAllowSnapshot: "false"} storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations[annotationSupportClone] = "true" + storageClassUpdate.Annotations[annotationAllowClone] = "true" csiDriver := newCSIDriver("csi.example.com") - snapshotClass := newSnapshotClass(storageClass) - //object exist + //Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) - //Action expected - fixture.expectCreateSnapshotClassAction(snapshotClass) - fixture.expectUpdateStorageClassAction(storageClassUpdate) - - //Run test - fixture.run(getKey(storageClass, t)) -} - -func TestDeleteCSIDriver(t *testing.T) { - fixture := newFixture(t, true) - storageClass := newStorageClass("csi-example", "csi.example.com") - storageClass.Annotations = map[string]string{annotationSupportSnapshot: "false", annotationSupportClone: "false"} - storageClassUpdate := storageClass.DeepCopy() - storageClassUpdate.Annotations = map[string]string{} - - //object exist - fixture.storageObjects = append(fixture.storageObjects, storageClass) - fixture.storageClassLister = append(fixture.storageClassLister, storageClass) - //Action expected fixture.expectUpdateStorageClassAction(storageClassUpdate) diff --git a/pkg/controller/storage/snapshot/snapshot_controller.go b/pkg/controller/storage/snapshot/snapshot_controller.go new file mode 100644 index 000000000..90808e28c --- /dev/null +++ b/pkg/controller/storage/snapshot/snapshot_controller.go @@ -0,0 +1,195 @@ +/* + + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package snapshot + +import ( + "context" + "fmt" + "time" + + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" + snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/typed/volumesnapshot/v1beta1" + snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/volumesnapshot/v1beta1" + snapshotlisters "github.com/kubernetes-csi/external-snapshotter/client/v3/listers/volumesnapshot/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + storageinformersv1 "k8s.io/client-go/informers/storage/v1" + "k8s.io/client-go/kubernetes/scheme" + storagelistersv1 "k8s.io/client-go/listers/storage/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog" + + crdscheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" +) + +type VolumeSnapshotClassController struct { + storageClassLister storagelistersv1.StorageClassLister + storageClassSynced cache.InformerSynced + snapshotClassClient snapshotclient.VolumeSnapshotClassInterface + snapshotClassLister snapshotlisters.VolumeSnapshotClassLister + snapshotClassSynced cache.InformerSynced + + snapshotClassWorkQueue workqueue.RateLimitingInterface +} + +//This controller is responseible to watch StorageClass +//When storageClass has created ,create snapshot class +func NewController( + storageClassInformer storageinformersv1.StorageClassInformer, + snapshotClassClient snapshotclient.VolumeSnapshotClassInterface, + snapshotClassInformer snapinformers.VolumeSnapshotClassInformer, +) *VolumeSnapshotClassController { + + utilruntime.Must(crdscheme.AddToScheme(scheme.Scheme)) + + controller := &VolumeSnapshotClassController{ + storageClassLister: storageClassInformer.Lister(), + storageClassSynced: storageClassInformer.Informer().HasSynced, + snapshotClassClient: snapshotClassClient, + snapshotClassLister: snapshotClassInformer.Lister(), + snapshotClassSynced: snapshotClassInformer.Informer().HasSynced, + snapshotClassWorkQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "SnapshotClass"), + } + + storageClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: controller.enqueueStorageClass, + DeleteFunc: controller.enqueueStorageClass, + }) + + return controller +} + +func (c *VolumeSnapshotClassController) Start(ctx context.Context) error { + return c.Run(5, ctx.Done()) +} + +func (c *VolumeSnapshotClassController) Run(threadCnt int, stopCh <-chan struct{}) error { + defer utilruntime.HandleCrash() + defer c.snapshotClassWorkQueue.ShutDown() + + klog.Info("Waiting for informer cache to sync.") + cacheSyncs := []cache.InformerSynced{ + c.storageClassSynced, + c.snapshotClassSynced, + } + + if ok := cache.WaitForCacheSync(stopCh, cacheSyncs...); !ok { + return fmt.Errorf("failed to wait for caches to syne") + } + + for i := 0; i < threadCnt; i++ { + go wait.Until(c.runWorker, time.Second, stopCh) + } + klog.Info("Started workers") + <-stopCh + klog.Info("Shutting down workers") + return nil +} + +func (c *VolumeSnapshotClassController) enqueueStorageClass(obj interface{}) { + var key string + var err error + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { + utilruntime.HandleError(err) + return + } + c.snapshotClassWorkQueue.Add(key) +} + +func (c *VolumeSnapshotClassController) runWorker() { + for c.processNextWorkItem() { + } +} + +func (c *VolumeSnapshotClassController) processNextWorkItem() bool { + obj, shutdown := c.snapshotClassWorkQueue.Get() + if shutdown { + return false + } + + err := func(obj interface{}) error { + defer c.snapshotClassWorkQueue.Done(obj) + var key string + var ok bool + if key, ok = obj.(string); !ok { + c.snapshotClassWorkQueue.Forget(obj) + utilruntime.HandleError(fmt.Errorf("expected string in workQueue but got %#v", obj)) + return nil + } + if err := c.syncHandler(key); err != nil { + c.snapshotClassWorkQueue.AddRateLimited(key) + return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) + } + c.snapshotClassWorkQueue.Forget(obj) + klog.Infof("Successfully synced '%s'", key) + return nil + }(obj) + + if err != nil { + utilruntime.HandleError(err) + return true + } + return true +} + +func (c *VolumeSnapshotClassController) syncHandler(key string) error { + _, name, err := cache.SplitMetaNamespaceKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("invalid resource key: %s", key)) + return nil + } + + storageClass, err := c.storageClassLister.Get(name) + if err != nil { + // StorageClass has been deleted, delete VolumeSnapshotClass + if errors.IsNotFound(err) { + err = c.deleteSnapshotClass(name) + } + return err + } + + // If VolumeSnapshotClass not exist, create it + _, err = c.snapshotClassLister.Get(name) + if err != nil { + if errors.IsNotFound(err) { + volumeSnapshotClassCreate := &snapshotv1beta1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Driver: storageClass.Provisioner, + DeletionPolicy: snapshotv1beta1.VolumeSnapshotContentDelete, + } + _, err = c.snapshotClassClient.Create(context.Background(), volumeSnapshotClassCreate, metav1.CreateOptions{}) + } + } + return err +} + +func (c *VolumeSnapshotClassController) deleteSnapshotClass(name string) error { + _, err := c.snapshotClassLister.Get(name) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + return err + } + klog.Infof("Delete SnapshotClass %s", name) + return c.snapshotClassClient.Delete(context.Background(), name, metav1.DeleteOptions{}) +} diff --git a/pkg/controller/storage/snapshot/snapshot_controller_test.go b/pkg/controller/storage/snapshot/snapshot_controller_test.go new file mode 100644 index 000000000..82b4bd660 --- /dev/null +++ b/pkg/controller/storage/snapshot/snapshot_controller_test.go @@ -0,0 +1,258 @@ +/* + + Copyright 2021 The KubeSphere Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +package snapshot + +import ( + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + snapbeta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" + snapfake "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/fake" + snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions" + storagev1 "k8s.io/api/storage/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/diff" + k8sinformers "k8s.io/client-go/informers" + k8sfake "k8s.io/client-go/kubernetes/fake" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + ksfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake" +) + +var ( + noReSyncPeriodFunc = func() time.Duration { return 0 } +) + +type fixture struct { + t *testing.T + snapshotSupported bool + // Clients + k8sClient *k8sfake.Clientset + snapshotClassClient *snapfake.Clientset + ksClient *ksfake.Clientset + // Objects from here preload into NewSimpleFake. + storageObjects []runtime.Object // include StorageClass + snapshotClassObjects []runtime.Object + // Objects to put in the store. + storageClassLister []*storagev1.StorageClass + snapshotClassLister []*snapbeta1.VolumeSnapshotClass + // Actions expected to happen on the client. + actions []core.Action +} + +func newFixture(t *testing.T) *fixture { + return &fixture{t: t} +} + +func (f *fixture) newController() (*VolumeSnapshotClassController, k8sinformers.SharedInformerFactory, snapinformers.SharedInformerFactory) { + f.k8sClient = k8sfake.NewSimpleClientset(f.storageObjects...) + f.snapshotClassClient = snapfake.NewSimpleClientset(f.snapshotClassObjects...) + + k8sInformers := k8sinformers.NewSharedInformerFactory(f.k8sClient, noReSyncPeriodFunc()) + snapshotInformers := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc()) + + c := NewController( + k8sInformers.Storage().V1().StorageClasses(), + f.snapshotClassClient.SnapshotV1beta1().VolumeSnapshotClasses(), + snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses(), + ) + + for _, storageClass := range f.storageClassLister { + _ = k8sInformers.Storage().V1().StorageClasses().Informer().GetIndexer().Add(storageClass) + } + for _, snapshotClass := range f.snapshotClassLister { + _ = snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass) + } + + return c, k8sInformers, snapshotInformers +} + +func (f *fixture) runController(scName string, startInformers bool, expectError bool) { + c, k8sI, snapI := f.newController() + + if startInformers { + stopCh := make(chan struct{}) + defer close(stopCh) + k8sI.Start(stopCh) + snapI.Start(stopCh) + } + + err := c.syncHandler(scName) + if !expectError && err != nil { + f.t.Errorf("error syncing: %v", err) + } else if expectError && err == nil { + f.t.Error("expected error syncing, got nil") + } + + var actions []core.Action + actions = append(actions, f.snapshotClassClient.Actions()...) + actions = append(actions, f.k8sClient.Actions()...) + //actions = append(actions, f.ksClient.Actions()...) + filerActions := filterInformerActions(actions) + if len(filerActions) != len(f.actions) { + f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions)) + return + } + for i, action := range filerActions { + expectedAction := f.actions[i] + checkAction(expectedAction, action, f.t) + } +} + +func (f *fixture) run(scName string) { + f.runController(scName, true, false) +} + +func (f *fixture) expectCreateSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) { + f.actions = append(f.actions, core.NewCreateAction( + schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass)) +} + +func (f *fixture) expectDeleteSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) { + f.actions = append(f.actions, core.NewDeleteAction( + schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass.Name)) +} + +// filterInformerActions filters list and watch actions for testing resources. +// Since list and watch don't change resource state we can filter it to lower +// nose level in our tests. +func filterInformerActions(actions []core.Action) []core.Action { + var ret []core.Action + for _, action := range actions { + if action.GetVerb() == "list" || action.GetVerb() == "watch" { + continue + } + ret = append(ret, action) + } + return ret +} + +// checkAction verifies that expected and actual actions are equal and both have +// same attached resources +func checkAction(expected, actual core.Action, t *testing.T) { + if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) { + t.Errorf("\nExpected\n\t%#v\ngot\n\t%#v", expected, actual) + return + } + + if reflect.TypeOf(actual) != reflect.TypeOf(expected) { + t.Errorf("Action has wrong type. Expected: %t. Got: %t", expected, actual) + return + } + + switch a := actual.(type) { + case core.CreateActionImpl: + e, _ := expected.(core.CreateActionImpl) + expObject := e.GetObject() + object := a.GetObject() + if difference := cmp.Diff(object, expObject); len(difference) > 0 { + t.Errorf("[CreateAction] %T differ (-got, +want): %s", expObject, difference) + } + case core.UpdateActionImpl: + e, _ := expected.(core.UpdateActionImpl) + expObject := e.GetObject() + object := a.GetObject() + if difference := cmp.Diff(object, expObject); len(difference) > 0 { + t.Errorf("[UpdateAction] %T differ (-got, +want): %s", expObject, difference) + } + case core.PatchActionImpl: + e, _ := expected.(core.PatchActionImpl) + expPatch := e.GetPatch() + patch := a.GetPatch() + if !reflect.DeepEqual(expPatch, patch) { + t.Errorf("Action %s %s has wrong patch\nDiff:\n %s", + a.GetVerb(), a.GetResource().Resource, diff.ObjectGoPrintSideBySide(expPatch, patch)) + } + case core.DeleteActionImpl: + e, _ := expected.(core.DeleteActionImpl) + if difference := cmp.Diff(e.Name, a.Name); len(difference) > 0 { + t.Errorf("[UpdateAction] %T differ (-got, +want): %s", e.Name, difference) + } + default: + t.Errorf("Uncaptured Action %s %s, you should explicitly add a case to capture it", + actual.GetVerb(), actual.GetResource().Resource) + } +} + +func newStorageClass(name string) *storagev1.StorageClass { + isExpansion := true + return &storagev1.StorageClass{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + }, + AllowVolumeExpansion: &isExpansion, + } +} + +func newSnapshotClass(storageClass *storagev1.StorageClass) *snapbeta1.VolumeSnapshotClass { + return &snapbeta1.VolumeSnapshotClass{ + ObjectMeta: v1.ObjectMeta{ + Name: storageClass.Name, + }, + Driver: storageClass.Provisioner, + DeletionPolicy: snapbeta1.VolumeSnapshotContentDelete, + } +} + +func getKey(sc *storagev1.StorageClass, t *testing.T) string { + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(sc) + if err != nil { + t.Errorf("Unexpected error getting key for %v: %v", sc.Name, err) + return "" + } + return key +} + +func TestCreateStorageClass(t *testing.T) { + fixture := newFixture(t) + storageClass := newStorageClass("csi-example") + snapshotClass := newSnapshotClass(storageClass) + + // Objects exist + fixture.storageObjects = append(fixture.storageObjects, storageClass) + fixture.storageClassLister = append(fixture.storageClassLister, storageClass) + + // Action expected + fixture.expectCreateSnapshotClassAction(snapshotClass) + + // Run test + fixture.run(getKey(storageClass, t)) +} + +func TestDeleteStorageClass(t *testing.T) { + storageClass := newStorageClass("csi-example") + snapshotClass := newSnapshotClass(storageClass) + + fixture := newFixture(t) + // Object exist + fixture.storageObjects = append(fixture.storageObjects, storageClass) + fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass) + fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass) + + // Action expected + fixture.expectDeleteSnapshotClassAction(snapshotClass) + + // Run test + fixture.run(getKey(storageClass, t)) +} From b3123547a47547ba3cd2a81402f3679f1764ad65 Mon Sep 17 00:00:00 2001 From: f10atin9 Date: Mon, 23 Aug 2021 18:01:28 +0800 Subject: [PATCH 3/4] Capability controller can add annotations, no matter whether the storageClass has CSIDriver. Snapshot controller will create volumeSnapshotClass based on allowSnapshot annotation. Signed-off-by: f10atin9 --- .../capability/capability_controller.go | 30 ++++++------- .../storage/snapshot/snapshot_controller.go | 43 ++++++++++++++----- .../snapshot/snapshot_controller_test.go | 1 + 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/pkg/controller/storage/capability/capability_controller.go b/pkg/controller/storage/capability/capability_controller.go index 9acd35bdf..bdbaef632 100644 --- a/pkg/controller/storage/capability/capability_controller.go +++ b/pkg/controller/storage/capability/capability_controller.go @@ -237,28 +237,24 @@ func (c *StorageCapabilityController) hasCSIDriver(storageClass *storagev1.Stora } func (c *StorageCapabilityController) addStorageClassSnapshotAnnotation(storageClass *storagev1.StorageClass, snapshotAllow bool) error { - if snapshotAllow { - if storageClass.Annotations == nil { - storageClass.Annotations = make(map[string]string) - } - _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowSnapshot]) - // err != nil means annotationAllowSnapshot is not illegal, include empty - if err != nil { - storageClass.Annotations[annotationAllowSnapshot] = strconv.FormatBool(snapshotAllow) - } + if storageClass.Annotations == nil { + storageClass.Annotations = make(map[string]string) + } + _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowSnapshot]) + // err != nil means annotationAllowSnapshot is not illegal, include empty + if err != nil { + storageClass.Annotations[annotationAllowSnapshot] = strconv.FormatBool(snapshotAllow) } return nil } func (c *StorageCapabilityController) addCloneVolumeAnnotation(storageClass *storagev1.StorageClass, cloneAllow bool) error { - if cloneAllow { - if storageClass.Annotations == nil { - storageClass.Annotations = make(map[string]string) - } - _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowClone]) - if err != nil { - storageClass.Annotations[annotationAllowClone] = strconv.FormatBool(cloneAllow) - } + if storageClass.Annotations == nil { + storageClass.Annotations = make(map[string]string) + } + _, err := strconv.ParseBool(storageClass.Annotations[annotationAllowClone]) + if err != nil { + storageClass.Annotations[annotationAllowClone] = strconv.FormatBool(cloneAllow) } return nil } diff --git a/pkg/controller/storage/snapshot/snapshot_controller.go b/pkg/controller/storage/snapshot/snapshot_controller.go index 90808e28c..696d10da4 100644 --- a/pkg/controller/storage/snapshot/snapshot_controller.go +++ b/pkg/controller/storage/snapshot/snapshot_controller.go @@ -21,8 +21,11 @@ package snapshot import ( "context" "fmt" + "strconv" "time" + storagev1 "k8s.io/api/storage/v1" + snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/client/v3/apis/volumesnapshot/v1beta1" snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned/typed/volumesnapshot/v1beta1" snapinformers "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions/volumesnapshot/v1beta1" @@ -41,6 +44,8 @@ import ( crdscheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" ) +const annotationAllowSnapshot = "storageclass.kubesphere.io/allow-snapshot" + type VolumeSnapshotClassController struct { storageClassLister storagelistersv1.StorageClassLister storageClassSynced cache.InformerSynced @@ -71,7 +76,15 @@ func NewController( } storageClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{ - AddFunc: controller.enqueueStorageClass, + AddFunc: controller.enqueueStorageClass, + UpdateFunc: func(old, new interface{}) { + newStorageClass := new.(*storagev1.StorageClass) + oldStorageClass := old.(*storagev1.StorageClass) + if newStorageClass.ResourceVersion == oldStorageClass.ResourceVersion { + return + } + controller.enqueueStorageClass(newStorageClass) + }, DeleteFunc: controller.enqueueStorageClass, }) @@ -167,19 +180,27 @@ func (c *VolumeSnapshotClassController) syncHandler(key string) error { return err } - // If VolumeSnapshotClass not exist, create it - _, err = c.snapshotClassLister.Get(name) - if err != nil { - if errors.IsNotFound(err) { - volumeSnapshotClassCreate := &snapshotv1beta1.VolumeSnapshotClass{ - ObjectMeta: metav1.ObjectMeta{Name: name}, - Driver: storageClass.Provisioner, - DeletionPolicy: snapshotv1beta1.VolumeSnapshotContentDelete, + if storageClass.Annotations != nil { + if annotationSnap, ok := storageClass.Annotations[annotationAllowSnapshot]; ok { + allowSnapshot, err := strconv.ParseBool(annotationSnap) + if err == nil && allowSnapshot { + // If VolumeSnapshotClass not exist, create it + _, err = c.snapshotClassLister.Get(name) + if err != nil { + if errors.IsNotFound(err) { + volumeSnapshotClassCreate := &snapshotv1beta1.VolumeSnapshotClass{ + ObjectMeta: metav1.ObjectMeta{Name: name}, + Driver: storageClass.Provisioner, + DeletionPolicy: snapshotv1beta1.VolumeSnapshotContentDelete, + } + _, err = c.snapshotClassClient.Create(context.Background(), volumeSnapshotClassCreate, metav1.CreateOptions{}) + } + } } - _, err = c.snapshotClassClient.Create(context.Background(), volumeSnapshotClassCreate, metav1.CreateOptions{}) + return err } } - return err + return nil } func (c *VolumeSnapshotClassController) deleteSnapshotClass(name string) error { diff --git a/pkg/controller/storage/snapshot/snapshot_controller_test.go b/pkg/controller/storage/snapshot/snapshot_controller_test.go index 82b4bd660..04ae8e1fb 100644 --- a/pkg/controller/storage/snapshot/snapshot_controller_test.go +++ b/pkg/controller/storage/snapshot/snapshot_controller_test.go @@ -227,6 +227,7 @@ func getKey(sc *storagev1.StorageClass, t *testing.T) string { func TestCreateStorageClass(t *testing.T) { fixture := newFixture(t) storageClass := newStorageClass("csi-example") + storageClass.Annotations = map[string]string{annotationAllowSnapshot: "true"} snapshotClass := newSnapshotClass(storageClass) // Objects exist From a8d36f232acf4022220934be04ea883eb19a5364 Mon Sep 17 00:00:00 2001 From: f10atin9 Date: Tue, 24 Aug 2021 11:46:56 +0800 Subject: [PATCH 4/4] rename snapshot_controller to snapshotclass_controller Signed-off-by: f10atin9 --- cmd/controller-manager/app/controllers.go | 4 ++-- .../storage/capability/capability_controller.go | 4 ++-- .../capability/capability_controller_test.go | 13 ++++++------- .../snapshotclass_controller.go} | 6 +++--- .../snapshotclass_controller_test.go} | 3 +-- 5 files changed, 14 insertions(+), 16 deletions(-) rename pkg/controller/storage/{snapshot/snapshot_controller.go => snapshotclass/snapshotclass_controller.go} (98%) rename pkg/controller/storage/{snapshot/snapshot_controller_test.go => snapshotclass/snapshotclass_controller_test.go} (99%) diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index 54ffd9ffb..709a6999b 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -23,7 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/kubefed/pkg/controller/util" - "kubesphere.io/kubesphere/pkg/controller/storage/snapshot" + "kubesphere.io/kubesphere/pkg/controller/storage/snapshotclass" iamv1alpha2 "kubesphere.io/api/iam/v1alpha2" @@ -103,7 +103,7 @@ func addControllers( kubernetesInformer.Storage().V1beta1().CSIDrivers(), ) - volumeSnapshotController := snapshot.NewController( + volumeSnapshotController := snapshotclass.NewController( kubernetesInformer.Storage().V1().StorageClasses(), client.Snapshot().SnapshotV1beta1().VolumeSnapshotClasses(), informerFactory.SnapshotSharedInformerFactory().Snapshot().V1beta1().VolumeSnapshotClasses(), diff --git a/pkg/controller/storage/capability/capability_controller.go b/pkg/controller/storage/capability/capability_controller.go index bdbaef632..8d205b312 100644 --- a/pkg/controller/storage/capability/capability_controller.go +++ b/pkg/controller/storage/capability/capability_controller.go @@ -205,10 +205,10 @@ func (c *StorageCapabilityController) syncHandler(key string) error { // Get StorageClass storageClass, err := c.storageClassLister.Get(name) - //Cloning and volumeSnapshot support only available for CSI drivers. + // Cloning and volumeSnapshot support only available for CSI drivers. isCSIStorage := c.hasCSIDriver(storageClass) - //Annotate storageClass + // Annotate storageClass storageClassUpdated := storageClass.DeepCopy() err = c.addStorageClassSnapshotAnnotation(storageClassUpdated, isCSIStorage) if err != nil { diff --git a/pkg/controller/storage/capability/capability_controller_test.go b/pkg/controller/storage/capability/capability_controller_test.go index d54ab3d66..0ebb7d684 100644 --- a/pkg/controller/storage/capability/capability_controller_test.go +++ b/pkg/controller/storage/capability/capability_controller_test.go @@ -105,7 +105,6 @@ func (f *fixture) runController(scName string, startInformers bool, expectError var actions []core.Action actions = append(actions, f.k8sClient.Actions()...) - //actions = append(actions, f.ksClient.Actions()...) filerActions := filterInformerActions(actions) if len(filerActions) != len(f.actions) { f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions)) @@ -241,15 +240,15 @@ func TestStorageClassHadAnnotation(t *testing.T) { storageClassUpdate := storageClass.DeepCopy() csiDriver := newCSIDriver("csi.example.com") - //Object exist + // Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) - //Action expected + // Action expected fixture.expectUpdateStorageClassAction(storageClassUpdate) - //Run test + // Run test fixture.run(getKey(storageClass, t)) } @@ -261,14 +260,14 @@ func TestStorageClassHadOneAnnotation(t *testing.T) { storageClassUpdate.Annotations[annotationAllowClone] = "true" csiDriver := newCSIDriver("csi.example.com") - //Object exist + // Object exist fixture.storageObjects = append(fixture.storageObjects, storageClass) fixture.storageClassLister = append(fixture.storageClassLister, storageClass) fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver) - //Action expected + // Action expected fixture.expectUpdateStorageClassAction(storageClassUpdate) - //Run test + // Run test fixture.run(getKey(storageClass, t)) } diff --git a/pkg/controller/storage/snapshot/snapshot_controller.go b/pkg/controller/storage/snapshotclass/snapshotclass_controller.go similarity index 98% rename from pkg/controller/storage/snapshot/snapshot_controller.go rename to pkg/controller/storage/snapshotclass/snapshotclass_controller.go index 696d10da4..e341ebc9c 100644 --- a/pkg/controller/storage/snapshot/snapshot_controller.go +++ b/pkg/controller/storage/snapshotclass/snapshotclass_controller.go @@ -16,7 +16,7 @@ */ -package snapshot +package snapshotclass import ( "context" @@ -56,8 +56,8 @@ type VolumeSnapshotClassController struct { snapshotClassWorkQueue workqueue.RateLimitingInterface } -//This controller is responseible to watch StorageClass -//When storageClass has created ,create snapshot class +// This controller is responsible to watch StorageClass +// When storageClass has created ,create snapshot class func NewController( storageClassInformer storageinformersv1.StorageClassInformer, snapshotClassClient snapshotclient.VolumeSnapshotClassInterface, diff --git a/pkg/controller/storage/snapshot/snapshot_controller_test.go b/pkg/controller/storage/snapshotclass/snapshotclass_controller_test.go similarity index 99% rename from pkg/controller/storage/snapshot/snapshot_controller_test.go rename to pkg/controller/storage/snapshotclass/snapshotclass_controller_test.go index 04ae8e1fb..db8ea4643 100644 --- a/pkg/controller/storage/snapshot/snapshot_controller_test.go +++ b/pkg/controller/storage/snapshotclass/snapshotclass_controller_test.go @@ -16,7 +16,7 @@ */ -package snapshot +package snapshotclass import ( "reflect" @@ -108,7 +108,6 @@ func (f *fixture) runController(scName string, startInformers bool, expectError var actions []core.Action actions = append(actions, f.snapshotClassClient.Actions()...) actions = append(actions, f.k8sClient.Actions()...) - //actions = append(actions, f.ksClient.Actions()...) filerActions := filterInformerActions(actions) if len(filerActions) != len(f.actions) { f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions))