capability for non CSI storage

cmd/controller-manager/app/controllers.go

@@ -17,7 +17,6 @@ limitations under the License.
 package app
 
 import (
-	"fmt"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog"
@@ -134,14 +133,14 @@ func addControllers(
 	}
 
 	storageCapabilityController := capability.NewController(
-		client.Kubernetes(),
+		client.KubeSphere().StorageV1alpha1().StorageClassCapabilities(),
-		client.KubeSphere(),
+		kubesphereInformer.Storage().V1alpha1(),
+		client.Kubernetes().StorageV1().StorageClasses(),
 		kubernetesInformer.Storage().V1().StorageClasses(),
+		capability.SnapshotSupported(client.Kubernetes().Discovery()),
+		client.Snapshot().SnapshotV1beta1().VolumeSnapshotClasses(),
 		informerFactory.SnapshotSharedInformerFactory().Snapshot().V1beta1().VolumeSnapshotClasses(),
-		kubesphereInformer.Storage().V1alpha1().StorageClassCapabilities(),
+		kubernetesInformer.Storage().V1beta1().CSIDrivers(),
-		func(storageClassProvisioner string) string {
-			return fmt.Sprintf(capability.CSIAddressFormat, storageClassProvisioner)
-		},
 	)
 
 	volumeExpansionController := expansion.NewVolumeExpansionController(
@@ -265,6 +264,7 @@ func addControllers(
 		"job-controller":                jobController,
 		"s2ibinary-controller":          s2iBinaryController,
 		"s2irun-controller":             s2iRunController,
+		"storagecapability-controller":  storageCapabilityController,
 		"volumeexpansion-controller":    volumeExpansionController,
 		"user-controller":               userController,
 		"cluster-controller":            clusterController,
@@ -281,10 +281,6 @@ func addControllers(
 		controllers["devopscredential-controller"] = devopsCredentialController
 	}
 
-	if storageCapabilityController.IsValidKubernetesVersion() {
-		controllers["storagecapability-controller"] = storageCapabilityController
-	}
-
 	if multiClusterEnabled {
 		controllers["globalrole-controller"] = globalRoleController
 		controllers["workspacerole-controller"] = workspaceRoleController

config/crds/storage.kubesphere.io_provisionercapabilities.yaml

@@ -0,0 +1,99 @@
+# according to https://kubernetes.io/blog/2018/07/12/resizing-persistent-volumes-using-kubernetes/,
+# and https://kubernetes.io/docs/concepts/storage/storage-classes/#allow-volume-expansion
+# volume expansion support for the following in-tree volume plugins:
+# AWS-EBS, GCE-PD, Azure Disk, Azure File, Glusterfs, Cinder, Portworx, and Ceph RBD.
+# online file system expansion support for the following in-tree plugin
+# GCE-PD, AWS-EBS, Cinder, and Ceph RBD
+
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: provisionercapabilities.storage.kubesphere.io
+spec:
+  group: storage.kubesphere.io
+  version: v1alpha1
+  preserveUnknownFields: false
+  names:
+    plural: provisionercapabilities
+    singular: provisionercapability
+    kind: ProvisionerCapability
+    shortNames:
+      - pcap
+  scope: Cluster
+  additionalPrinterColumns:
+    - name: Provisioner
+      type: string
+      description: The provisioner name should be the same as name
+      JSONPath: .spec.pluginInfo.name
+    - name: Expand
+      type: string
+      JSONPath: .spec.features.volume.expandMode
+    - name: Age
+      type: date
+      JSONPath: .metadata.creationTimestamp
+  validation:
+    openAPIV3Schema:
+      required:
+        - spec
+      type: object
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation of an object.'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this object represents.'
+          type: string
+        spec:
+          type: object
+          description: 'spec defines the desired characteristics of obejct'
+          properties:
+            pluginInfo:
+              type: object
+              description: 'Plugininfo represents plugin metadata'
+              properties:
+                name:
+                  description: 'provisioner name'
+                  type: string
+                version:
+                  description: 'plugin version'
+                  type: string
+            features:
+              type: object
+              description: 'Features represents plugin capability'
+              properties:
+                topology:
+                  description: 'topology determines whether a provisioner support topology by looking up GetPluginCapabilities.PluginCapability'
+                  type: boolean
+                volume:
+                  type: object
+                  description: 'Volume represents whether plugin supports volume features'
+                  properties:
+                    create:
+                      description: 'Determined by ControllerGetCapabilities in ControllerServer'
+                      type: boolean
+                    attach:
+                      description: 'Determined by ControllerGetCapabilities in ControllerServer'
+                      type: boolean
+                    list:
+                      description: 'Determined by ControllerGetCapabilities in ControllerServer'
+                      type: boolean
+                    clone:
+                      description: 'Determined by ControllerGetCapabilities in ControllerServer'
+                      type: boolean
+                    stats:
+                      description: 'Determined by NodeGetCapabilities in NodeServer'
+                      type: boolean
+                    expandMode:
+                      description: 'Determined by GetPluginCapabilities in IdentityServer'
+                      type: string
+                      items:
+                        type: string
+                        enum: ["UNKNOWN", "OFFLINE", "ONLINE"]
+                snapshot:
+                  type: object
+                  description: 'Snapshot represents whether plugin supports snapshot features'
+                  properties:
+                    create:
+                      type: boolean
+                    list:
+                      type: boolean

config/crds/storage.kubesphere.io_storageclasscapabilities.yaml

@@ -19,6 +19,12 @@ spec:
     - name: Volume
       type: boolean
       JSONPath: .spec.features.volume.create
+    - name: Expand
+      type: string
+      JSONPath: .spec.features.volume.expandMode
+    - name: Clone
+      type: boolean
+      JSONPath: .spec.features.volume.clone
     - name: Snapshot
       type: boolean
       JSONPath: .spec.features.snapshot.create

config/default/provisonercapability/aws-ebs.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-aws-ebs
+spec:
+  pluginInfo:
+    name: kubernetes.io/aws-ebs
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: ONLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/azure-disk.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-azure-disk
+spec:
+  pluginInfo:
+    name: kubernetes.io/azure-disk
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: OFFLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/azure-file.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-azure-file
+spec:
+  pluginInfo:
+    name: kubernetes.io/azure-file
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: OFFLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/cinder.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-cinder
+spec:
+  pluginInfo:
+    name: kubernetes.io/cinder
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: ONLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/gce-pd.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-gce-pd
+spec:
+  pluginInfo:
+    name: kubernetes.io/gce-pd
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: ONLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/glusterfs.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-glusterfs
+spec:
+  pluginInfo:
+    name: kubernetes.io/glusterfs
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: OFFLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/portworx-volume.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-portworx-volume
+spec:
+  pluginInfo:
+    name: kubernetes.io/portworx-volume
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: OFFLINE
+    snapshot:
+      create: false
+      list: false

config/default/provisonercapability/rbd.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-rbd
+spec:
+  pluginInfo:
+    name: kubernetes.io/rbd
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: false
+      expandMode: ONLINE
+    snapshot:
+      create: false
+      list: false

config/samples/storage_v1alpha1_provisionercapability.yaml

@@ -0,0 +1,20 @@
+apiVersion: storage.kubesphere.io/v1alpha1
+kind: ProvisionerCapability
+metadata:
+  name: kubernetes-io-no-provisioner
+spec:
+  pluginInfo:
+    name: kubernetes.io/no-provisioner
+    version: ""
+  features:
+    topology: false
+    volume:
+      create: true
+      attach: true
+      clone: false
+      list: false
+      stats: fasle
+      expandMode: OFFLINE
+    snapshot:
+      create: fasle
+      list: false

pkg/apis/storage/v1alpha1/capability_types.go

@@ -28,6 +28,35 @@ const (
 	ExpandModeOnline  ExpandMode = "ONLINE"
 )
 
+// VolumeFeature describe volume features
+type VolumeFeature struct {
+	Create bool       `json:"create"`
+	Attach bool       `json:"attach"`
+	List   bool       `json:"list"`
+	Clone  bool       `json:"clone"`
+	Stats  bool       `json:"stats"`
+	Expand ExpandMode `json:"expandMode"`
+}
+
+// SnapshotFeature describe snapshot features
+type SnapshotFeature struct {
+	Create bool `json:"create"`
+	List   bool `json:"list"`
+}
+
+// CapabilityFeatures describe storage features
+type CapabilityFeatures struct {
+	Topology bool            `json:"topology"`
+	Volume   VolumeFeature   `json:"volume"`
+	Snapshot SnapshotFeature `json:"snapshot"`
+}
+
+// PluginInfo describes plugin info
+type PluginInfo struct {
+	Name    string `json:"name"`
+	Version string `json:"version"`
+}
+
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +genclient:noStatus
@@ -45,30 +74,7 @@ type StorageClassCapability struct {
 // StorageClassCapabilitySpec defines the desired state of StorageClassCapability
 type StorageClassCapabilitySpec struct {
 	Provisioner string             `json:"provisioner"`
-	Features    StorageClassCapabilitySpecFeatures `json:"features"`
+	Features    CapabilityFeatures `json:"features"`
-}
-
-// StorageClassCapabilitySpecFeatures describe storage class features
-type StorageClassCapabilitySpecFeatures struct {
-	Topology bool                                       `json:"topology"`
-	Volume   StorageClassCapabilitySpecFeaturesVolume   `json:"volume"`
-	Snapshot StorageClassCapabilitySpecFeaturesSnapshot `json:"snapshot"`
-}
-
-// StorageClassCapabilitySpecFeaturesVolume describe volume features
-type StorageClassCapabilitySpecFeaturesVolume struct {
-	Create bool       `json:"create"`
-	Attach bool       `json:"attach"`
-	List   bool       `json:"list"`
-	Clone  bool       `json:"clone"`
-	Stats  bool       `json:"stats"`
-	Expand ExpandMode `json:"expandMode"`
-}
-
-// StorageClassCapabilitySpecFeaturesSnapshot describe snapshot features
-type StorageClassCapabilitySpecFeaturesSnapshot struct {
-	Create bool `json:"create"`
-	List   bool `json:"list"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -81,8 +87,37 @@ type StorageClassCapabilityList struct {
 	Items           []StorageClassCapability `json:"items"`
 }
 
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+// +genclient:noStatus
+// +genclient:nonNamespaced
+
+// ProvisionerCapability is the schema for the provisionercapability API
+// +k8s:openapi-gen=true
+type ProvisionerCapability struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec ProvisionerCapabilitySpec `json:"spec"`
+}
+
+// ProvisionerCapabilitySpec defines the desired state of ProvisionerCapability
+type ProvisionerCapabilitySpec struct {
+	PluginInfo PluginInfo         `json:"pluginInfo"`
+	Features   CapabilityFeatures `json:"features"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+type ProvisionerCapabilityList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+	Items           []ProvisionerCapability `json:"items"`
+}
+
 func init() {
 	SchemeBuilder.Register(
 		&StorageClassCapability{},
-		&StorageClassCapabilityList{})
+		&StorageClassCapabilityList{},
+		&ProvisionerCapability{},
+		&ProvisionerCapabilityList{})
 }

pkg/apis/storage/v1alpha1/zz_generated.deepcopy.go

@@ -20,9 +20,41 @@ limitations under the License.
 
 package v1alpha1
 
-import (
+import "k8s.io/apimachinery/pkg/runtime"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+
-)
+// DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PluginInfo) DeepCopyInto(out *PluginInfo) {
+	*out = *in
+	return
+}
+
+// DeepCopy is an autogenerated deep copy function, copying the receiver, creating a new ProvisionerCapabilitySpecPluginInfo.
+func (in *PluginInfo) DeepCopy() *PluginInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(PluginInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
+func (in *CapabilityFeatures) DeepCopyInto(out *CapabilityFeatures) {
+	*out = *in
+	out.Volume = in.Volume
+	out.Snapshot = in.Snapshot
+	return
+}
+
+// DeepCopy is an autogenerated deep copy function, copying the receiver, creating a new CapabilityFeatures.
+func (in *CapabilityFeatures) DeepCopy() *CapabilityFeatures {
+	if in == nil {
+		return nil
+	}
+	out := new(CapabilityFeatures)
+	in.DeepCopyInto(out)
+	return out
+}
 
 // DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StorageClassCapability) DeepCopyInto(out *StorageClassCapability) {
@@ -102,51 +134,79 @@ func (in *StorageClassCapabilitySpec) DeepCopy() *StorageClassCapabilitySpec {
 }
 
 // DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
-func (in *StorageClassCapabilitySpecFeatures) DeepCopyInto(out *StorageClassCapabilitySpecFeatures) {
+func (in *ProvisionerCapability) DeepCopyInto(out *ProvisionerCapability) {
 	*out = *in
-	out.Volume = in.Volume
+	out.TypeMeta = in.TypeMeta
-	out.Snapshot = in.Snapshot
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Spec = in.Spec
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageClassCapabilitySpecFeatures.
+// DeepCopy is an autogenerated deep copy function, copying the receiver, creating a new ProvisionerCapability.
-func (in *StorageClassCapabilitySpecFeatures) DeepCopy() *StorageClassCapabilitySpecFeatures {
+func (in *ProvisionerCapability) DeepCopy() *ProvisionerCapability {
 	if in == nil {
 		return nil
 	}
-	out := new(StorageClassCapabilitySpecFeatures)
+	out := new(ProvisionerCapability)
 	in.DeepCopyInto(out)
 	return out
 }
 
+// DeepCopyObject is an autogenerated deep copy function, copying the receiver, creating a new runtime.Object.
+func (in *ProvisionerCapability) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
 // DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
-func (in *StorageClassCapabilitySpecFeaturesSnapshot) DeepCopyInto(out *StorageClassCapabilitySpecFeaturesSnapshot) {
+func (in *ProvisionerCapabilityList) DeepCopyInto(out *ProvisionerCapabilityList) {
 	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ProvisionerCapability, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerCapabilitySpecFeaturesSnapshot.
+// DeepCopy is an autogenerated deep copy function, copying the receiver, creating a new ProvisionerCapabilityList.
-func (in *StorageClassCapabilitySpecFeaturesSnapshot) DeepCopy() *StorageClassCapabilitySpecFeaturesSnapshot {
+func (in *ProvisionerCapabilityList) DeepCopy() *ProvisionerCapabilityList {
 	if in == nil {
 		return nil
 	}
-	out := new(StorageClassCapabilitySpecFeaturesSnapshot)
+	out := new(ProvisionerCapabilityList)
 	in.DeepCopyInto(out)
 	return out
 }
 
+// DeepCopyObject is an autogenerated deep copy function, copying the receiver, creating a new runtime.Object.
+func (in *ProvisionerCapabilityList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
 // DeepCopyInto is an autogenerated deep copy function, copying the receiver, writing into out. in must be non-nil.
-func (in *StorageClassCapabilitySpecFeaturesVolume) DeepCopyInto(out *StorageClassCapabilitySpecFeaturesVolume) {
+func (in *ProvisionerCapabilitySpec) DeepCopyInto(out *ProvisionerCapabilitySpec) {
 	*out = *in
+	out.PluginInfo = in.PluginInfo
+	out.Features = in.Features
 	return
 }
 
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProvisionerCapabilitySpecFeaturesVolume.
+// DeepCopy is an autogenerated deep copy function, copying the receiver, creating a new ProvisionerCapabilitySpec.
-func (in *StorageClassCapabilitySpecFeaturesVolume) DeepCopy() *StorageClassCapabilitySpecFeaturesVolume {
+func (in *ProvisionerCapabilitySpec) DeepCopy() *ProvisionerCapabilitySpec {
 	if in == nil {
 		return nil
 	}
-	out := new(StorageClassCapabilitySpecFeaturesVolume)
+	out := new(ProvisionerCapabilitySpec)
 	in.DeepCopyInto(out)
 	return out
 }

pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+)
+
+// FakeProvisionerCapabilities implements ProvisionerCapabilityInterface
+type FakeProvisionerCapabilities struct {
+	Fake *FakeStorageV1alpha1
+}
+
+var provisionercapabilitiesResource = schema.GroupVersionResource{Group: "storage.kubesphere.io", Version: "v1alpha1", Resource: "provisionercapabilities"}
+
+var provisionercapabilitiesKind = schema.GroupVersionKind{Group: "storage.kubesphere.io", Version: "v1alpha1", Kind: "ProvisionerCapability"}
+
+// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any.
+func (c *FakeProvisionerCapabilities) Get(name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootGetAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors.
+func (c *FakeProvisionerCapabilities) List(opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootListAction(provisionercapabilitiesResource, provisionercapabilitiesKind, opts), &v1alpha1.ProvisionerCapabilityList{})
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1alpha1.ProvisionerCapabilityList{ListMeta: obj.(*v1alpha1.ProvisionerCapabilityList).ListMeta}
+	for _, item := range obj.(*v1alpha1.ProvisionerCapabilityList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested provisionerCapabilities.
+func (c *FakeProvisionerCapabilities) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewRootWatchAction(provisionercapabilitiesResource, opts))
+}
+
+// Create takes the representation of a provisionerCapability and creates it.  Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *FakeProvisionerCapabilities) Create(provisionerCapability *v1alpha1.ProvisionerCapability) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootCreateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *FakeProvisionerCapabilities) Update(provisionerCapability *v1alpha1.ProvisionerCapability) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs.
+func (c *FakeProvisionerCapabilities) Delete(name string, options *v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewRootDeleteAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{})
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeProvisionerCapabilities) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	action := testing.NewRootDeleteCollectionAction(provisionercapabilitiesResource, listOptions)
+
+	_, err := c.Fake.Invokes(action, &v1alpha1.ProvisionerCapabilityList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched provisionerCapability.
+func (c *FakeProvisionerCapabilities) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootPatchSubresourceAction(provisionercapabilitiesResource, name, pt, data, subresources...), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}

pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go

@@ -28,6 +28,10 @@ type FakeStorageV1alpha1 struct {
 	*testing.Fake
 }
 
+func (c *FakeStorageV1alpha1) ProvisionerCapabilities() v1alpha1.ProvisionerCapabilityInterface {
+	return &FakeProvisionerCapabilities{c}
+}
+
 func (c *FakeStorageV1alpha1) StorageClassCapabilities() v1alpha1.StorageClassCapabilityInterface {
 	return &FakeStorageClassCapabilities{c}
 }

pkg/client/clientset/versioned/typed/storage/v1alpha1/generated_expansion.go

@@ -18,4 +18,6 @@ limitations under the License.
 
 package v1alpha1
 
+type ProvisionerCapabilityExpansion interface{}
+
 type StorageClassCapabilityExpansion interface{}

pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go

@@ -0,0 +1,164 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+	scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+)
+
+// ProvisionerCapabilitiesGetter has a method to return a ProvisionerCapabilityInterface.
+// A group's client should implement this interface.
+type ProvisionerCapabilitiesGetter interface {
+	ProvisionerCapabilities() ProvisionerCapabilityInterface
+}
+
+// ProvisionerCapabilityInterface has methods to work with ProvisionerCapability resources.
+type ProvisionerCapabilityInterface interface {
+	Create(*v1alpha1.ProvisionerCapability) (*v1alpha1.ProvisionerCapability, error)
+	Update(*v1alpha1.ProvisionerCapability) (*v1alpha1.ProvisionerCapability, error)
+	Delete(name string, options *v1.DeleteOptions) error
+	DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error
+	Get(name string, options v1.GetOptions) (*v1alpha1.ProvisionerCapability, error)
+	List(opts v1.ListOptions) (*v1alpha1.ProvisionerCapabilityList, error)
+	Watch(opts v1.ListOptions) (watch.Interface, error)
+	Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error)
+	ProvisionerCapabilityExpansion
+}
+
+// provisionerCapabilities implements ProvisionerCapabilityInterface
+type provisionerCapabilities struct {
+	client rest.Interface
+}
+
+// newProvisionerCapabilities returns a ProvisionerCapabilities
+func newProvisionerCapabilities(c *StorageV1alpha1Client) *provisionerCapabilities {
+	return &provisionerCapabilities{
+		client: c.RESTClient(),
+	}
+}
+
+// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any.
+func (c *provisionerCapabilities) Get(name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Get().
+		Resource("provisionercapabilities").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do().
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors.
+func (c *provisionerCapabilities) List(opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1alpha1.ProvisionerCapabilityList{}
+	err = c.client.Get().
+		Resource("provisionercapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do().
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested provisionerCapabilities.
+func (c *provisionerCapabilities) Watch(opts v1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Resource("provisionercapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch()
+}
+
+// Create takes the representation of a provisionerCapability and creates it.  Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *provisionerCapabilities) Create(provisionerCapability *v1alpha1.ProvisionerCapability) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Post().
+		Resource("provisionercapabilities").
+		Body(provisionerCapability).
+		Do().
+		Into(result)
+	return
+}
+
+// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *provisionerCapabilities) Update(provisionerCapability *v1alpha1.ProvisionerCapability) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Put().
+		Resource("provisionercapabilities").
+		Name(provisionerCapability.Name).
+		Body(provisionerCapability).
+		Do().
+		Into(result)
+	return
+}
+
+// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs.
+func (c *provisionerCapabilities) Delete(name string, options *v1.DeleteOptions) error {
+	return c.client.Delete().
+		Resource("provisionercapabilities").
+		Name(name).
+		Body(options).
+		Do().
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *provisionerCapabilities) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error {
+	var timeout time.Duration
+	if listOptions.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOptions.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Resource("provisionercapabilities").
+		VersionedParams(&listOptions, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(options).
+		Do().
+		Error()
+}
+
+// Patch applies the patch and returns the patched provisionerCapability.
+func (c *provisionerCapabilities) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Patch(pt).
+		Resource("provisionercapabilities").
+		SubResource(subresources...).
+		Name(name).
+		Body(data).
+		Do().
+		Into(result)
+	return
+}

pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go

@@ -26,6 +26,7 @@ import (
 
 type StorageV1alpha1Interface interface {
 	RESTClient() rest.Interface
+	ProvisionerCapabilitiesGetter
 	StorageClassCapabilitiesGetter
 }
 
@@ -34,6 +35,10 @@ type StorageV1alpha1Client struct {
 	restClient rest.Interface
 }
 
+func (c *StorageV1alpha1Client) ProvisionerCapabilities() ProvisionerCapabilityInterface {
+	return newProvisionerCapabilities(c)
+}
+
 func (c *StorageV1alpha1Client) StorageClassCapabilities() StorageClassCapabilityInterface {
 	return newStorageClassCapabilities(c)
 }

pkg/client/informers/externalversions/generic.go

@@ -112,6 +112,8 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Servicemesh().V1alpha2().Strategies().Informer()}, nil
 
 		// Group=storage.kubesphere.io, Version=v1alpha1
+	case storagev1alpha1.SchemeGroupVersion.WithResource("provisionercapabilities"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().ProvisionerCapabilities().Informer()}, nil
 	case storagev1alpha1.SchemeGroupVersion.WithResource("storageclasscapabilities"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().StorageClassCapabilities().Informer()}, nil
 

pkg/client/informers/externalversions/storage/v1alpha1/interface.go

@@ -24,6 +24,8 @@ import (
 
 // Interface provides access to all the informers in this group version.
 type Interface interface {
+	// ProvisionerCapabilities returns a ProvisionerCapabilityInformer.
+	ProvisionerCapabilities() ProvisionerCapabilityInformer
 	// StorageClassCapabilities returns a StorageClassCapabilityInformer.
 	StorageClassCapabilities() StorageClassCapabilityInformer
 }
@@ -39,6 +41,11 @@ func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakList
 	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
 }
 
+// ProvisionerCapabilities returns a ProvisionerCapabilityInformer.
+func (v *version) ProvisionerCapabilities() ProvisionerCapabilityInformer {
+	return &provisionerCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}
+
 // StorageClassCapabilities returns a StorageClassCapabilityInformer.
 func (v *version) StorageClassCapabilities() StorageClassCapabilityInformer {
 	return &storageClassCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}

pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	time "time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+	storagev1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+	versioned "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	internalinterfaces "kubesphere.io/kubesphere/pkg/client/informers/externalversions/internalinterfaces"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1"
+)
+
+// ProvisionerCapabilityInformer provides access to a shared informer and lister for
+// ProvisionerCapabilities.
+type ProvisionerCapabilityInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1alpha1.ProvisionerCapabilityLister
+}
+
+type provisionerCapabilityInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// NewProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.StorageV1alpha1().ProvisionerCapabilities().List(options)
+			},
+			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.StorageV1alpha1().ProvisionerCapabilities().Watch(options)
+			},
+		},
+		&storagev1alpha1.ProvisionerCapability{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *provisionerCapabilityInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *provisionerCapabilityInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&storagev1alpha1.ProvisionerCapability{}, f.defaultInformer)
+}
+
+func (f *provisionerCapabilityInformer) Lister() v1alpha1.ProvisionerCapabilityLister {
+	return v1alpha1.NewProvisionerCapabilityLister(f.Informer().GetIndexer())
+}

pkg/client/listers/storage/v1alpha1/expansion_generated.go

@@ -18,6 +18,10 @@ limitations under the License.
 
 package v1alpha1
 
+// ProvisionerCapabilityListerExpansion allows custom methods to be added to
+// ProvisionerCapabilityLister.
+type ProvisionerCapabilityListerExpansion interface{}
+
 // StorageClassCapabilityListerExpansion allows custom methods to be added to
 // StorageClassCapabilityLister.
 type StorageClassCapabilityListerExpansion interface{}

pkg/client/listers/storage/v1alpha1/provisionercapability.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by lister-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+)
+
+// ProvisionerCapabilityLister helps list ProvisionerCapabilities.
+type ProvisionerCapabilityLister interface {
+	// List lists all ProvisionerCapabilities in the indexer.
+	List(selector labels.Selector) (ret []*v1alpha1.ProvisionerCapability, err error)
+	// Get retrieves the ProvisionerCapability from the index for a given name.
+	Get(name string) (*v1alpha1.ProvisionerCapability, error)
+	ProvisionerCapabilityListerExpansion
+}
+
+// provisionerCapabilityLister implements the ProvisionerCapabilityLister interface.
+type provisionerCapabilityLister struct {
+	indexer cache.Indexer
+}
+
+// NewProvisionerCapabilityLister returns a new ProvisionerCapabilityLister.
+func NewProvisionerCapabilityLister(indexer cache.Indexer) ProvisionerCapabilityLister {
+	return &provisionerCapabilityLister{indexer: indexer}
+}
+
+// List lists all ProvisionerCapabilities in the indexer.
+func (s *provisionerCapabilityLister) List(selector labels.Selector) (ret []*v1alpha1.ProvisionerCapability, err error) {
+	err = cache.ListAll(s.indexer, selector, func(m interface{}) {
+		ret = append(ret, m.(*v1alpha1.ProvisionerCapability))
+	})
+	return ret, err
+}
+
+// Get retrieves the ProvisionerCapability from the index for a given name.
+func (s *provisionerCapabilityLister) Get(name string) (*v1alpha1.ProvisionerCapability, error) {
+	obj, exists, err := s.indexer.GetByKey(name)
+	if err != nil {
+		return nil, err
+	}
+	if !exists {
+		return nil, errors.NewNotFound(v1alpha1.Resource("provisionercapability"), name)
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), nil
+}

pkg/controller/storage/capability/capability_controller.go

@@ -20,82 +20,111 @@ package capability
 
 import (
 	"fmt"
-	"os"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/version"
+	"k8s.io/client-go/discovery"
 	"reflect"
+	"strconv"
+	"strings"
 	"time"
 
-	snapapi "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
+	snapshotv1beta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
+	snapshotclient "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/typed/volumesnapshot/v1beta1"
 	snapinformers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions/volumesnapshot/v1beta1"
-	snaplisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1"
+	snapshotlisters "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/listers/volumesnapshot/v1beta1"
-	v1strorage "k8s.io/api/storage/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	storagev1beta1 "k8s.io/api/storage/v1beta1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/version"
 	"k8s.io/apimachinery/pkg/util/wait"
-	scinformers "k8s.io/client-go/informers/storage/v1"
+	storageinformersv1 "k8s.io/client-go/informers/storage/v1"
-	"k8s.io/client-go/kubernetes"
+	storageinformersv1beta1 "k8s.io/client-go/informers/storage/v1beta1"
 	"k8s.io/client-go/kubernetes/scheme"
-	sclisters "k8s.io/client-go/listers/storage/v1"
+	storageclient "k8s.io/client-go/kubernetes/typed/storage/v1"
+	storagelistersv1 "k8s.io/client-go/listers/storage/v1"
+	storagelistersv1beta1 "k8s.io/client-go/listers/storage/v1beta1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
-	crdapi "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+	capability "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
-	clientset "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
 	crdscheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
-	storageinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/storage/v1alpha1"
+	capabilityclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned/typed/storage/v1alpha1"
-	crdlisters "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1"
+	capabilityinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/storage/v1alpha1"
+	capabilitylisters "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1"
 )
 
 const (
-	minKubernetesVersion = "v1.17.0"
+	minSnapshotSupportedVersion = "v1.17.0"
-	CSIAddressFormat     = "/var/lib/kubelet/plugins/%s/csi.sock"
+	csiAddressFormat            = "/var/lib/kubelet/plugins/%s/csi.sock"
+	annotationSupportSnapshot   = "storageclass.kubesphere.io/support-snapshot"
 )
 
 type csiAddressGetter func(storageClassProvisioner string) string
 
 type StorageCapabilityController struct {
-	k8sClient                    kubernetes.Interface
+	storageClassCapabilityClient capabilityclient.StorageClassCapabilityInterface
-	storageClassCapabilityClient clientset.Interface
+	storageCapabilityLister      capabilitylisters.StorageClassCapabilityLister
-	storageClassLister           sclisters.StorageClassLister
-	storageClassSynced           cache.InformerSynced
-	snapshotClassLister          snaplisters.VolumeSnapshotClassLister
-	snapshotClassSynced          cache.InformerSynced
-	storageClassCapabilityLister crdlisters.StorageClassCapabilityLister
 	storageClassCapabilitySynced cache.InformerSynced
-	workQueue                    workqueue.RateLimitingInterface
+
+	provisionerCapabilityLister capabilitylisters.ProvisionerCapabilityLister
+	provisionerCapabilitySynced cache.InformerSynced
+
+	storageClassClient storageclient.StorageClassInterface
+	storageClassLister storagelistersv1.StorageClassLister
+	storageClassSynced cache.InformerSynced
+
+	snapshotSupported   bool
+	snapshotClassClient snapshotclient.VolumeSnapshotClassInterface
+	snapshotClassLister snapshotlisters.VolumeSnapshotClassLister
+	snapshotClassSynced cache.InformerSynced
+
+	csiDriverLister storagelistersv1beta1.CSIDriverLister
+	csiDriverSynced cache.InformerSynced
+
 	csiAddressGetter csiAddressGetter
+
+	workQueue workqueue.RateLimitingInterface
 }
 
 // This controller is responsible to watch StorageClass, SnapshotClass.
 // And then update StorageClassCapability CRD resource object to the newest status.
 func NewController(
-	k8sClient kubernetes.Interface,
+	capabilityClient capabilityclient.StorageClassCapabilityInterface,
-	storageClassCapabilityClient clientset.Interface,
+	capabilityInformer capabilityinformers.Interface,
-	storageClassInformer scinformers.StorageClassInformer,
+	storageClassClient storageclient.StorageClassInterface,
+	storageClassInformer storageinformersv1.StorageClassInformer,
+	snapshotSupported bool,
+	snapshotClassClient snapshotclient.VolumeSnapshotClassInterface,
 	snapshotClassInformer snapinformers.VolumeSnapshotClassInformer,
-	storageClassCapabilityInformer storageinformers.StorageClassCapabilityInformer,
+	csiDriverInformer storageinformersv1beta1.CSIDriverInformer,
-	csiAddressGetter csiAddressGetter,
 ) *StorageCapabilityController {
 
 	utilruntime.Must(crdscheme.AddToScheme(scheme.Scheme))
 	controller := &StorageCapabilityController{
-		k8sClient:                    k8sClient,
+		storageClassCapabilityClient: capabilityClient,
-		storageClassCapabilityClient: storageClassCapabilityClient,
+		storageCapabilityLister:      capabilityInformer.StorageClassCapabilities().Lister(),
+		storageClassCapabilitySynced: capabilityInformer.StorageClassCapabilities().Informer().HasSynced,
+		provisionerCapabilityLister:  capabilityInformer.ProvisionerCapabilities().Lister(),
+		provisionerCapabilitySynced:  capabilityInformer.ProvisionerCapabilities().Informer().HasSynced,
+		storageClassClient:           storageClassClient,
 		storageClassLister:           storageClassInformer.Lister(),
 		storageClassSynced:           storageClassInformer.Informer().HasSynced,
+		snapshotSupported:            snapshotSupported,
+		snapshotClassClient:          snapshotClassClient,
 		snapshotClassLister:          snapshotClassInformer.Lister(),
 		snapshotClassSynced:          snapshotClassInformer.Informer().HasSynced,
-		storageClassCapabilityLister: storageClassCapabilityInformer.Lister(),
+		csiDriverLister:              csiDriverInformer.Lister(),
-		storageClassCapabilitySynced: storageClassCapabilityInformer.Informer().HasSynced,
+		csiDriverSynced:              csiDriverInformer.Informer().HasSynced,
+		csiAddressGetter:             csiAddress,
 		workQueue:                    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "StorageClasses"),
-		csiAddressGetter:             csiAddressGetter,
 	}
+
 	storageClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: controller.enqueueStorageClass,
 		UpdateFunc: func(old, new interface{}) {
-			newStorageClass := new.(*v1strorage.StorageClass)
+			newStorageClass := new.(*storagev1.StorageClass)
-			oldStorageClass := old.(*v1strorage.StorageClass)
+			oldStorageClass := old.(*storagev1.StorageClass)
 			if newStorageClass.ResourceVersion == oldStorageClass.ResourceVersion {
 				return
 			}
@@ -103,13 +132,15 @@ func NewController(
 		},
 		DeleteFunc: controller.enqueueStorageClass,
 	})
-	snapshotClassInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+
-		AddFunc: controller.enqueueSnapshotClass,
+	csiDriverInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		UpdateFunc: func(old, new interface{}) {
+		AddFunc: controller.handlerCSIDriver,
+		UpdateFunc: func(oldObj, newObj interface{}) {
 			return
 		},
-		DeleteFunc: controller.enqueueSnapshotClass,
+		DeleteFunc: controller.handlerCSIDriver,
 	})
+
 	return controller
 }
 
@@ -123,7 +154,14 @@ func (c *StorageCapabilityController) Run(threadCnt int, stopCh <-chan struct{})
 
 	// Wait for the caches to be synced before starting workers
 	klog.Info("Waiting for informer caches to sync")
-	if ok := cache.WaitForCacheSync(stopCh, c.storageClassSynced, c.snapshotClassSynced, c.storageClassCapabilitySynced); !ok {
+	cacheSyncs := []cache.InformerSynced{
+		c.storageClassCapabilitySynced,
+		c.provisionerCapabilitySynced,
+		c.storageClassSynced,
+		c.csiDriverSynced,
+	}
+
+	if ok := cache.WaitForCacheSync(stopCh, cacheSyncs...); !ok {
 		return fmt.Errorf("failed to wait for caches to sync")
 	}
 
@@ -136,27 +174,22 @@ func (c *StorageCapabilityController) Run(threadCnt int, stopCh <-chan struct{})
 	return nil
 }
 
-func (c *StorageCapabilityController) enqueueStorageClass(obj interface{}) {
+func (c *StorageCapabilityController) handlerCSIDriver(obj interface{}) {
-	storageClass := obj.(*v1strorage.StorageClass)
+	csiDriver := obj.(*storagev1beta1.CSIDriver)
-	if !fileExist(c.csiAddressGetter(storageClass.Provisioner)) {
+	storageClasses, err := c.storageClassLister.List(labels.Everything())
-		klog.V(4).Infof("CSI address of storage class: %s, provisioner :%s not exist", storageClass.Name, storageClass.Provisioner)
+	if err != nil {
+		klog.Error("list StorageClass error when handler csiDriver", err)
 		return
 	}
-	var key string
+	for _, storageClass := range storageClasses {
-	var err error
+		if storageClass.Provisioner == csiDriver.Name {
-	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
+			klog.Info("enqueue StorageClass when handler csiDriver", storageClass)
-		utilruntime.HandleError(err)
+			c.enqueueStorageClass(storageClass)
-		return
+		}
 	}
-	c.workQueue.Add(key)
 }
 
-func (c *StorageCapabilityController) enqueueSnapshotClass(obj interface{}) {
+func (c *StorageCapabilityController) enqueueStorageClass(obj interface{}) {
-	snapshotClass := obj.(*snapapi.VolumeSnapshotClass)
-	if !fileExist(c.csiAddressGetter(snapshotClass.Driver)) {
-		klog.V(4).Infof("CSI address of snapshot class: %s, driver:%s not exist", snapshotClass.Name, snapshotClass.Driver)
-		return
-	}
 	var key string
 	var err error
 	if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
@@ -214,67 +247,191 @@ func (c *StorageCapabilityController) syncHandler(key string) error {
 
 	// Get StorageClass
 	storageClass, err := c.storageClassLister.Get(name)
-	klog.V(4).Infof("Get storageClass %s: entity %v", name, storageClass)
 	if err != nil {
+		// StorageClass has been deleted, delete StorageClassCapability and VolumeSnapshotClass
 		if errors.IsNotFound(err) {
-			_, err = c.storageClassCapabilityLister.Get(name)
+			if c.snapshotSupported {
+				err = c.deleteSnapshotClass(name)
 				if err != nil {
-				if errors.IsNotFound(err) {
-					return nil
-				}
 					return err
 				}
-			return c.storageClassCapabilityClient.StorageV1alpha1().StorageClassCapabilities().Delete(name, &metav1.DeleteOptions{})
+			}
+			return c.deleteStorageCapability(name)
 		}
 		return err
 	}
 
-	// Get SnapshotClass
+	// Get capability spec
-	snapshotClassCreated := true
+	capabilitySpec, err := c.getCapabilitySpec(storageClass)
-	_, err = c.snapshotClassLister.Get(storageClass.Name)
 	if err != nil {
-		if errors.IsNotFound(err) {
-			snapshotClassCreated = false
-		} else {
 		return err
 	}
+	// No capability because csi-plugin not installed
+	if capabilitySpec == nil {
+		klog.Infof("StorageClass %s has no capability", name)
+		err = c.updateStorageClassSnapshotSupported(storageClass, false)
+		if err != nil {
+			return err
 		}
+		return c.deleteStorageCapability(name)
+	}
+	klog.Infof("StorageClass %s has capability %v", name, capabilitySpec)
 
-	// Get exist StorageClassCapability
+	// Handle VolumeSnapshotClass with same name of StorageClass
-	storageClassCapabilityExist, err := c.storageClassCapabilityLister.Get(storageClass.Name)
+	// annotate "support-snapshot" of StorageClass
+	withSnapshotCapability := false
+	if c.snapshotSupported && capabilitySpec.Features.Snapshot.Create {
+		_, err = c.snapshotClassLister.Get(name)
+		if err != nil {
+			// If VolumeSnapshotClass not exist, create it
 			if errors.IsNotFound(err) {
-		// If the resource doesn't exist, we'll create it
+				volumeSnapshotClassCreate := &snapshotv1beta1.VolumeSnapshotClass{
-		klog.V(4).Infof("Create StorageClassProvisioner %s", storageClass.GetName())
+					ObjectMeta:     metav1.ObjectMeta{Name: name},
-		storageClassCapabilityCreate := &crdapi.StorageClassCapability{ObjectMeta: metav1.ObjectMeta{Name: storageClass.Name}}
+					Driver:         storageClass.Provisioner,
-		err = c.addSpec(&storageClassCapabilityCreate.Spec, storageClass, snapshotClassCreated)
+					DeletionPolicy: snapshotv1beta1.VolumeSnapshotContentDelete,
+				}
+				_, err = c.snapshotClassClient.Create(volumeSnapshotClassCreate)
 				if err != nil {
 					return err
 				}
-		klog.V(4).Info("Create StorageClassCapability: ", storageClassCapabilityCreate)
-		_, err = c.storageClassCapabilityClient.StorageV1alpha1().StorageClassCapabilities().Create(storageClassCapabilityCreate)
-		return err
 			}
+		}
+		withSnapshotCapability = true
+	}
+	err = c.updateStorageClassSnapshotSupported(storageClass, withSnapshotCapability)
 	if err != nil {
 		return err
 	}
 
-	// If the resource exist, we can update it.
+	// Handle StorageClassCapability with the same name of StorageClass
+	storageClassCapabilityExist, err := c.storageCapabilityLister.Get(storageClass.Name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// If StorageClassCapability doesn't exist, create it
+			storageClassCapabilityCreate := &capability.StorageClassCapability{ObjectMeta: metav1.ObjectMeta{Name: storageClass.Name}}
+			storageClassCapabilityCreate.Spec = *capabilitySpec
+			klog.Info("Create StorageClassCapability: ", storageClassCapabilityCreate)
+			_, err = c.storageClassCapabilityClient.Create(storageClassCapabilityCreate)
+			return err
+		}
+		return err
+	}
+	// If StorageClassCapability exist, update it.
 	storageClassCapabilityUpdate := storageClassCapabilityExist.DeepCopy()
-	err = c.addSpec(&storageClassCapabilityUpdate.Spec, storageClass, snapshotClassCreated)
+	storageClassCapabilityUpdate.Spec = *capabilitySpec
-	if err != nil {
-		return err
-	}
 	if !reflect.DeepEqual(storageClassCapabilityExist, storageClassCapabilityUpdate) {
-		klog.V(4).Info("Update StorageClassCapability: ", storageClassCapabilityUpdate)
+		klog.Info("Update StorageClassCapability: ", storageClassCapabilityUpdate)
-		_, err = c.storageClassCapabilityClient.StorageV1alpha1().StorageClassCapabilities().Update(storageClassCapabilityUpdate)
+		_, err = c.storageClassCapabilityClient.Update(storageClassCapabilityUpdate)
 		return err
 	}
 	return nil
 }
 
-func (c *StorageCapabilityController) IsValidKubernetesVersion() bool {
+func (c *StorageCapabilityController) updateStorageClassSnapshotSupported(storageClass *storagev1.StorageClass, snapshotSupported bool) error {
-	minVer := version.MustParseGeneric(minKubernetesVersion)
+	if storageClass.Annotations == nil {
-	rawVer, err := c.k8sClient.Discovery().ServerVersion()
+		storageClass.Annotations = make(map[string]string)
+	}
+	snapshotSupportedAnnotated, err := strconv.ParseBool(storageClass.Annotations[annotationSupportSnapshot])
+	// err != nil means annotationSupportSnapshot is not illegal, include empty
+	if err != nil || snapshotSupported != snapshotSupportedAnnotated {
+		storageClass.Annotations[annotationSupportSnapshot] = strconv.FormatBool(snapshotSupported)
+		_, err = c.storageClassClient.Update(storageClass)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *StorageCapabilityController) deleteStorageCapability(name string) error {
+	_, err := c.storageCapabilityLister.Get(name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	klog.Infof("Delete StorageClassCapability %s", name)
+	return c.storageClassCapabilityClient.Delete(name, &metav1.DeleteOptions{})
+}
+
+func (c *StorageCapabilityController) deleteSnapshotClass(name string) error {
+	if !c.snapshotSupported {
+		return nil
+	}
+	_, err := c.snapshotClassLister.Get(name)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	klog.Infof("Delete SnapshotClass %s", name)
+	return c.snapshotClassClient.Delete(name, &metav1.DeleteOptions{})
+}
+
+func (c *StorageCapabilityController) nonCSICapability(provisioner string) (*capability.StorageClassCapabilitySpec, error) {
+	provisionerCapability, err := c.provisionerCapabilityLister.Get(getProvisionerCapabilityName(provisioner))
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	capabilitySpec := &capability.StorageClassCapabilitySpec{
+		Features: provisionerCapability.Spec.Features,
+	}
+	return capabilitySpec, nil
+}
+
+func (c *StorageCapabilityController) getCapabilitySpec(storageClass *storagev1.StorageClass) (*capability.StorageClassCapabilitySpec, error) {
+	isCsi, err := c.isCSIStorage(storageClass.Provisioner)
+	if err != nil {
+		return nil, err
+	}
+
+	var capabilitySpec *capability.StorageClassCapabilitySpec
+	if isCsi {
+		capabilitySpec, err = csiCapability(c.csiAddressGetter(storageClass.Provisioner))
+	} else {
+		capabilitySpec, err = c.nonCSICapability(storageClass.Provisioner)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	if capabilitySpec != nil {
+		capabilitySpec.Provisioner = storageClass.Provisioner
+		if storageClass.AllowVolumeExpansion == nil || !*storageClass.AllowVolumeExpansion {
+			capabilitySpec.Features.Volume.Expand = capability.ExpandModeUnknown
+		}
+		if !c.snapshotSupported {
+			capabilitySpec.Features.Snapshot.Create = false
+			capabilitySpec.Features.Snapshot.List = false
+		}
+	}
+	return capabilitySpec, nil
+}
+
+func (c *StorageCapabilityController) isCSIStorage(provisioner string) (bool, error) {
+	_, err := c.csiDriverLister.Get(provisioner)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+// this is used for test of CSIDriver on windows
+func (c *StorageCapabilityController) setCSIAddressGetter(getter csiAddressGetter) {
+	c.csiAddressGetter = getter
+}
+
+func SnapshotSupported(discoveryInterface discovery.DiscoveryInterface) bool {
+	minVer := version.MustParseGeneric(minSnapshotSupportedVersion)
+	rawVer, err := discoveryInterface.ServerVersion()
 	if err != nil {
 		return false
 	}
@@ -285,29 +442,10 @@ func (c *StorageCapabilityController) IsValidKubernetesVersion() bool {
 	return ver.AtLeast(minVer)
 }
 
-func (c *StorageCapabilityController) addSpec(spec *crdapi.StorageClassCapabilitySpec, storageClass *v1strorage.StorageClass, snapshotClassCreated bool) error {
+func csiAddress(provisioner string) string {
-	csiCapability, err := csiCapability(c.csiAddressGetter(storageClass.Provisioner))
+	return fmt.Sprintf(csiAddressFormat, provisioner)
-	if err != nil {
-		return err
-	}
-	spec.Provisioner = storageClass.Provisioner
-	spec.Features.Volume = csiCapability.Features.Volume
-	spec.Features.Topology = csiCapability.Features.Topology
-	if *storageClass.AllowVolumeExpansion {
-		spec.Features.Volume.Expand = csiCapability.Features.Volume.Expand
-	} else {
-		spec.Features.Volume.Expand = crdapi.ExpandModeUnknown
-	}
-	if snapshotClassCreated {
-		spec.Features.Snapshot = csiCapability.Features.Snapshot
-	} else {
-		spec.Features.Snapshot.Create = false
-		spec.Features.Snapshot.List = false
-	}
-	return nil
 }
 
-func fileExist(name string) bool {
+func getProvisionerCapabilityName(provisioner string) string {
-	_, err := os.Stat(name)
+	return strings.NewReplacer(".", "-", "/", "-").Replace(provisioner)
-	return !os.IsNotExist(err)
 }

pkg/controller/storage/capability/capability_controller_test.go

@@ -20,98 +20,122 @@ package capability
 
 import (
 	"github.com/google/go-cmp/cmp"
+	"math/rand"
+
+	//"github.com/google/go-cmp/cmp"
 	snapbeta1 "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1"
 	snapfake "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned/fake"
 	snapinformers "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/informers/externalversions"
 	storagev1 "k8s.io/api/storage/v1"
+	storagev1beta1 "k8s.io/api/storage/v1beta1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/diff"
-	kubeinformers "k8s.io/client-go/informers"
+	k8sinformers "k8s.io/client-go/informers"
 	k8sfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
-	crdv1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
+	ksv1alpha1 "kubesphere.io/kubesphere/pkg/apis/storage/v1alpha1"
-	crdfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
+	ksfake "kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
-	crdinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
 	"reflect"
 	"testing"
 	"time"
 )
 
 var (
-	alwaysReady        = func() bool { return true }
 	noReSyncPeriodFunc = func() time.Duration { return 0 }
 )
 
 type fixture struct {
 	t                 *testing.T
+	snapshotSupported bool
 	// Clients
 	k8sClient           *k8sfake.Clientset
 	snapshotClassClient *snapfake.Clientset
-	storageClassCapabilitiesClient *crdfake.Clientset
+	ksClient            *ksfake.Clientset
 	// Objects from here preload into NewSimpleFake.
-	storageClassObjects           []runtime.Object
+	storageObjects       []runtime.Object // include StorageClass and CSIDriver
 	snapshotClassObjects []runtime.Object
-	storageClassCapabilityObjects []runtime.Object
+	capabilityObjects    []runtime.Object //  include StorageClassCapability and ProvisionerCapability
 	// Objects to put in the store.
 	storageClassLister           []*storagev1.StorageClass
 	snapshotClassLister          []*snapbeta1.VolumeSnapshotClass
-	storageClassCapabilityLister []*crdv1alpha1.StorageClassCapability
+	storageClassCapabilityLister []*ksv1alpha1.StorageClassCapability
+	provisionerCapabilityLister  []*ksv1alpha1.ProvisionerCapability
+	csiDriverLister              []*storagev1beta1.CSIDriver
 	// Actions expected to happen on the client.
-	storageClassCapabilitiesActions []core.Action
+	actions []core.Action
 	// CSI server
+	runCSIServer  bool
 	fakeCSIServer *fakeCSIServer
 }
 
-func newFixture(t *testing.T) *fixture {
+func newFixture(t *testing.T, snapshotSupported bool, runCSIServer bool) *fixture {
 	return &fixture{
 		t:                 t,
+		snapshotSupported: snapshotSupported,
+		runCSIServer:      runCSIServer,
 	}
 }
 
-func (f *fixture) newController() (*StorageCapabilityController, kubeinformers.SharedInformerFactory,
+func (f *fixture) newController() (*StorageCapabilityController,
-	crdinformers.SharedInformerFactory, snapinformers.SharedInformerFactory) {
+	k8sinformers.SharedInformerFactory,
+	ksinformers.SharedInformerFactory,
+	snapinformers.SharedInformerFactory) {
 
-	fakeCSIServer, address := newTestCSIServer()
+	f.k8sClient = k8sfake.NewSimpleClientset(f.storageObjects...)
-	f.fakeCSIServer = fakeCSIServer
+	f.ksClient = ksfake.NewSimpleClientset(f.capabilityObjects...)
-
-	f.k8sClient = k8sfake.NewSimpleClientset(f.storageClassObjects...)
-	f.storageClassCapabilitiesClient = crdfake.NewSimpleClientset(f.storageClassCapabilityObjects...)
 	f.snapshotClassClient = snapfake.NewSimpleClientset(f.snapshotClassObjects...)
 
-	k8sI := kubeinformers.NewSharedInformerFactory(f.k8sClient, noReSyncPeriodFunc())
+	k8sInformers := k8sinformers.NewSharedInformerFactory(f.k8sClient, noReSyncPeriodFunc())
-	crdI := crdinformers.NewSharedInformerFactory(f.storageClassCapabilitiesClient, noReSyncPeriodFunc())
+	ksInformers := ksinformers.NewSharedInformerFactory(f.ksClient, noReSyncPeriodFunc())
-	snapI := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc())
+	snapshotInformers := snapinformers.NewSharedInformerFactory(f.snapshotClassClient, noReSyncPeriodFunc())
 
 	c := NewController(
-		f.k8sClient,
+		f.ksClient.StorageV1alpha1().StorageClassCapabilities(),
-		f.storageClassCapabilitiesClient,
+		ksInformers.Storage().V1alpha1(),
-		k8sI.Storage().V1().StorageClasses(),
+		f.k8sClient.StorageV1().StorageClasses(),
-		snapI.Snapshot().V1beta1().VolumeSnapshotClasses(),
+		k8sInformers.Storage().V1().StorageClasses(),
-		crdI.Storage().V1alpha1().StorageClassCapabilities(),
+		f.snapshotSupported,
-		func(storageClassProvisioner string) string { return address },
+		f.snapshotClassClient.SnapshotV1beta1().VolumeSnapshotClasses(),
+		snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses(),
+		k8sInformers.Storage().V1beta1().CSIDrivers(),
 	)
+	if f.runCSIServer {
+		port := 30000 + rand.Intn(100)
+		fakeCSIServer, address := newTestCSIServer(port)
+		f.fakeCSIServer = fakeCSIServer
+		c.setCSIAddressGetter(func(storageClassProvisioner string) string { return address })
+	}
 
 	for _, storageClass := range f.storageClassLister {
-		_ = k8sI.Storage().V1().StorageClasses().Informer().GetIndexer().Add(storageClass)
+		_ = k8sInformers.Storage().V1().StorageClasses().Informer().GetIndexer().Add(storageClass)
+	}
+	for _, csiDriver := range f.csiDriverLister {
+		_ = k8sInformers.Storage().V1beta1().CSIDrivers().Informer().GetIndexer().Add(csiDriver)
 	}
 	for _, snapshotClass := range f.snapshotClassLister {
-		_ = snapI.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass)
+		_ = snapshotInformers.Snapshot().V1beta1().VolumeSnapshotClasses().Informer().GetIndexer().Add(snapshotClass)
 	}
 	for _, storageClassCapability := range f.storageClassCapabilityLister {
-		_ = crdI.Storage().V1alpha1().StorageClassCapabilities().Informer().GetIndexer().Add(storageClassCapability)
+		_ = ksInformers.Storage().V1alpha1().StorageClassCapabilities().Informer().GetIndexer().Add(storageClassCapability)
+	}
+	for _, provisionerCapability := range f.provisionerCapabilityLister {
+		_ = ksInformers.Storage().V1alpha1().ProvisionerCapabilities().Informer().GetIndexer().Add(provisionerCapability)
 	}
 
-	return c, k8sI, crdI, snapI
+	return c, k8sInformers, ksInformers, snapshotInformers
 }
 
 func (f *fixture) runController(scName string, startInformers bool, expectError bool) {
 	c, k8sI, crdI, snapI := f.newController()
 
+	if f.runCSIServer {
 		f.fakeCSIServer.run()
 		defer f.fakeCSIServer.stop()
+	}
 
 	if startInformers {
 		stopCh := make(chan struct{})
@@ -120,9 +144,6 @@ func (f *fixture) runController(scName string, startInformers bool, expectError
 		crdI.Start(stopCh)
 		snapI.Start(stopCh)
 	}
-	c.storageClassSynced = alwaysReady
-	c.snapshotClassSynced = alwaysReady
-	c.storageClassCapabilitySynced = alwaysReady
 
 	err := c.syncHandler(scName)
 	if !expectError && err != nil {
@@ -131,13 +152,17 @@ func (f *fixture) runController(scName string, startInformers bool, expectError
 		f.t.Error("expected error syncing, got nil")
 	}
 
-	actions := filterInformerActions(f.storageClassCapabilitiesClient.Actions())
+	var actions []core.Action
-	for i, action := range actions {
+	actions = append(actions, f.snapshotClassClient.Actions()...)
-		if len(f.storageClassCapabilitiesActions) < i+1 {
+	actions = append(actions, f.k8sClient.Actions()...)
-			f.t.Errorf("%d unexpected actions: %+v", len(actions)-len(f.storageClassCapabilitiesActions), actions[i:])
+	actions = append(actions, f.ksClient.Actions()...)
-			break
+	filerActions := filterInformerActions(actions)
+	if len(filerActions) != len(f.actions) {
+		f.t.Errorf("count of actions: differ (-got, +want): %s", cmp.Diff(filerActions, f.actions))
+		return
 	}
-		expectedAction := f.storageClassCapabilitiesActions[i]
+	for i, action := range filerActions {
+		expectedAction := f.actions[i]
 		checkAction(expectedAction, action, f.t)
 	}
 }
@@ -146,21 +171,36 @@ func (f *fixture) run(scName string) {
 	f.runController(scName, true, false)
 }
 
-func (f *fixture) expectCreateStorageClassCapabilitiesAction(storageClassCapability *crdv1alpha1.StorageClassCapability) {
+func (f *fixture) expectCreateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
-	f.storageClassCapabilitiesActions = append(f.storageClassCapabilitiesActions, core.NewCreateAction(
+	f.actions = append(f.actions, core.NewCreateAction(
 		schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability))
 }
 
-func (f *fixture) expectUpdateStorageClassCapabilitiesAction(storageClassCapability *crdv1alpha1.StorageClassCapability) {
+func (f *fixture) expectUpdateStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
-	f.storageClassCapabilitiesActions = append(f.storageClassCapabilitiesActions, core.NewUpdateAction(
+	f.actions = append(f.actions, core.NewUpdateAction(
 		schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability))
 }
 
-func (f *fixture) expectDeleteStorageClassCapabilitiesAction(storageClassCapability *crdv1alpha1.StorageClassCapability) {
+func (f *fixture) expectDeleteStorageClassCapabilitiesAction(storageClassCapability *ksv1alpha1.StorageClassCapability) {
-	f.storageClassCapabilitiesActions = append(f.storageClassCapabilitiesActions, core.NewDeleteAction(
+	f.actions = append(f.actions, core.NewDeleteAction(
 		schema.GroupVersionResource{Resource: "storageclasscapabilities"}, storageClassCapability.Namespace, storageClassCapability.Name))
 }
 
+func (f *fixture) expectUpdateStorageClassAction(storageClass *storagev1.StorageClass) {
+	f.actions = append(f.actions, core.NewUpdateAction(
+		schema.GroupVersionResource{Resource: "storageclasses"}, storageClass.Namespace, storageClass))
+}
+
+func (f *fixture) expectCreateSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) {
+	f.actions = append(f.actions, core.NewCreateAction(
+		schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass))
+}
+
+func (f *fixture) expectDeleteSnapshotClassAction(snapshotClass *snapbeta1.VolumeSnapshotClass) {
+	f.actions = append(f.actions, core.NewDeleteAction(
+		schema.GroupVersionResource{Resource: "volumesnapshotclasses"}, snapshotClass.Namespace, snapshotClass.Name))
+}
+
 // filterInformerActions filters list and watch actions for testing resources.
 // Since list and watch don't change resource state we can filter it to lower
 // nose level in our tests.
@@ -179,7 +219,7 @@ func filterInformerActions(actions []core.Action) []core.Action {
 // same attached resources
 func checkAction(expected, actual core.Action, t *testing.T) {
 	if !(expected.Matches(actual.GetVerb(), actual.GetResource().Resource) && actual.GetSubresource() == expected.GetSubresource()) {
-		t.Errorf("Expected\n\t%#v\ngot\n\t%#v", expected, actual)
+		t.Errorf("\nExpected\n\t%#v\ngot\n\t%#v", expected, actual)
 		return
 	}
 
@@ -233,20 +273,37 @@ func newStorageClass(name string, provisioner string) *storagev1.StorageClass {
 	}
 }
 
-func newStorageClassCapability(storageClass *storagev1.StorageClass) *crdv1alpha1.StorageClassCapability {
+func newStorageClassCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.StorageClassCapability {
-	storageClassCapability := &crdv1alpha1.StorageClassCapability{}
+	storageClassCapability := &ksv1alpha1.StorageClassCapability{}
 	storageClassCapability.Name = storageClass.Name
 	storageClassCapability.Spec = *newStorageClassCapabilitySpec()
 	storageClassCapability.Spec.Provisioner = storageClass.Provisioner
 	return storageClassCapability
 }
 
+func newProvisionerCapability(storageClass *storagev1.StorageClass) *ksv1alpha1.ProvisionerCapability {
+	provisionerCapability := &ksv1alpha1.ProvisionerCapability{}
+	provisionerCapability.Name = getProvisionerCapabilityName(storageClass.Provisioner)
+	provisionerCapability.Spec.PluginInfo.Name = storageClass.Provisioner
+	provisionerCapability.Spec.Features = newStorageClassCapabilitySpec().Features
+	// ProvisionerCapability snapshot is always false
+	provisionerCapability.Spec.Features.Snapshot.Create = false
+	return provisionerCapability
+}
+
+func newCSIDriver(storageClass *storagev1.StorageClass) *storagev1beta1.CSIDriver {
+	csiDriver := &storagev1beta1.CSIDriver{}
+	csiDriver.Name = storageClass.Provisioner
+	return csiDriver
+}
+
 func newSnapshotClass(storageClass *storagev1.StorageClass) *snapbeta1.VolumeSnapshotClass {
 	return &snapbeta1.VolumeSnapshotClass{
 		ObjectMeta: v1.ObjectMeta{
 			Name: storageClass.Name,
 		},
 		Driver:         storageClass.Provisioner,
+		DeletionPolicy: snapbeta1.VolumeSnapshotContentDelete,
 	}
 }
 
@@ -260,18 +317,22 @@ func getKey(sc *storagev1.StorageClass, t *testing.T) string {
 }
 
 func TestCreateStorageClass(t *testing.T) {
-	fixture := newFixture(t)
+	fixture := newFixture(t, true, true)
 	storageClass := newStorageClass("csi-example", "csi.example.com")
+	storageClassUpdate := storageClass.DeepCopy()
+	storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "true"}
 	snapshotClass := newSnapshotClass(storageClass)
 	storageClassCapability := newStorageClassCapability(storageClass)
+	csiDriver := newCSIDriver(storageClass)
 
 	// Objects exist
-	fixture.storageClassObjects = append(fixture.storageClassObjects, storageClass)
+	fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
 	fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
-	fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass)
+	fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
-	fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass)
 
 	// Action expected
+	fixture.expectCreateSnapshotClassAction(snapshotClass)
+	fixture.expectUpdateStorageClassAction(storageClassUpdate)
 	fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
 
 	// Run test
@@ -280,20 +341,26 @@ func TestCreateStorageClass(t *testing.T) {
 
 func TestUpdateStorageClass(t *testing.T) {
 	storageClass := newStorageClass("csi-example", "csi.example.com")
+	storageClass.Annotations = map[string]string{annotationSupportSnapshot: "true"}
 	snapshotClass := newSnapshotClass(storageClass)
+	storageClassCapabilityUpdate := newStorageClassCapability(storageClass)
 	storageClassCapability := newStorageClassCapability(storageClass)
+	//old and new should have deference
+	storageClassCapability.Spec.Features.Volume.Create = !storageClassCapability.Spec.Features.Volume.Create
+	csiDriver := newCSIDriver(storageClass)
 
-	fixture := newFixture(t)
+	fixture := newFixture(t, true, true)
 	// Object exist
-	fixture.storageClassObjects = append(fixture.storageClassObjects, storageClass)
+	fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
 	fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
+	fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
 	fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass)
 	fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass)
-	fixture.storageClassCapabilityObjects = append(fixture.storageClassCapabilityObjects, storageClassCapability)
+	fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability)
 	fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability)
 
 	// Action expected
-	fixture.expectUpdateStorageClassCapabilitiesAction(storageClassCapability)
+	fixture.expectUpdateStorageClassCapabilitiesAction(storageClassCapabilityUpdate)
 
 	// Run test
 	fixture.run(getKey(storageClass, t))
@@ -304,36 +371,70 @@ func TestDeleteStorageClass(t *testing.T) {
 	snapshotClass := newSnapshotClass(storageClass)
 	storageClassCapability := newStorageClassCapability(storageClass)
 
-	fixture := newFixture(t)
+	csiDriver := newCSIDriver(storageClass)
+
+	fixture := newFixture(t, true, true)
 	// Object exist
+	fixture.storageObjects = append(fixture.storageObjects, csiDriver)
+	fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
 	fixture.snapshotClassObjects = append(fixture.snapshotClassObjects, snapshotClass)
 	fixture.snapshotClassLister = append(fixture.snapshotClassLister, snapshotClass)
-	fixture.storageClassCapabilityObjects = append(fixture.storageClassCapabilityObjects, storageClassCapability)
+	fixture.capabilityObjects = append(fixture.capabilityObjects, storageClassCapability)
 	fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability)
 
 	// Action expected
+	fixture.expectDeleteSnapshotClassAction(snapshotClass)
 	fixture.expectDeleteStorageClassCapabilitiesAction(storageClassCapability)
 
 	// Run test
 	fixture.run(getKey(storageClass, t))
 }
 
-func TestDeleteSnapshotClass(t *testing.T) {
+func TestCreateStorageClassNotSupportSnapshot(t *testing.T) {
+	fixture := newFixture(t, false, true)
 	storageClass := newStorageClass("csi-example", "csi.example.com")
+	storageClassUpdate := storageClass.DeepCopy()
+	storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"}
 	storageClassCapability := newStorageClassCapability(storageClass)
+	storageClassCapability.Spec.Features.Snapshot.Create = false
+	storageClassCapability.Spec.Features.Snapshot.List = false
+	provisionerCapability := newProvisionerCapability(storageClass)
+	csiDriver := newCSIDriver(storageClass)
 
-	fixture := newFixture(t)
+	// Objects exist
-	// Object exist
+	fixture.storageObjects = append(fixture.storageObjects, storageClass, csiDriver)
-	fixture.storageClassCapabilityObjects = append(fixture.storageClassCapabilityObjects, storageClassCapability)
-	fixture.storageClassCapabilityLister = append(fixture.storageClassCapabilityLister, storageClassCapability)
-	fixture.storageClassObjects = append(fixture.storageClassObjects, storageClass)
 	fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
+	fixture.csiDriverLister = append(fixture.csiDriverLister, csiDriver)
+	fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability)
+	fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability)
 
 	// Action expected
-	storageClassCapabilityUpdate := storageClassCapability.DeepCopy()
+	fixture.expectUpdateStorageClassAction(storageClassUpdate)
-	storageClassCapabilityUpdate.Spec.Features.Snapshot.Create = false
+	fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
-	storageClassCapabilityUpdate.Spec.Features.Snapshot.List = false
+
-	fixture.expectUpdateStorageClassCapabilitiesAction(storageClassCapabilityUpdate)
+	// Run test
+	fixture.run(getKey(storageClass, t))
+}
+
+func TestCreateStorageClassInTree(t *testing.T) {
+	// InTree Storage has no snapshot capability
+	fixture := newFixture(t, true, true)
+	storageClass := newStorageClass("csi-example", "csi.example.com")
+	storageClassUpdate := storageClass.DeepCopy()
+	storageClassUpdate.Annotations = map[string]string{annotationSupportSnapshot: "false"}
+	storageClassCapability := newStorageClassCapability(storageClass)
+	storageClassCapability.Spec.Features.Snapshot.Create = false
+	provisionerCapability := newProvisionerCapability(storageClass)
+
+	// Objects exist
+	fixture.storageObjects = append(fixture.storageObjects, storageClass)
+	fixture.storageClassLister = append(fixture.storageClassLister, storageClass)
+	fixture.capabilityObjects = append(fixture.capabilityObjects, provisionerCapability)
+	fixture.provisionerCapabilityLister = append(fixture.provisionerCapabilityLister, provisionerCapability)
+
+	// Action expected
+	fixture.expectUpdateStorageClassAction(storageClassUpdate)
+	fixture.expectCreateStorageClassCapabilitiesAction(storageClassCapability)
 
 	// Run test
 	fixture.run(getKey(storageClass, t))

pkg/controller/storage/capability/csi_capability_test.go

@@ -19,6 +19,7 @@ package capability
 
 import (
 	"context"
+	"fmt"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
@@ -72,9 +73,9 @@ type fakeCSIServer struct {
 	server  *grpc.Server
 }
 
-func newTestCSIServer() (csiServer *fakeCSIServer, address string) {
+func newTestCSIServer(port int) (csiServer *fakeCSIServer, address string) {
 	if runtime.GOOS == "windows" {
-		address = "localhost:38886"
+		address = fmt.Sprintf("localhost:%d", +port)
 		csiServer = newFakeCSIServer("tcp", address)
 	} else {
 		address = filepath.Join(os.TempDir(), "csi.sock"+rand.String(4))
@@ -151,7 +152,7 @@ func (*fakeCSIServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
 }
 
 func Test_CSICapability(t *testing.T) {
-	fakeCSIServer, address := newTestCSIServer()
+	fakeCSIServer, address := newTestCSIServer(30087)
 	fakeCSIServer.run()
 	defer fakeCSIServer.stop()
 
@@ -168,9 +169,9 @@ func Test_CSICapability(t *testing.T) {
 
 func newStorageClassCapabilitySpec() *v1alpha1.StorageClassCapabilitySpec {
 	return &v1alpha1.StorageClassCapabilitySpec{
-		Features: v1alpha1.StorageClassCapabilitySpecFeatures{
+		Features: v1alpha1.CapabilityFeatures{
 			Topology: false,
-			Volume: v1alpha1.StorageClassCapabilitySpecFeaturesVolume{
+			Volume: v1alpha1.VolumeFeature{
 				Create: true,
 				Attach: false,
 				List:   false,
@@ -178,7 +179,7 @@ func newStorageClassCapabilitySpec() *v1alpha1.StorageClassCapabilitySpec {
 				Stats:  true,
 				Expand: v1alpha1.ExpandModeOffline,
 			},
-			Snapshot: v1alpha1.StorageClassCapabilitySpecFeaturesSnapshot{
+			Snapshot: v1alpha1.SnapshotFeature{
 				Create: true,
 				List:   false,
 			},

pkg/models/resources/v1alpha3/volumesnapshot/volumesnapshot.go

@@ -93,7 +93,7 @@ func (v *volumeSnapshotGetter) filter(object runtime.Object, filter query.Filter
 
 func snapshotStatus(item *v1beta1.VolumeSnapshot) string {
 	status := statusCreating
-	if *item.Status.ReadyToUse {
+	if item != nil && item.Status != nil && item.Status.ReadyToUse != nil && *item.Status.ReadyToUse {
 		status = statusReady
 	}
 	return status

pkg/models/resources/v1alpha3/volumesnapshot/volumesnapshot_test.go

@@ -178,5 +178,43 @@ func TestListVolumeSnapshot(t *testing.T) {
 		})
 	})
 
+	Describe("snapshot status", func() {
+		snapshot := newVolumeSnapshot("snap-0")
+		It("snapshot == nil", func() {
+			Expect(snapshotStatus(nil)).To(Equal(statusCreating))
+		})
+		It("snapshot.Status == nil", func() {
+			snapshot.Status = nil
+			Expect(snapshotStatus(snapshot)).To(Equal(statusCreating))
+		})
+		It("snapshot.Status.ReadyToUse == nil", func() {
+			snapshot.Status = &v1beta1.VolumeSnapshotStatus{
+				ReadyToUse: nil,
+			}
+			Expect(snapshotStatus(snapshot)).To(Equal(statusCreating))
+		})
+		It("snapshot.Status.ReadyToUse == false", func() {
+			readyToUse := false
+			snapshot.Status = &v1beta1.VolumeSnapshotStatus{
+				ReadyToUse: &readyToUse,
+			}
+			Expect(snapshotStatus(snapshot)).To(Equal(statusCreating))
+		})
+
+		It("snapshot.Status.ReadyToUse == true", func() {
+			readyToUse := true
+			snapshot.Status = &v1beta1.VolumeSnapshotStatus{
+				ReadyToUse: &readyToUse,
+			}
+			Expect(snapshotStatus(snapshot)).To(Equal(statusReady))
+		})
+	})
+
 	RunSpecs(t, "volume snapshot getter list")
 }
+
+//func TestVolumeSnapshotStatus( t *testing.T)  {
+//	RegisterFailHandler(Fail)
+//
+//	RunSpecs(t, "volume snapshot status")
+//}