[release-3.3] Fix ingress P95 delay time promql statement (#5132 )

Fix ingress P95 delay time promql statement Co-authored-by: Xinzhao Xu <z2d@jifangcheng.com>
[release-3.3] Adjust container terminal priority: bash, sh (#5076 )
2022-08-14 16:49:35 +08:00 · 2022-07-21 11:16:29 +08:00 · 2022-07-21 11:16:14 +08:00 · 2022-07-06 18:08:34 +08:00 · 2022-06-20 10:32:52 +08:00 · 2022-06-15 10:12:21 +08:00
454 changed files with 18139 additions and 15173 deletions
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +0,0 @@
-# exclude all files and folders except bin folder
-!bin
-.idea/
-.vscode/
--- a/.env
+++ b/.env
@@ -1,2 +0,0 @@
-DATA_PATH=./tmp
-KUBESPHERE_LOG_LEVEL=debug
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -1,5 +1,6 @@
 ---
 name: Bug report
+labels: ["kind/bug"]
 about: Create a report to help us improve
 ---

--- a/.github/workflows/kind/kind.yaml
+++ b/.github/workflows/kind/kind.yaml
@@ -2,7 +2,7 @@ kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
 nodes:
 - role: control-plane
-  image: kindest/node:v1.19.7
+  image: kindest/node:v1.21.1
  extraMounts:
  - hostPath: /etc/localtime
    containerPath: /etc/localtime
--- a/.gitmodules
+++ b/.gitmodules
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -0,0 +1,62 @@
+#
+#  Copyright 2022 The KubeSphere Authors.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+header:
+  license:
+    spdx-id: Apache-2.0
+    copyright-owner: KubeSphere Authors
+
+    content: |
+      Copyright 2022 The KubeSphere Authors.
+  
+      Licensed under the Apache License, Version 2.0 (the "License");
+      you may not use this file except in compliance with the License.
+      You may obtain a copy of the License at
+      
+          http://www.apache.org/licenses/LICENSE-2.0
+      
+      Unless required by applicable law or agreed to in writing, software
+      distributed under the License is distributed on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      See the License for the specific language governing permissions and
+      limitations under the License.
+
+
+  paths-ignore:
+    - 'api'
+    - 'build'
+    - 'docs'
+    - 'config'
+    - 'test'
+    - 'install'
+    - 'hack'
+    - 'vendor'
+    - 'staging'
+    - 'LICENSE'
+    - 'OWNERS'
+    - 'Makefile'
+    - 'pkg/client/**'
+    - 'pkg/simple/client/**'
+    - '**/*.md'
+    - '**/*.json'
+    - '**/go.mod'
+    - '**/go.sum'
+    - '.github/**'
+    - '.gitignore'
+    - '.gitattributes'
+    - 'pkg/controller/application/status.go'
+
+  comment: on-failure
--- a/25
+++ b/25
@@ -6,7 +6,11 @@
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"

-GV="network:v1alpha1 servicemesh:v1alpha2 tenant:v1alpha1 tenant:v1alpha2 devops:v1alpha1 iam:v1alpha2 devops:v1alpha3 cluster:v1alpha1 storage:v1alpha1 auditing:v1alpha1 types:v1beta1 quota:v1alpha2 application:v1alpha1 notification:v2beta1"
+GV="network:v1alpha1 servicemesh:v1alpha2 tenant:v1alpha1 tenant:v1alpha2 devops:v1alpha1 iam:v1alpha2 devops:v1alpha3 cluster:v1alpha1 storage:v1alpha1 auditing:v1alpha1 types:v1beta1 quota:v1alpha2 application:v1alpha1 notification:v2beta1 gateway:v1alpha1"
+MANIFESTS="application/* cluster/* iam/* network/v1alpha1 quota/* storage/* tenant/* gateway/*"
+
+# App Version
+APP_VERSION = v3.2.0

 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -52,7 +56,7 @@ binary: | ks-apiserver ks-controller-manager; $(info $(M)...Build all of binary.

 # Build ks-apiserver binary
 ks-apiserver: ; $(info $(M)...Begin to build ks-apiserver binary.)  @ ## Build ks-apiserver.
-	 hack/gobuild.sh cmd/ks-apiserver; 
+	 hack/gobuild.sh cmd/ks-apiserver;

 # Build ks-controller-manager binary
 ks-controller-manager: ; $(info $(M)...Begin to build ks-controller-manager binary.)  @ ## Build ks-controller-manager.
@@ -69,7 +73,7 @@ e2e: ;$(info $(M)...Begin to build e2e binary.)  @ ## Build e2e binary.
 kind-e2e: ;$(info $(M)...Run e2e test.) @ ## Run e2e test in kind.
 	hack/kind_e2e.sh

-# Run go fmt against code 
+# Run go fmt against code
 fmt: ;$(info $(M)...Begin to run go fmt against code.)  @ ## Run go fmt against code.
 	gofmt -w ./pkg ./cmd ./tools ./api

@@ -83,14 +87,7 @@ vet: ;$(info $(M)...Begin to run go vet against code.)  @ ## Run go vet against

 # Generate manifests e.g. CRD, RBAC etc.
 manifests: ;$(info $(M)...Begin to generate manifests e.g. CRD, RBAC etc..)  @ ## Generate manifests e.g. CRD, RBAC etc.
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/application/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/cluster/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/devops/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/iam/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/network/v1alpha1/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/quota/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/storage/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
-	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/tenant/... rbac:roleName=controller-perms ${CRD_OPTIONS} output:crd:artifacts:config=config/crds
+	hack/generate_manifests.sh ${CRD_OPTIONS} ${MANIFESTS}

 deploy: manifests ;$(info $(M)...Begin to deploy.)  @ ## Deploy.
 	kubectl apply -f config/crds
@@ -127,7 +124,7 @@ container-cross-push: ; $(info $(M)...Begin to build and push.)  @ ## Build and

 helm-package: ; $(info $(M)...Begin to helm-package.)  @ ## Helm-package.
 	ls config/crds/ | xargs -i cp -r config/crds/{} config/ks-core/crds/
-	helm package config/ks-core --app-version=v3.1.0 --version=0.1.0 -d ./bin
+	helm package config/ks-core --app-version=${APP_VERSION} --version=0.1.0 -d ./bin

 helm-deploy: ; $(info $(M)...Begin to helm-deploy.)  @ ## Helm-deploy.
 	ls config/crds/ | xargs -i cp -r config/crds/{} config/ks-core/crds/
@@ -158,3 +155,7 @@ clean: ;$(info $(M)...Begin to clean.)  @ ## Clean.

 clientset:  ;$(info $(M)...Begin to find or download controller-gen.)  @ ## Find or download controller-gen,download controller-gen if necessary.
 	./hack/generate_client.sh ${GV}
+
+# Fix invalid file's license.
+update-licenses: ;$(info $(M)...Begin to update licenses.)
+	@hack/update-licenses.sh
--- a/3
+++ b/3
@@ -1,3 +0,0 @@
-version: "1"
-domain: kubesphere.io
-repo: kubesphere.io/kubesphere
--- a/README.md
+++ b/README.md
@@ -1,5 +1,5 @@
 <p align="center">
-<img src="docs/images/kubesphere-logo.png" alt="banner" width="200px">
+<a href="https://kubesphere.io/"><img src="docs/images/kubesphere-icon.gif" alt="banner" width="200px"></a>
 </p>

 <p align="center">
@@ -45,7 +45,7 @@ The following screenshots give a close insight into KubeSphere. Please check [Wh
  </tr>
 </table>

-## Demo Environment
+## Demo environment

 🎮 Using the account `demo1 / Demo123` to log in the [demo environment](https://demo.kubesphere.io/). Please note the account is granted view access. 

@@ -103,6 +103,12 @@ The following screenshots give a close insight into KubeSphere. Please check [Wh
  Provide unified authentication with fine-grained roles and three-tier authorization system, and support AD/LDAP authentication.
  </details>

+<details>
+  <summary><b>🧠 GPU Workloads Scheduling and Monitoring</b></summary>
+  Create GPU workloads on the GUI, schedule GPU resources, and manage GPU resource quotas by tenant.
+  </details>
+
+
 ## Architecture

 KubeSphere uses a loosely-coupled architecture that separates the [frontend](https://github.com/kubesphere/console) from the [backend](https://github.com/kubesphere/kubesphere). External systems can access the components of the backend through the REST APIs. 
@@ -111,32 +117,43 @@ KubeSphere uses a loosely-coupled architecture that separates the [frontend](htt

 ----

-## Latest Release
-
-🎉 KubeSphere 3.1.1 is now available! See the [Release Notes For 3.1.1](https://kubesphere.io/docs/release/release-v311/) for the updates.
+## Latest release

+🎉 KubeSphere 3.2.1 was released on Dec 20! It brought enhancements and better user experience, see the [Release Notes For 3.2.1](https://kubesphere.io/docs/release/release-v321/) for the updates.
 ## Installation

-KubeSphere can run anywhere from on-premise datacenter to any cloud to edge. In addition, it can be deployed on any version-compatible Kubernetes cluster.
+KubeSphere can run anywhere from on-premise datacenter to any cloud to edge. In addition, it can be deployed on any version-compatible Kubernetes cluster. The installer will start a minimal installation by default, you can [enable other pluggable components before or after installation](https://kubesphere.io/docs/quick-start/enable-pluggable-components/).
+### Quick start
+#### Installing on K8s/K3s

-### Quick Start
-
-1. Run the following commands to install KubeSphere on an exiting Kubernetes cluster:
+If your cluster meets the [prerequisites](https://kubesphere.io/docs/quick-start/minimal-kubesphere-on-k8s/#prerequisites), then run the following commands to install KubeSphere on an exiting Kubernetes cluster:

 ```yaml
-kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/kubesphere-installer.yaml
+kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/kubesphere-installer.yaml
   
-kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/cluster-configuration.yaml
+kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/cluster-configuration.yaml
+```
+#### All-in-one
+
+👨‍💻 No Kubernetes? You can use [KubeKey](https://github.com/kubesphere/kubekey) to install both KubeSphere and Kubernetes/K3s in single-node mode on your Linux machine. Let's take K3s as an example:
+
+```yaml
+# Download KubeKey
+curl -sfL https://get-kk.kubesphere.io | VERSION=v2.0.0 sh -
+# Make kk executable
+chmod +x kk
+# Create a cluster
+./kk create cluster --with-kubernetes v1.21.4-k3s --with-kubesphere v3.2.1
 ```

-2. You can run the following command to view the installation logs. After KubeSphere is successfully installed, you can use `http://IP:30880` to access the KubeSphere Console with the default account and password (admin/P@88w0rd).
+You can run the following command to view the installation logs. After KubeSphere is successfully installed, you can access the KubeSphere web console at `http://IP:30880` and log in using the default administrator account (admin/P@88w0rd).

 ```yaml
 kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f
-```
-
-> 👨‍💻 No Kubernetes cluster? Try [All-in-one](https://kubesphere.io/docs/quick-start/all-in-one-on-linux/) to install a single-node Kubernetes and KubeSphere on your Linux machine. 
+``` 
+### 🐯 Katacoda for quick learning

+[Katacoda](https://www.katacoda.com/) allows you to explore how to install KubeSphere on an existing Kubernetes cluster in a browser. You can start the [Katacoda scenario with KubeSphere](https://www.katacoda.com/kubesphere/scenarios/install-kubesphere-on-kubernetes) in minutes.
 ### KubeSphere for hosted Kubernetes services

 KubeSphere is hosted on the following cloud providers, you can try KubeSphere by one-click installation on their hosted Kubernetes services. 
@@ -149,8 +166,7 @@ KubeSphere is hosted on the following cloud providers, you can try KubeSphere by
 You can also install KubeSphere on other hosted Kubernetes services within minutes, see the [step-by-step guides](https://kubesphere.io/docs/installing-on-kubernetes/) to get started.

 > 👨‍💻 No internet access? Refer to the [Air-gapped Installation on Kubernetes](https://kubesphere.io/docs/installing-on-kubernetes/on-prem-kubernetes/install-ks-on-linux-airgapped/) or [Air-gapped Installation on Linux](https://kubesphere.io/docs/installing-on-linux/introduction/air-gapped-installation/) for instructions on how to use private registry to install KubeSphere.
-
-## Contributing, Support, Discussion, and Community
+## Contributing, support, discussion, and community

 We :heart: your contribution. The [community](https://github.com/kubesphere/community) walks you through how to get started contributing KubeSphere. The [development guide](https://github.com/kubesphere/community/tree/master/developer-guide/development) explains how to set up development environment.

@@ -159,11 +175,9 @@ We :heart: your contribution. The [community](https://github.com/kubesphere/comm
 - [Follow us on Twitter](https://twitter.com/KubeSphere)

 Please submit any KubeSphere bugs, issues, and feature requests to [KubeSphere GitHub Issue](https://github.com/kubesphere/kubesphere/issues).
-
 ## Who are using KubeSphere

 The [user case studies](https://kubesphere.io/case/) page includes the user list of the project. You can [leave a comment](https://github.com/kubesphere/kubesphere/issues/4123) to let us know your use case.
-
 ## Landscapes

 <p align="center">
--- a/README_zh.md
+++ b/README_zh.md
@@ -1,9 +1,9 @@
 <p align="center">
-<img src="docs/images/kubesphere-logo.png" alt="banner" width="200px">
+<a href="https://kubesphere.com.cn/"><img src="docs/images/kubesphere-icon.gif" alt="banner" width="200px"></a>
 </p>

 <p align="center">
-<b>为<i> Kubernetes 多云、数据中心和边缘 </i>管理而定制的容器平台</b>
+<b>适用于<i> Kubernetes 多云、数据中心和边缘 </i>管理的容器平台</b>
 </p>

 <p align=center>
@@ -55,54 +55,59 @@

 <details>
  <summary><b>🕸 部署 Kubernetes 集群</b></summary>
-  支持在任何基础设施上部署 Kubernetes，支持在线安装和离线安装，<a href="https://kubesphere.io/zh/docs/installing-on-linux/introduction/intro/">了解更多</a> 。
+  支持在任何基础设施上部署 Kubernetes，支持在线安装和离线安装，<a href="https://kubesphere.io/zh/docs/installing-on-linux/introduction/intro/">了解更多</a>。
  </details>

 <details>
  <summary><b>🔗 Kubernetes 多集群管理</b></summary>
-  提供集中控制平台来管理多个 Kubernetes 集群，支持将应用程序发布到跨不同云供应商的多个k8集群上。
+  提供集中控制平台来管理多个 Kubernetes 集群，支持将应用程序发布到跨不同云供应商的多个 k8s 集群上。
  </details>

 <details>
  <summary><b>🤖 Kubernetes DevOps</b></summary>
-  提供开箱即用的基于 Jenkins 的 CI/CD，并内置自动化流水线插件，包括Binary-to-Image (B2I) 和Source-to-Image (S2I)，<a href="https://kubesphere.io/zh/devops/">了解更多</a> 。
+  提供开箱即用的基于 Jenkins 的 CI/CD，并内置自动化流水线插件，包括 Binary-to-Image (B2I) 和 Source-to-Image (S2I)，<a href="https://kubesphere.io/zh/devops/">了解更多</a>。
  </details>

 <details>
  <summary><b>🔎 云原生可观测性</b></summary>
-  支持多维度监控、事件和审计日志；内置多租户日志查询和收集，告警和通知，<a href="https://kubesphere.io/zh/observability/">了解更多</a> 。
+  支持多维度监控、事件和审计日志；内置多租户日志查询和收集，告警和通知，<a href="https://kubesphere.io/zh/observability/">了解更多</a>。
  </details>

 <details>
  <summary><b>🧩 基于 Istio 的微服务治理</b></summary>
-  为分布式微服务应用程序提供细粒度的流量管理、可观测性和服务跟踪，支持可视化的流量拓扑，<a href="https://kubesphere.io/zh/service-mesh/">了解更多</a> 。
+  为分布式微服务应用程序提供细粒度的流量管理、可观测性和服务跟踪，支持可视化的流量拓扑，<a href="https://kubesphere.io/zh/service-mesh/">了解更多</a>。
  </details>

 <details>
  <summary><b>💻 应用商店</b></summary>
-  为基于 Helm 的应用程序提供应用商店，并在 Kubernetes 平台上提供应用程序生命周期管理功能，<a href="https://kubesphere.io/zh/docs/pluggable-components/app-store/">了解更多</a> 。
+  为基于 Helm 的应用程序提供应用商店，并在 Kubernetes 平台上提供应用程序生命周期管理功能，<a href="https://kubesphere.io/zh/docs/pluggable-components/app-store/">了解更多</a>。
  </details>

 <details>
  <summary><b>💡 Kubernetes 边缘节点管理</b></summary>
-  基于 <a href="https://kubeedge.io/zh/">KubeEdge</a> 实现应用与工作负载在云端与边缘节点的统一分发与管理，解决在海量边、端设备上完成应用交付、运维、管控的需求，<a href= "https://kubesphere.io/zh/docs/pluggable-components/kubeedge/">了解更多</a> 。
+  基于 <a href="https://kubeedge.io/zh/">KubeEdge</a> 实现应用与工作负载在云端与边缘节点的统一分发与管理，解决在海量边、端设备上完成应用交付、运维、管控的需求，<a href= "https://kubesphere.io/zh/docs/pluggable-components/kubeedge/">了解更多</a>。
  </details>

 <details>
  <summary><b>📊 多维度计量与计费</b></summary>
-  提供基于集群与租户的多维度资源计量与计费的监控报表，让 Kubernetes 运营成本更透明，<a href="https://kubesphere.io/zh/docs/toolbox/metering-and-billing/view-resource-consumption/">了解更多</a> 。
+  提供基于集群与租户的多维度资源计量与计费的监控报表，让 Kubernetes 运营成本更透明，<a href="https://kubesphere.io/zh/docs/toolbox/metering-and-billing/view-resource-consumption/">了解更多</a>。
  </details>

 <details>
  <summary><b>🗃 支持多种存储和网络解决方案</b></summary>
-  <li>支持 GlusterFS、CephRBD、NFS、LocalPV ，并提供多个 CSI 插件对接公有云与企业级存储。</li><li>提供Kubernetes在裸机、边缘和虚拟化中的负载均衡器实现 <a href="https://github.com/kubesphere/openelb">OpenELB</a> 。</li><li>提供网络策略和容器组 IP 池管理，支持 Calico、Flannel、Kube-OVN。</li>
+  <li>支持 GlusterFS、CephRBD、NFS、LocalPV ，并提供多个 CSI 插件对接公有云与企业级存储。</li><li>提供 Kubernetes 在裸机、边缘和虚拟化中的负载均衡器实现 <a href="https://github.com/kubesphere/openelb">OpenELB</a> 。</li><li>提供网络策略和容器组 IP 池管理，支持 Calico、Flannel、Kube-OVN。</li>
  </details>

 <details>
-  <summary><b>🏘 多租户</b></summary>
+  <summary><b>🏘 多租户与统一鉴权认证</b></summary>
  提供统一的认证鉴权与细粒度的基于角色的授权系统，支持对接 AD/LDAP 。
  </details>

+<details>
+  <summary><b>🧠 GPU 工作负载调度与监控</b></summary>
+  支持可视化创建 GPU 工作负载，支持 GPU 监控，同时还支持对 GPU 资源进行租户级配额管理。
+  </details>
+
 ## 架构说明

 KubeSphere 使用前后端分离的架构，将 [前端](https://github.com/kubesphere/console) 与 [后端](https://github.com/kubesphere/kubesphere) 分开。后端的各个功能组件可通过 REST API 对接外部系统。
@@ -113,30 +118,41 @@ KubeSphere 使用前后端分离的架构，将 [前端](https://github.com/kube

 ## 最新版本

-🎉 KubeSphere 3.1.1 全新发布！相关更新信息，请参阅 [Release Notes For 3.1.1](https://kubesphere.io/zh/docs/release/release-v311/) 。
-
+🎉 KubeSphere 3.2.1 全新发布！！多项功能优化，带来更好的用户体验，详见 [v3.2.1 发行记录](https://kubesphere.com.cn/docs/release/release-v321/) 。
 ## 安装

-KubeSphere 支持在任意平台运行，从本地数据中心到混合多云再走向边缘。此外，KubeSphere 可以部署在任何版本兼容的 Kubernetes 集群上。
-
+KubeSphere 支持在任意平台运行，从本地数据中心到混合多云再走向边缘。此外，KubeSphere 可以部署在任何版本兼容的 Kubernetes 集群上。Installer 默认将执行最小化安装，您可以在安装前或安装后自定义[安装可插拔功能组件](https://kubesphere.com.cn/docs/quick-start/enable-pluggable-components/)。
 ### 快速入门
+#### 在 K8s/K3s 上安装

-1. 运行以下命令以在现有 Kubernetes 集群上安装 KubeSphere：
+请确保您的集群满足安装的[前提条件](https://kubesphere.io/zh/docs/quick-start/minimal-kubesphere-on-k8s/)，运行以下命令以在现有 Kubernetes 集群上安装 KubeSphere：

 ```yaml
-kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/kubesphere-installer.yaml
+kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/kubesphere-installer.yaml
   
-kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.1.1/cluster-configuration.yaml
+kubectl apply -f https://github.com/kubesphere/ks-installer/releases/download/v3.2.1/cluster-configuration.yaml
+```
+#### All-in-one（Linux 单节点安装）
+
+👨‍💻 没有 Kubernetes 集群? 可以用 [KubeKey](https://github.com/kubesphere/kubekey) 在 Linux 环境以 All-in-one 快速安装单节点 K8s/K3s 和 KubeSphere，下面以 K3s 为例：
+
+```yaml
+# 下载 KubeKey
+curl -sfL https://get-kk.kubesphere.io | VERSION=v1.2.0 sh -
+# 为 kk 赋予可执行权限
+chmod +x kk
+# 创建集群
+./kk create cluster --with-kubernetes v1.21.4-k3s --with-kubesphere v3.2.1
 ```

-2. 您可以运行以下命令查看安装日志。 KubeSphere 安装成功后，您可以使用`http://IP:30880` 以默认账号和密码（admin/P@88w0rd）访问KubeSphere 控制台。
+可使用以下命令查看安装日志。如果安装成功，可使用 `http://IP:30880` 访问 KubeSphere Console，管理员登录帐密为 `admin/P@88w0rd`。

 ```yaml
 kubectl logs -n kubesphere-system $(kubectl get pod -n kubesphere-system -l app=ks-install -o jsonpath='{.items[0].metadata.name}') -f
-```
-
-> 👨‍💻 没有 Kubernetes 集群? 可以尝试在 Linux 上以[All-in-one](https://kubesphere.io/zh/docs/quick-start/all-in-one-on-linux/) 模式来安装单节点 Kubernetes 和 KubeSphere。
+``` 
+### 🐯 使用 Katacoda 在线安装体验 KubeSphere

+[Katacoda](https://www.katacoda.com/) 是一个在线的云原生技术学习实验平台，你可以使用它在浏览器中快速 [安装体验 KubeSphere](https://www.katacoda.com/kubesphere/scenarios/install-kubesphere-on-kubernetes) 。
 ### 在托管 Kubernetes 上部署 KubeSphere

 KubeSphere 托管在以下云供应商上，您可以通过在其托管的 Kubernetes 服务上一键安装来部署 KubeSphere。
@@ -157,10 +173,10 @@ KubeSphere 托管在以下云供应商上，您可以通过在其托管的 Kuber
 - [中文论坛](https://kubesphere.com.cn/forum/)
 - [社区微信群（见官网底部）](https://kubesphere.com.cn/)
 - [Slack Channel](https://join.slack.com/t/kubesphere/shared_invite/enQtNTE3MDIxNzUxNzQ0LTZkNTdkYWNiYTVkMTM5ZThhODY1MjAyZmVlYWEwZmQ3ODQ1NmM1MGVkNWEzZTRhNzk0MzM5MmY4NDc3ZWVhMjE)
- [Youtube](https://www.youtube.com/channel/UCyTdUQUYjf7XLjxECx63Hpw)
+- [Bilibili](https://space.bilibili.com/438908638)
 - [在推特上关注我们](https://twitter.com/KubeSphere)

-请将任何 KubeSphere 错误、问题和功能请求提交到 [KubeSphere GitHub Issue](https://github.com/kubesphere/kubesphere/issues) 。
+请将任何 KubeSphere 的 Bug、问题和需求提交到 [KubeSphere GitHub Issue](https://github.com/kubesphere/kubesphere/issues) 。

 ## 谁在使用 KubeSphere

--- a/SECURITY.md
+++ b/SECURITY.md
@@ -0,0 +1,50 @@
+# Security Policy
+
+## Supported Versions
+
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 3.2.x   | :white_check_mark: |
+| 3.1.x   | :white_check_mark: |
+| 3.0.x   | :white_check_mark: |
+| 2.1.x   | :white_check_mark: |
+| < 2.1.x   | :x:                |
+
+## Reporting a Vulnerability
+
+# Security Vulnerability Disclosure and Response Process
+
+To ensure KubeSphere security, a security vulnerability disclosure and response process is adopted. And the security team is set up in KubeSphere community, also any issue and PR is welcome for every contributors.
+
+The primary goal of this process is to reduce the total exposure time of users to publicly known vulnerabilities. To quickly fix vulnerabilities of KubeSphere, the security team is responsible for the entire vulnerability management process, including internal communication and external disclosure.
+
+If you find a vulnerability or encounter a security incident involving vulnerabilities of KubeSphere, please report it as soon as possible to the KubeSphere security team (security@kubesphere.io).
+
+Please kindly help provide as much vulnerability information as possible in the following format:
+
+- Issue title(Please add 'Security' lable)*:
+
+- Overview*:
+
+- Affected components and version number*:
+
+- CVE number (if any):
+
+- Vulnerability verification process*:
+
+- Contact information*:
+
+The asterisk (*) indicates the required field.
+
+# Response Time
+
+The KubeSphere security team will confirm the vulnerabilities and contact you within 2 working days after your submission.
+
+We will publicly thank you after fixing the security vulnerability. To avoid negative impact, please keep the vulnerability confidential until we fix it. We would appreciate it if you could obey the following code of conduct:
+
+The vulnerability will not be disclosed until KubeSphere releases a patch for it.
+
+The details of the vulnerability, for example, exploits code, will not be disclosed.
--- a/api/ks-openapi-spec/swagger.json
+++ b/api/ks-openapi-spec/swagger.json
--- a/api/openapi-spec/swagger.json
+++ b/api/openapi-spec/swagger.json
@@ -10720,6 +10720,10 @@
          "description": "Desired state of the cluster",
          "type": "boolean"
        },
+        "externalKubeAPIEnabled": {
+          "description": "ExternalKubeAPIEnabled export kubeapiserver to public use a lb type service if connection type is proxy",
+          "type": "boolean"
+        },
        "joinFederation": {
          "description": "Join cluster as a kubefed cluster",
          "type": "boolean"
@@ -10766,6 +10770,10 @@
          "description": "Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.",
          "type": "string"
        },
+        "uid": {
+          "description": "UID is the kube-system namespace UID of the cluster, which represents the unique ID of the cluster.",
+          "type": "string"
+        },
        "zones": {
          "description": "Zones are the names of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.",
          "type": "array",
@@ -10779,6 +10787,10 @@
    "io.kubesphere.api.cluster.v1alpha1.Connection": {
      "type": "object",
      "properties": {
+        "externalKubernetesAPIEndpoint": {
+          "description": "External Kubernetes API Server endpoint Will be populated by ks-apiserver if connection type is proxy and ExternalKubeAPIEnabled is true.",
+          "type": "string"
+        },
        "kubeconfig": {
          "description": "KubeConfig content used to connect to cluster api server Should provide this field explicitly if connection type is direct. Will be populated by ks-proxy if connection type is proxy.",
          "type": "string",
--- a/build/ks-controller-manager/Dockerfile
+++ b/build/ks-controller-manager/Dockerfile
@@ -9,7 +9,7 @@ ARG TARGETARCH
 ARG TARGETOS
 ARG HELM_VERSION=v3.5.2
 ARG KUSTOMIZE_VERSION=v4.2.0
-ARG INGRESS_NGINX_VERSION=3.35.0
+ARG INGRESS_NGINX_VERSION=4.0.13

 ENV OUTDIR=/out
 RUN mkdir -p ${OUTDIR}/usr/local/bin
@@ -26,7 +26,7 @@ RUN mv /tmp/${TARGETOS}-${TARGETARCH}/helm ${OUTDIR}/usr/local/bin/

 # install kustomize
 ADD https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2F${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz /tmp
-RUN tar xvzf /tmp/kustomize_${KUSTOMIZE_VERSION}_linux_${TARGETARCH}.tar.gz -C /tmp
+RUN tar xvzf /tmp/kustomize_${KUSTOMIZE_VERSION}_${TARGETOS}_${TARGETARCH}.tar.gz -C /tmp
 RUN mv /tmp/kustomize ${OUTDIR}/usr/local/bin/


--- a/cmd/controller-manager/app/controllers.go
+++ b/cmd/controller-manager/app/controllers.go
@@ -17,15 +17,41 @@ limitations under the License.
 package app

 import (
+	"fmt"
+	"time"
+
+	"github.com/kubesphere/pvc-autoresizer/runners"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog"
+	ctrl "sigs.k8s.io/controller-runtime"
 	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/kubefed/pkg/controller/util"

-	"kubesphere.io/kubesphere/pkg/controller/storage/snapshotclass"
+	"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
+	"kubesphere.io/kubesphere/pkg/controller/application"
+	"kubesphere.io/kubesphere/pkg/controller/helm"
+	"kubesphere.io/kubesphere/pkg/controller/namespace"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmapplication"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmcategory"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrelease"
+	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrepo"
+	"kubesphere.io/kubesphere/pkg/controller/quota"
+	"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
+	"kubesphere.io/kubesphere/pkg/controller/user"
+	"kubesphere.io/kubesphere/pkg/controller/workspace"
+	"kubesphere.io/kubesphere/pkg/controller/workspacerole"
+	"kubesphere.io/kubesphere/pkg/controller/workspacerolebinding"
+	"kubesphere.io/kubesphere/pkg/controller/workspacetemplate"
+	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
+	"kubesphere.io/kubesphere/pkg/simple/client/devops"
+	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
+	ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
+	"kubesphere.io/kubesphere/pkg/simple/client/s3"

-	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
+	"kubesphere.io/kubesphere/pkg/controller/storage/snapshotclass"

 	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"

@@ -46,198 +72,506 @@ import (
 	"kubesphere.io/kubesphere/pkg/controller/storage/capability"
 	"kubesphere.io/kubesphere/pkg/controller/virtualservice"
 	"kubesphere.io/kubesphere/pkg/informers"
-	"kubesphere.io/kubesphere/pkg/simple/client/devops"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
-	ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
-	"kubesphere.io/kubesphere/pkg/simple/client/multicluster"
-	"kubesphere.io/kubesphere/pkg/simple/client/network"
 	ippoolclient "kubesphere.io/kubesphere/pkg/simple/client/network/ippool"
-	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 )

-func addControllers(
-	mgr manager.Manager,
-	client k8s.Client,
-	informerFactory informers.InformerFactory,
-	devopsClient devops.Interface,
-	s3Client s3.Interface,
-	ldapClient ldapclient.Interface,
-	options *k8s.KubernetesOptions,
-	authenticationOptions *authentication.Options,
-	multiClusterOptions *multicluster.Options,
-	networkOptions *network.Options,
-	serviceMeshEnabled bool,
-	kubectlImage string,
-	stopCh <-chan struct{}) error {
+var allControllers = []string{
+	"user",
+	"workspacetemplate",
+	"workspace",
+	"workspacerole",
+	"workspacerolebinding",
+	"namespace",

+	"helmrepo",
+	"helmcategory",
+	"helmapplication",
+	"helmapplicationversion",
+	"helmrelease",
+	"helm",
+
+	"application",
+	"serviceaccount",
+	"resourcequota",
+
+	"virtualservice",
+	"destinationrule",
+	"job",
+	"storagecapability",
+	"volumesnapshot",
+	"pvcautoresizer",
+	"workloadrestart",
+	"loginrecord",
+	"cluster",
+	"nsnp",
+	"ippool",
+	"csr",
+
+	"clusterrolebinding",
+
+	"fedglobalrolecache",
+	"globalrole",
+	"fedglobalrolebindingcache",
+	"globalrolebinding",
+
+	"groupbinding",
+	"group",
+
+	"notification",
+}
+
+// setup all available controllers one by one
+func addAllControllers(mgr manager.Manager, client k8s.Client, informerFactory informers.InformerFactory,
+	cmOptions *options.KubeSphereControllerManagerOptions,
+	stopCh <-chan struct{}) error {
+	var err error
+
+	////////////////////////////////////
+	// begin init necessary informers
+	////////////////////////////////////
 	kubernetesInformer := informerFactory.KubernetesSharedInformerFactory()
 	istioInformer := informerFactory.IstioSharedInformerFactory()
 	kubesphereInformer := informerFactory.KubeSphereSharedInformerFactory()
+	////////////////////////////////////
+	// end informers
+	////////////////////////////////////

-	multiClusterEnabled := multiClusterOptions.Enable
+	////////////////////////////////////
+	// begin init necessary clients
+	////////////////////////////////////
+	kubeconfigClient := kubeconfig.NewOperator(client.Kubernetes(),
+		informerFactory.KubernetesSharedInformerFactory().Core().V1().ConfigMaps().Lister(),
+		client.Config())

-	var vsController, drController manager.Runnable
+	var devopsClient devops.Interface
+	if cmOptions.DevopsOptions != nil && len(cmOptions.DevopsOptions.Host) != 0 {
+		devopsClient, err = jenkins.NewDevopsClient(cmOptions.DevopsOptions)
+		if err != nil {
+			return fmt.Errorf("failed to connect jenkins, please check jenkins status, error: %v", err)
+		}
+	}
+
+	var ldapClient ldapclient.Interface
+	// when there is no ldapOption, we set ldapClient as nil, which means we don't need to sync user info into ldap.
+	if cmOptions.LdapOptions != nil && len(cmOptions.LdapOptions.Host) != 0 {
+		if cmOptions.LdapOptions.Host == ldapclient.FAKE_HOST { // for debug only
+			ldapClient = ldapclient.NewSimpleLdap()
+		} else {
+			ldapClient, err = ldapclient.NewLdapClient(cmOptions.LdapOptions, stopCh)
+			if err != nil {
+				return fmt.Errorf("failed to connect to ldap service, please check ldap status, error: %v", err)
+			}
+		}
+	} else {
+		klog.Warning("ks-controller-manager starts without ldap provided, it will not sync user into ldap")
+	}
+	////////////////////////////////////
+	// end init clients
+	////////////////////////////////////
+
+	////////////////////////////////////////////////////////
+	// begin init controller and add to manager one by one
+	////////////////////////////////////////////////////////
+
+	// "user" controller
+	if cmOptions.IsControllerEnabled("user") {
+		userController := &user.Reconciler{
+			MultiClusterEnabled:     cmOptions.MultiClusterOptions.Enable,
+			MaxConcurrentReconciles: 4,
+			LdapClient:              ldapClient,
+			DevopsClient:            devopsClient,
+			KubeconfigClient:        kubeconfigClient,
+			AuthenticationOptions:   cmOptions.AuthenticationOptions,
+		}
+		addControllerWithSetup(mgr, "user", userController)
+	}
+
+	// "workspacetemplate" controller
+	if cmOptions.IsControllerEnabled("workspacetemplate") {
+		workspaceTemplateReconciler := &workspacetemplate.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
+		addControllerWithSetup(mgr, "workspacetemplate", workspaceTemplateReconciler)
+	}
+
+	// "workspace" controller
+	if cmOptions.IsControllerEnabled("workspace") {
+		workspaceReconciler := &workspace.Reconciler{}
+		addControllerWithSetup(mgr, "workspace", workspaceReconciler)
+	}
+
+	// "workspacerole" controller
+	if cmOptions.IsControllerEnabled("workspacerole") {
+		workspaceRoleReconciler := &workspacerole.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
+		addControllerWithSetup(mgr, "workspacerole", workspaceRoleReconciler)
+	}
+
+	// "workspacerolebinding" controller
+	if cmOptions.IsControllerEnabled("workspacerolebinding") {
+		workspaceRoleBindingReconciler := &workspacerolebinding.Reconciler{MultiClusterEnabled: cmOptions.MultiClusterOptions.Enable}
+		addControllerWithSetup(mgr, "workspacerolebinding", workspaceRoleBindingReconciler)
+	}
+
+	// "namespace" controller
+	if cmOptions.IsControllerEnabled("namespace") {
+		namespaceReconciler := &namespace.Reconciler{GatewayOptions: cmOptions.GatewayOptions}
+		addControllerWithSetup(mgr, "namespace", namespaceReconciler)
+	}
+
+	// "helmrepo" controller
+	if cmOptions.IsControllerEnabled("helmrepo") {
+		helmRepoReconciler := &helmrepo.ReconcileHelmRepo{}
+		addControllerWithSetup(mgr, "helmrepo", helmRepoReconciler)
+	}
+
+	// "helmcategory" controller
+	if cmOptions.IsControllerEnabled("helmcategory") {
+		helmCategoryReconciler := &helmcategory.ReconcileHelmCategory{}
+		addControllerWithSetup(mgr, "helmcategory", helmCategoryReconciler)
+	}
+
+	var opS3Client s3.Interface
+	if !cmOptions.OpenPitrixOptions.AppStoreConfIsEmpty() {
+		opS3Client, err = s3.NewS3Client(cmOptions.OpenPitrixOptions.S3Options)
+		if err != nil {
+			klog.Fatalf("failed to connect to s3, please check openpitrix s3 service status, error: %v", err)
+		}
+
+		// "helmapplication" controller
+		if cmOptions.IsControllerEnabled("helmapplication") {
+			reconcileHelmApp := (&helmapplication.ReconcileHelmApplication{})
+			addControllerWithSetup(mgr, "helmapplication", reconcileHelmApp)
+		}
+
+		// "helmapplicationversion" controller
+		if cmOptions.IsControllerEnabled("helmapplicationversion") {
+			reconcileHelmAppVersion := (&helmapplication.ReconcileHelmApplicationVersion{})
+			addControllerWithSetup(mgr, "helmapplicationversion", reconcileHelmAppVersion)
+		}
+	}
+
+	// "helmrelease" controller
+	if cmOptions.IsControllerEnabled("helmrelease") {
+		reconcileHelmRelease := &helmrelease.ReconcileHelmRelease{
+			// nil interface is valid value.
+			StorageClient:      opS3Client,
+			KsFactory:          informerFactory.KubeSphereSharedInformerFactory(),
+			MultiClusterEnable: cmOptions.MultiClusterOptions.Enable,
+			WaitTime:           cmOptions.OpenPitrixOptions.ReleaseControllerOptions.WaitTime,
+			MaxConcurrent:      cmOptions.OpenPitrixOptions.ReleaseControllerOptions.MaxConcurrent,
+			StopChan:           stopCh,
+		}
+		addControllerWithSetup(mgr, "helmrelease", reconcileHelmRelease)
+	}
+
+	// "helm" controller
+	if cmOptions.IsControllerEnabled("helm") {
+		if !cmOptions.GatewayOptions.IsEmpty() {
+			helmReconciler := &helm.Reconciler{GatewayOptions: cmOptions.GatewayOptions}
+			addControllerWithSetup(mgr, "helm", helmReconciler)
+		}
+	}
+
+	// "application" controller
+	if cmOptions.IsControllerEnabled("application") {
+		selector, _ := labels.Parse(cmOptions.ApplicationSelector)
+		applicationReconciler := &application.ApplicationReconciler{
+			Scheme:              mgr.GetScheme(),
+			Client:              mgr.GetClient(),
+			Mapper:              mgr.GetRESTMapper(),
+			ApplicationSelector: selector,
+		}
+		addControllerWithSetup(mgr, "application", applicationReconciler)
+	}
+
+	// "serviceaccount" controller
+	if cmOptions.IsControllerEnabled("serviceaccount") {
+		saReconciler := &serviceaccount.Reconciler{}
+		addControllerWithSetup(mgr, "serviceaccount", saReconciler)
+	}
+
+	// "resourcequota" controller
+	if cmOptions.IsControllerEnabled("resourcequota") {
+		resourceQuotaReconciler := &quota.Reconciler{
+			MaxConcurrentReconciles: quota.DefaultMaxConcurrentReconciles,
+			ResyncPeriod:            quota.DefaultResyncPeriod,
+			InformerFactory:         informerFactory.KubernetesSharedInformerFactory(),
+		}
+		addControllerWithSetup(mgr, "resourcequota", resourceQuotaReconciler)
+	}
+
+	serviceMeshEnabled := cmOptions.ServiceMeshOptions != nil && len(cmOptions.ServiceMeshOptions.IstioPilotHost) != 0
 	if serviceMeshEnabled {
-		vsController = virtualservice.NewVirtualServiceController(kubernetesInformer.Core().V1().Services(),
-			istioInformer.Networking().V1alpha3().VirtualServices(),
-			istioInformer.Networking().V1alpha3().DestinationRules(),
-			kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
+		// "virtualservice" controller
+		if cmOptions.IsControllerEnabled("virtualservice") {
+			vsController := virtualservice.NewVirtualServiceController(kubernetesInformer.Core().V1().Services(),
+				istioInformer.Networking().V1alpha3().VirtualServices(),
+				istioInformer.Networking().V1alpha3().DestinationRules(),
+				kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
+				client.Kubernetes(),
+				client.Istio(),
+				client.KubeSphere())
+			addController(mgr, "virtualservice", vsController)
+		}
+
+		// "destinationrule" controller
+		if cmOptions.IsControllerEnabled("destinationrule") {
+			drController := destinationrule.NewDestinationRuleController(kubernetesInformer.Apps().V1().Deployments(),
+				istioInformer.Networking().V1alpha3().DestinationRules(),
+				kubernetesInformer.Core().V1().Services(),
+				kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
+				client.Kubernetes(),
+				client.Istio(),
+				client.KubeSphere())
+			addController(mgr, "destinationrule", drController)
+		}
+	}
+
+	// "job" controller
+	if cmOptions.IsControllerEnabled("job") {
+		jobController := job.NewJobController(kubernetesInformer.Batch().V1().Jobs(), client.Kubernetes())
+		addController(mgr, "job", jobController)
+	}
+
+	// "storagecapability" controller
+	if cmOptions.IsControllerEnabled("storagecapability") {
+		storageCapabilityController := capability.NewController(
+			client.Kubernetes().StorageV1().StorageClasses(),
+			kubernetesInformer.Storage().V1().StorageClasses(),
+			kubernetesInformer.Storage().V1().CSIDrivers(),
+		)
+		addController(mgr, "storagecapability", storageCapabilityController)
+	}
+
+	// "volumesnapshot" controller
+	if cmOptions.IsControllerEnabled("volumesnapshot") {
+		volumeSnapshotController := snapshotclass.NewController(
+			kubernetesInformer.Storage().V1().StorageClasses(),
+			client.Snapshot().SnapshotV1().VolumeSnapshotClasses(),
+			informerFactory.SnapshotSharedInformerFactory().Snapshot().V1().VolumeSnapshotClasses(),
+		)
+		addController(mgr, "volumesnapshot", volumeSnapshotController)
+	}
+
+	// "pvc-autoresizer"
+	monitoringOptionsEnable := cmOptions.MonitoringOptions != nil && len(cmOptions.MonitoringOptions.Endpoint) != 0
+	if monitoringOptionsEnable {
+		if cmOptions.IsControllerEnabled("pvc-autoresizer") {
+			if err := runners.SetupIndexer(mgr, false); err != nil {
+				return err
+			}
+			promClient, err := runners.NewPrometheusClient(cmOptions.MonitoringOptions.Endpoint)
+			if err != nil {
+				return err
+			}
+			pvcAutoResizerController := runners.NewPVCAutoresizer(
+				promClient,
+				mgr.GetClient(),
+				ctrl.Log.WithName("pvc-autoresizer"),
+				1*time.Minute,
+				mgr.GetEventRecorderFor("pvc-autoresizer"),
+			)
+			addController(mgr, "pvcautoresizer", pvcAutoResizerController)
+		}
+	}
+
+	if cmOptions.IsControllerEnabled("pvc-workload-restarter") {
+		restarter := runners.NewRestarter(
+			mgr.GetClient(),
+			ctrl.Log.WithName("pvc-workload-restarter"),
+			1*time.Minute,
+			mgr.GetEventRecorderFor("pvc-workload-restarter"),
+		)
+		addController(mgr, "pvcworkloadrestarter", restarter)
+	}
+
+	// "loginrecord" controller
+	if cmOptions.IsControllerEnabled("loginrecord") {
+		loginRecordController := loginrecord.NewLoginRecordController(
 			client.Kubernetes(),
-			client.Istio(),
-			client.KubeSphere())
-
-		drController = destinationrule.NewDestinationRuleController(kubernetesInformer.Apps().V1().Deployments(),
-			istioInformer.Networking().V1alpha3().DestinationRules(),
-			kubernetesInformer.Core().V1().Services(),
-			kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
-			client.Kubernetes(),
-			client.Istio(),
-			client.KubeSphere())
+			client.KubeSphere(),
+			kubesphereInformer.Iam().V1alpha2().LoginRecords(),
+			kubesphereInformer.Iam().V1alpha2().Users(),
+			cmOptions.AuthenticationOptions.LoginHistoryRetentionPeriod,
+			cmOptions.AuthenticationOptions.LoginHistoryMaximumEntries)
+		addController(mgr, "loginrecord", loginRecordController)
 	}

-	jobController := job.NewJobController(kubernetesInformer.Batch().V1().Jobs(), client.Kubernetes())
+	// "csr" controller
+	if cmOptions.IsControllerEnabled("csr") {
+		csrController := certificatesigningrequest.NewController(client.Kubernetes(),
+			kubernetesInformer.Certificates().V1().CertificateSigningRequests(),
+			kubernetesInformer.Core().V1().ConfigMaps(), client.Config())
+		addController(mgr, "csr", csrController)
+	}

-	storageCapabilityController := capability.NewController(
-		client.Kubernetes().StorageV1().StorageClasses(),
-		kubernetesInformer.Storage().V1().StorageClasses(),
-		kubernetesInformer.Storage().V1().CSIDrivers(),
-	)
+	// "clusterrolebinding" controller
+	if cmOptions.IsControllerEnabled("clusterrolebinding") {
+		clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(),
+			kubernetesInformer.Rbac().V1().ClusterRoleBindings(),
+			kubernetesInformer.Apps().V1().Deployments(),
+			kubernetesInformer.Core().V1().Pods(),
+			kubesphereInformer.Iam().V1alpha2().Users(),
+			cmOptions.AuthenticationOptions.KubectlImage)
+		addController(mgr, "clusterrolebinding", clusterRoleBindingController)
+	}

-	volumeSnapshotController := snapshotclass.NewController(
-		kubernetesInformer.Storage().V1().StorageClasses(),
-		client.Snapshot().SnapshotV1().VolumeSnapshotClasses(),
-		informerFactory.SnapshotSharedInformerFactory().Snapshot().V1().VolumeSnapshotClasses(),
-	)
-
-	var fedGlobalRoleBindingCache, fedGlobalRoleCache cache.Store
-	var fedGlobalRoleBindingCacheController, fedGlobalRoleCacheController cache.Controller
-
-	if multiClusterEnabled {
-		fedGlobalRoleClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleResource)
-		if err != nil {
-			klog.Error(err)
-			return err
+	// "fedglobalrolecache" controller
+	var fedGlobalRoleCache cache.Store
+	var fedGlobalRoleCacheController cache.Controller
+	if cmOptions.IsControllerEnabled("fedglobalrolecache") {
+		if cmOptions.MultiClusterOptions.Enable {
+			fedGlobalRoleClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleResource)
+			if err != nil {
+				klog.Fatalf("Unable to create FedGlobalRole controller: %v", err)
+			}
+			fedGlobalRoleCache, fedGlobalRoleCacheController = util.NewResourceInformer(fedGlobalRoleClient, "",
+				&iamv1alpha2.FedGlobalRoleResource, func(object runtimeclient.Object) {})
+			go fedGlobalRoleCacheController.Run(stopCh)
+			addSuccessfullyControllers.Insert("fedglobalrolecache")
 		}
-		fedGlobalRoleBindingClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleBindingResource)
-		if err != nil {
-			klog.Error(err)
-			return err
+	}
+
+	// "globalrole" controller
+	if cmOptions.IsControllerEnabled("globalrole") {
+		if cmOptions.MultiClusterOptions.Enable {
+			globalRoleController := globalrole.NewController(client.Kubernetes(), client.KubeSphere(),
+				kubesphereInformer.Iam().V1alpha2().GlobalRoles(), fedGlobalRoleCache, fedGlobalRoleCacheController)
+			addController(mgr, "globalrole", globalRoleController)
 		}
-
-		fedGlobalRoleCache, fedGlobalRoleCacheController = util.NewResourceInformer(fedGlobalRoleClient, "", &iamv1alpha2.FedGlobalRoleResource, func(object runtimeclient.Object) {})
-		fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController = util.NewResourceInformer(fedGlobalRoleBindingClient, "", &iamv1alpha2.FedGlobalRoleBindingResource, func(object runtimeclient.Object) {})
-
-		go fedGlobalRoleCacheController.Run(stopCh)
-		go fedGlobalRoleBindingCacheController.Run(stopCh)
 	}

-	loginRecordController := loginrecord.NewLoginRecordController(
-		client.Kubernetes(),
-		client.KubeSphere(),
-		kubesphereInformer.Iam().V1alpha2().LoginRecords(),
-		kubesphereInformer.Iam().V1alpha2().Users(),
-		authenticationOptions.LoginHistoryRetentionPeriod,
-		authenticationOptions.LoginHistoryMaximumEntries)
-
-	csrController := certificatesigningrequest.NewController(client.Kubernetes(),
-		kubernetesInformer.Certificates().V1().CertificateSigningRequests(),
-		kubernetesInformer.Core().V1().ConfigMaps(), client.Config())
-
-	clusterRoleBindingController := clusterrolebinding.NewController(client.Kubernetes(),
-		kubernetesInformer.Rbac().V1().ClusterRoleBindings(),
-		kubernetesInformer.Apps().V1().Deployments(),
-		kubernetesInformer.Core().V1().Pods(),
-		kubesphereInformer.Iam().V1alpha2().Users(),
-		kubectlImage)
-
-	globalRoleController := globalrole.NewController(client.Kubernetes(), client.KubeSphere(),
-		kubesphereInformer.Iam().V1alpha2().GlobalRoles(), fedGlobalRoleCache, fedGlobalRoleCacheController)
-
-	globalRoleBindingController := globalrolebinding.NewController(client.Kubernetes(), client.KubeSphere(),
-		kubesphereInformer.Iam().V1alpha2().GlobalRoleBindings(),
-		fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController,
-		multiClusterEnabled)
-
-	groupBindingController := groupbinding.NewController(client.Kubernetes(), client.KubeSphere(),
-		kubesphereInformer.Iam().V1alpha2().GroupBindings(),
-		kubesphereInformer.Types().V1beta1().FederatedGroupBindings(),
-		multiClusterEnabled)
-
-	groupController := group.NewController(client.Kubernetes(), client.KubeSphere(),
-		kubesphereInformer.Iam().V1alpha2().Groups(),
-		kubesphereInformer.Types().V1beta1().FederatedGroups(),
-		multiClusterEnabled)
-
-	var clusterController manager.Runnable
-	if multiClusterEnabled {
-		clusterController = cluster.NewClusterController(
-			client.Kubernetes(),
-			client.Config(),
-			kubesphereInformer.Cluster().V1alpha1().Clusters(),
-			client.KubeSphere().ClusterV1alpha1().Clusters(),
-			multiClusterOptions.ClusterControllerResyncPeriod,
-			multiClusterOptions.HostClusterName)
-	}
-
-	var nsnpController manager.Runnable
-	if networkOptions.EnableNetworkPolicy {
-		nsnpProvider, err := provider.NewNsNetworkPolicyProvider(client.Kubernetes(), kubernetesInformer.Networking().V1().NetworkPolicies())
-		if err != nil {
-			return err
+	// "fedglobalrolebindingcache" controller
+	var fedGlobalRoleBindingCache cache.Store
+	var fedGlobalRoleBindingCacheController cache.Controller
+	if cmOptions.IsControllerEnabled("fedglobalrolebindingcache") {
+		if cmOptions.MultiClusterOptions.Enable {
+			fedGlobalRoleBindingClient, err := util.NewResourceClient(client.Config(), &iamv1alpha2.FedGlobalRoleBindingResource)
+			if err != nil {
+				klog.Fatalf("Unable to create FedGlobalRoleBinding controller: %v", err)
+			}
+			fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController = util.NewResourceInformer(fedGlobalRoleBindingClient, "",
+				&iamv1alpha2.FedGlobalRoleBindingResource, func(object runtimeclient.Object) {})
+			go fedGlobalRoleBindingCacheController.Run(stopCh)
+			addSuccessfullyControllers.Insert("fedglobalrolebindingcache")
 		}
-
-		nsnpController = nsnetworkpolicy.NewNSNetworkPolicyController(client.Kubernetes(),
-			client.KubeSphere().NetworkV1alpha1(),
-			kubesphereInformer.Network().V1alpha1().NamespaceNetworkPolicies(),
-			kubernetesInformer.Core().V1().Services(),
-			kubernetesInformer.Core().V1().Nodes(),
-			kubesphereInformer.Tenant().V1alpha1().Workspaces(),
-			kubernetesInformer.Core().V1().Namespaces(), nsnpProvider, networkOptions.NSNPOptions)
 	}

-	var ippoolController manager.Runnable
-	ippoolProvider := ippoolclient.NewProvider(kubernetesInformer, client.KubeSphere(), client.Kubernetes(), networkOptions.IPPoolType, options)
-	if ippoolProvider != nil {
-		ippoolController = ippool.NewIPPoolController(kubesphereInformer, kubernetesInformer, client.Kubernetes(), client.KubeSphere(), ippoolProvider)
+	// "globalrolebinding" controller
+	if cmOptions.IsControllerEnabled("globalrolebinding") {
+		globalRoleBindingController := globalrolebinding.NewController(client.Kubernetes(), client.KubeSphere(),
+			kubesphereInformer.Iam().V1alpha2().GlobalRoleBindings(),
+			fedGlobalRoleBindingCache, fedGlobalRoleBindingCacheController,
+			cmOptions.MultiClusterOptions.Enable)
+		addController(mgr, "globalrolebinding", globalRoleBindingController)
 	}

-	controllers := map[string]manager.Runnable{
-		"virtualservice-controller":     vsController,
-		"destinationrule-controller":    drController,
-		"job-controller":                jobController,
-		"storagecapability-controller":  storageCapabilityController,
-		"volumesnapshot-controller":     volumeSnapshotController,
-		"loginrecord-controller":        loginRecordController,
-		"cluster-controller":            clusterController,
-		"nsnp-controller":               nsnpController,
-		"csr-controller":                csrController,
-		"clusterrolebinding-controller": clusterRoleBindingController,
-		"globalrolebinding-controller":  globalRoleBindingController,
-		"ippool-controller":             ippoolController,
-		"groupbinding-controller":       groupBindingController,
-		"group-controller":              groupController,
+	// "groupbinding" controller
+	if cmOptions.IsControllerEnabled("groupbinding") {
+		groupBindingController := groupbinding.NewController(client.Kubernetes(), client.KubeSphere(),
+			kubesphereInformer.Iam().V1alpha2().GroupBindings(),
+			kubesphereInformer.Types().V1beta1().FederatedGroupBindings(),
+			cmOptions.MultiClusterOptions.Enable)
+		addController(mgr, "groupbinding", groupBindingController)
 	}

-	if multiClusterEnabled {
-		controllers["globalrole-controller"] = globalRoleController
-		notificationController, err := notification.NewController(client.Kubernetes(), mgr.GetClient(), mgr.GetCache())
-		if err != nil {
-			return err
+	// "group" controller
+	if cmOptions.IsControllerEnabled("group") {
+		groupController := group.NewController(client.Kubernetes(), client.KubeSphere(),
+			kubesphereInformer.Iam().V1alpha2().Groups(),
+			kubesphereInformer.Types().V1beta1().FederatedGroups(),
+			cmOptions.MultiClusterOptions.Enable)
+		addController(mgr, "group", groupController)
+	}
+
+	// "cluster" controller
+	if cmOptions.IsControllerEnabled("cluster") {
+		if cmOptions.MultiClusterOptions.Enable {
+			clusterController := cluster.NewClusterController(
+				client.Kubernetes(),
+				client.KubeSphere(),
+				client.Config(),
+				kubesphereInformer.Cluster().V1alpha1().Clusters(),
+				kubesphereInformer.Iam().V1alpha2().Users().Lister(),
+				cmOptions.MultiClusterOptions.ClusterControllerResyncPeriod,
+				cmOptions.MultiClusterOptions.HostClusterName,
+			)
+			addController(mgr, "cluster", clusterController)
 		}
-		controllers["notification-controller"] = notificationController
 	}

-	for name, ctrl := range controllers {
-		if ctrl == nil {
-			klog.V(4).Infof("%s is not going to run due to dependent component disabled.", name)
-			continue
-		}
+	// "nsnp" controller
+	if cmOptions.IsControllerEnabled("nsnp") {
+		if cmOptions.NetworkOptions.EnableNetworkPolicy {
+			nsnpProvider, err := provider.NewNsNetworkPolicyProvider(client.Kubernetes(), kubernetesInformer.Networking().V1().NetworkPolicies())
+			if err != nil {
+				klog.Fatalf("Unable to create NSNetworkPolicy controller: %v", err)
+			}

-		if err := mgr.Add(ctrl); err != nil {
-			klog.Error(err, "add controller to manager failed", "name", name)
-			return err
+			nsnpController := nsnetworkpolicy.NewNSNetworkPolicyController(client.Kubernetes(),
+				client.KubeSphere().NetworkV1alpha1(),
+				kubesphereInformer.Network().V1alpha1().NamespaceNetworkPolicies(),
+				kubernetesInformer.Core().V1().Services(),
+				kubernetesInformer.Core().V1().Nodes(),
+				kubesphereInformer.Tenant().V1alpha1().Workspaces(),
+				kubernetesInformer.Core().V1().Namespaces(), nsnpProvider, cmOptions.NetworkOptions.NSNPOptions)
+			addController(mgr, "nsnp", nsnpController)
+		}
+	}
+
+	// "ippool" controller
+	if cmOptions.IsControllerEnabled("ippool") {
+		ippoolProvider := ippoolclient.NewProvider(kubernetesInformer, client.KubeSphere(), client.Kubernetes(),
+			cmOptions.NetworkOptions.IPPoolType, cmOptions.KubernetesOptions)
+		if ippoolProvider != nil {
+			ippoolController := ippool.NewIPPoolController(kubesphereInformer, kubernetesInformer, client.Kubernetes(),
+				client.KubeSphere(), ippoolProvider)
+			addController(mgr, "ippool", ippoolController)
+		}
+	}
+
+	// "notification" controller
+	if cmOptions.IsControllerEnabled("notification") {
+		if cmOptions.MultiClusterOptions.Enable {
+			notificationController, err := notification.NewController(client.Kubernetes(), mgr.GetClient(), mgr.GetCache())
+			if err != nil {
+				klog.Fatalf("Unable to create Notification controller: %v", err)
+			}
+			addController(mgr, "notification", notificationController)
+		}
+	}
+
+	// log all controllers process result
+	for _, name := range allControllers {
+		if cmOptions.IsControllerEnabled(name) {
+			if addSuccessfullyControllers.Has(name) {
+				klog.Infof("%s controller is enabled and added successfully.", name)
+			} else {
+				klog.Infof("%s controller is enabled but is not going to run due to its dependent component being disabled.", name)
+			}
+		} else {
+			klog.Infof("%s controller is disabled by controller selectors.", name)
 		}
 	}

 	return nil
 }
+
+var addSuccessfullyControllers = sets.NewString()
+
+type setupableController interface {
+	SetupWithManager(mgr ctrl.Manager) error
+}
+
+func addControllerWithSetup(mgr manager.Manager, name string, controller setupableController) {
+	if err := controller.SetupWithManager(mgr); err != nil {
+		klog.Fatalf("Unable to create %v controller: %v", name, err)
+	}
+	addSuccessfullyControllers.Insert(name)
+}
+
+func addController(mgr manager.Manager, name string, controller manager.Runnable) {
+	if err := mgr.Add(controller); err != nil {
+		klog.Fatalf("Unable to create %v controller: %v", name, err)
+	}
+	addSuccessfullyControllers.Insert(name)
+}
--- a/cmd/controller-manager/app/options/options.go
+++ b/cmd/controller-manager/app/options/options.go
@@ -18,9 +18,16 @@ package options

 import (
 	"flag"
+	"fmt"
 	"strings"
 	"time"

+	"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus"
+
+	controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication"

 	"k8s.io/apimachinery/pkg/labels"
@@ -52,6 +59,7 @@ type KubeSphereControllerManagerOptions struct {
 	MultiClusterOptions   *multicluster.Options
 	ServiceMeshOptions    *servicemesh.Options
 	GatewayOptions        *gateway.Options
+	MonitoringOptions     *prometheus.Options
 	LeaderElect           bool
 	LeaderElection        *leaderelection.LeaderElectionConfig
 	WebhookCertDir        string
@@ -64,6 +72,19 @@ type KubeSphereControllerManagerOptions struct {
 	//      "kubesphere.io/creator=" means reconcile applications with this label key
 	//      "!kubesphere.io/creator" means exclude applications with this key
 	ApplicationSelector string
+
+	// ControllerGates is the list of controller gates to enable or disable controller.
+	// '*' means "all enabled by default controllers"
+	// 'foo' means "enable 'foo'"
+	// '-foo' means "disable 'foo'"
+	// first item for a particular name wins.
+	//     e.g. '-foo,foo' means "disable foo", 'foo,-foo' means "enable foo"
+	// * has the lowest priority.
+	//     e.g. *,-foo, means "disable 'foo'"
+	ControllerGates []string
+
+	// Enable gops or not.
+	GOPSEnabled bool
 }

 func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions {
@@ -86,12 +107,13 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions
 		LeaderElect:         false,
 		WebhookCertDir:      "",
 		ApplicationSelector: "",
+		ControllerGates:     []string{"*"},
 	}

 	return s
 }

-func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
+func (s *KubeSphereControllerManagerOptions) Flags(allControllerNameSelectors []string) cliflag.NamedFlagSets {
 	fss := cliflag.NamedFlagSets{}

 	s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
@@ -120,6 +142,13 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
 	gfs.StringVar(&s.ApplicationSelector, "application-selector", s.ApplicationSelector, ""+
 		"Only reconcile application(sigs.k8s.io/application) objects match given selector, this could avoid conflicts with "+
 		"other projects built on top of sig-application. Default behavior is to reconcile all of application objects.")
+	gfs.StringSliceVar(&s.ControllerGates, "controllers", []string{"*"}, fmt.Sprintf(""+
+		"A list of controllers to enable. '*' enables all on-by-default controllers, 'foo' enables the controller "+
+		"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s",
+		strings.Join(allControllerNameSelectors, ", ")))
+
+	gfs.BoolVar(&s.GOPSEnabled, "gops", s.GOPSEnabled, "Whether to enable gops or not.  When enabled this option, "+
+		"controller-manager will listen on a random port on 127.0.0.1, then you can use the gops tool to list and diagnose the controller-manager currently running.")

 	kfs := fss.FlagSet("klog")
 	local := flag.NewFlagSet("klog", flag.ExitOnError)
@@ -132,26 +161,58 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
 	return fss
 }

-func (s *KubeSphereControllerManagerOptions) Validate() []error {
+// Validate Options and Genetic Options
+func (o *KubeSphereControllerManagerOptions) Validate(allControllerNameSelectors []string) []error {
 	var errs []error
-	errs = append(errs, s.DevopsOptions.Validate()...)
-	errs = append(errs, s.KubernetesOptions.Validate()...)
-	errs = append(errs, s.S3Options.Validate()...)
-	errs = append(errs, s.OpenPitrixOptions.Validate()...)
-	errs = append(errs, s.NetworkOptions.Validate()...)
-	errs = append(errs, s.LdapOptions.Validate()...)
-	errs = append(errs, s.MultiClusterOptions.Validate()...)
+	errs = append(errs, o.DevopsOptions.Validate()...)
+	errs = append(errs, o.KubernetesOptions.Validate()...)
+	errs = append(errs, o.S3Options.Validate()...)
+	errs = append(errs, o.OpenPitrixOptions.Validate()...)
+	errs = append(errs, o.NetworkOptions.Validate()...)
+	errs = append(errs, o.LdapOptions.Validate()...)
+	errs = append(errs, o.MultiClusterOptions.Validate()...)

-	if len(s.ApplicationSelector) != 0 {
-		_, err := labels.Parse(s.ApplicationSelector)
+	// genetic option: application-selector
+	if len(o.ApplicationSelector) != 0 {
+		_, err := labels.Parse(o.ApplicationSelector)
 		if err != nil {
 			errs = append(errs, err)
 		}
 	}

+	// genetic option: controllers, check all selectors are valid
+	allControllersNameSet := sets.NewString(allControllerNameSelectors...)
+	for _, selector := range o.ControllerGates {
+		if selector == "*" {
+			continue
+		}
+		selector = strings.TrimPrefix(selector, "-")
+		if !allControllersNameSet.Has(selector) {
+			errs = append(errs, fmt.Errorf("%q is not in the list of known controllers", selector))
+		}
+	}
+
 	return errs
 }

+// IsControllerEnabled check if a specified controller enabled or not.
+func (o *KubeSphereControllerManagerOptions) IsControllerEnabled(name string) bool {
+	hasStar := false
+	for _, ctrl := range o.ControllerGates {
+		if ctrl == name {
+			return true
+		}
+		if ctrl == "-"+name {
+			return false
+		}
+		if ctrl == "*" {
+			hasStar = true
+		}
+	}
+
+	return hasStar
+}
+
 func (s *KubeSphereControllerManagerOptions) bindLeaderElectionFlags(l *leaderelection.LeaderElectionConfig, fs *pflag.FlagSet) {
 	fs.DurationVar(&l.LeaseDuration, "leader-elect-lease-duration", l.LeaseDuration, ""+
 		"The duration that non-leader candidates will wait after observing a leadership "+
@@ -167,3 +228,18 @@ func (s *KubeSphereControllerManagerOptions) bindLeaderElectionFlags(l *leaderel
 		"The duration the clients should wait between attempting acquisition and renewal "+
 		"of a leadership. This is only applicable if leader election is enabled.")
 }
+
+// MergeConfig merge new config without validation
+// When misconfigured, the app should just crash directly
+func (s *KubeSphereControllerManagerOptions) MergeConfig(cfg *controllerconfig.Config) {
+	s.KubernetesOptions = cfg.KubernetesOptions
+	s.DevopsOptions = cfg.DevopsOptions
+	s.S3Options = cfg.S3Options
+	s.AuthenticationOptions = cfg.AuthenticationOptions
+	s.LdapOptions = cfg.LdapOptions
+	s.OpenPitrixOptions = cfg.OpenPitrixOptions
+	s.NetworkOptions = cfg.NetworkOptions
+	s.MultiClusterOptions = cfg.MultiClusterOptions
+	s.ServiceMeshOptions = cfg.ServiceMeshOptions
+	s.GatewayOptions = cfg.GatewayOptions
+}
--- a/cmd/controller-manager/app/options/options_test.go
+++ b/cmd/controller-manager/app/options/options_test.go
@@ -0,0 +1,81 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package options
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// ref: https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/controller-manager/app/helper_test.go
+func TestIsControllerEnabled(t *testing.T) {
+	testcases := []struct {
+		name            string
+		controllerName  string
+		controllerFlags []string
+		expected        bool
+	}{
+		{
+			name:            "on by name",
+			controllerName:  "bravo",
+			controllerFlags: []string{"alpha", "bravo", "-charlie"},
+			expected:        true,
+		},
+		{
+			name:            "off by name",
+			controllerName:  "charlie",
+			controllerFlags: []string{"alpha", "bravo", "-charlie"},
+			expected:        false,
+		},
+		{
+			name:            "on by default",
+			controllerName:  "alpha",
+			controllerFlags: []string{"*"},
+			expected:        true,
+		},
+		{
+			name:            "on by star, not off by name",
+			controllerName:  "alpha",
+			controllerFlags: []string{"*", "-charlie"},
+			expected:        true,
+		},
+		{
+			name:            "off by name with star",
+			controllerName:  "charlie",
+			controllerFlags: []string{"*", "-charlie"},
+			expected:        false,
+		},
+		{
+			name:            "off then on",
+			controllerName:  "alpha",
+			controllerFlags: []string{"-alpha", "alpha"},
+			expected:        false,
+		},
+		{
+			name:            "on then off",
+			controllerName:  "alpha",
+			controllerFlags: []string{"alpha", "-alpha"},
+			expected:        true,
+		},
+	}
+
+	for _, tc := range testcases {
+		option := NewKubeSphereControllerManagerOptions()
+		option.ControllerGates = tc.controllerFlags
+		actual := option.IsControllerEnabled(tc.controllerName)
+		assert.Equal(t, tc.expected, actual, "%v: expected %v, got %v", tc.name, tc.expected, actual)
+	}
+}
--- a/cmd/controller-manager/app/server.go
+++ b/cmd/controller-manager/app/server.go
@@ -21,11 +21,9 @@ import (
 	"fmt"
 	"os"

-	"kubesphere.io/kubesphere/pkg/models/kubeconfig"
-
+	"github.com/google/gops/agent"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/labels"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog"
@@ -38,26 +36,11 @@ import (
 	"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
 	"kubesphere.io/kubesphere/pkg/apis"
 	controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
-	"kubesphere.io/kubesphere/pkg/controller/application"
-	"kubesphere.io/kubesphere/pkg/controller/helm"
-	"kubesphere.io/kubesphere/pkg/controller/namespace"
 	"kubesphere.io/kubesphere/pkg/controller/network/webhooks"
-	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmapplication"
-	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmcategory"
-	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrelease"
-	"kubesphere.io/kubesphere/pkg/controller/openpitrix/helmrepo"
 	"kubesphere.io/kubesphere/pkg/controller/quota"
-	"kubesphere.io/kubesphere/pkg/controller/serviceaccount"
 	"kubesphere.io/kubesphere/pkg/controller/user"
-	"kubesphere.io/kubesphere/pkg/controller/workspace"
-	"kubesphere.io/kubesphere/pkg/controller/workspacerole"
-	"kubesphere.io/kubesphere/pkg/controller/workspacerolebinding"
-	"kubesphere.io/kubesphere/pkg/controller/workspacetemplate"
 	"kubesphere.io/kubesphere/pkg/informers"
-	"kubesphere.io/kubesphere/pkg/simple/client/devops"
-	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
-	ldapclient "kubesphere.io/kubesphere/pkg/simple/client/ldap"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/utils/metrics"
 	"kubesphere.io/kubesphere/pkg/utils/term"
@@ -80,6 +63,7 @@ func NewControllerManagerCommand() *cobra.Command {
 			MultiClusterOptions:   conf.MultiClusterOptions,
 			ServiceMeshOptions:    conf.ServiceMeshOptions,
 			GatewayOptions:        conf.GatewayOptions,
+			MonitoringOptions:     conf.MonitoringOptions,
 			LeaderElection:        s.LeaderElection,
 			LeaderElect:           s.LeaderElect,
 			WebhookCertDir:        s.WebhookCertDir,
@@ -90,14 +74,22 @@ func NewControllerManagerCommand() *cobra.Command {

 	cmd := &cobra.Command{
 		Use:  "controller-manager",
-		Long: `KubeSphere controller manager is a daemon that`,
+		Long: `KubeSphere controller manager is a daemon that embeds the control loops shipped with KubeSphere.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			if errs := s.Validate(); len(errs) != 0 {
+			if errs := s.Validate(allControllers); len(errs) != 0 {
 				klog.Error(utilerrors.NewAggregate(errs))
 				os.Exit(1)
 			}

-			if err = run(s, signals.SetupSignalHandler()); err != nil {
+			if s.GOPSEnabled {
+				// Add agent to report additional information such as the current stack trace, Go version, memory stats, etc.
+				// Bind to a random port on address 127.0.0.1
+				if err := agent.Listen(agent.Options{}); err != nil {
+					klog.Fatal(err)
+				}
+			}
+
+			if err = Run(s, controllerconfig.WatchConfigChange(), signals.SetupSignalHandler()); err != nil {
 				klog.Error(err)
 				os.Exit(1)
 			}
@@ -106,7 +98,7 @@ func NewControllerManagerCommand() *cobra.Command {
 	}

 	fs := cmd.Flags()
-	namedFlagSets := s.Flags()
+	namedFlagSets := s.Flags(allControllers)

 	for _, f := range namedFlagSets.FlagSets {
 		fs.AddFlagSet(f)
@@ -132,6 +124,40 @@ func NewControllerManagerCommand() *cobra.Command {
 	return cmd
 }

+func Run(s *options.KubeSphereControllerManagerOptions, configCh <-chan controllerconfig.Config, ctx context.Context) error {
+	ictx, cancelFunc := context.WithCancel(context.TODO())
+	errCh := make(chan error)
+	defer close(errCh)
+	go func() {
+		if err := run(s, ictx); err != nil {
+			errCh <- err
+		}
+	}()
+
+	// The ctx (signals.SetupSignalHandler()) is to control the entire program life cycle,
+	// The ictx(internal context)  is created here to control the life cycle of the controller-manager(all controllers, sharedInformer, webhook etc.)
+	// when config changed, stop server and renew context, start new server
+	for {
+		select {
+		case <-ctx.Done():
+			cancelFunc()
+			return nil
+		case cfg := <-configCh:
+			cancelFunc()
+			s.MergeConfig(&cfg)
+			ictx, cancelFunc = context.WithCancel(context.TODO())
+			go func() {
+				if err := run(s, ictx); err != nil {
+					errCh <- err
+				}
+			}()
+		case err := <-errCh:
+			cancelFunc()
+			return err
+		}
+	}
+}
+
 func run(s *options.KubeSphereControllerManagerOptions, ctx context.Context) error {

 	kubernetesClient, err := k8s.NewKubernetesClient(s.KubernetesOptions)
@@ -140,32 +166,8 @@ func run(s *options.KubeSphereControllerManagerOptions, ctx context.Context) err
 		return err
 	}

-	var devopsClient devops.Interface
-	if s.DevopsOptions != nil && len(s.DevopsOptions.Host) != 0 {
-		devopsClient, err = jenkins.NewDevopsClient(s.DevopsOptions)
-		if err != nil {
-			return fmt.Errorf("failed to connect jenkins, please check jenkins status, error: %v", err)
-		}
-	}
-
-	var ldapClient ldapclient.Interface
-	// when there is no ldapOption, we set ldapClient as nil, which means we don't need to sync user info into ldap.
-	if s.LdapOptions != nil && len(s.LdapOptions.Host) != 0 {
-		if s.LdapOptions.Host == ldapclient.FAKE_HOST { // for debug only
-			ldapClient = ldapclient.NewSimpleLdap()
-		} else {
-			ldapClient, err = ldapclient.NewLdapClient(s.LdapOptions, ctx.Done())
-			if err != nil {
-				return fmt.Errorf("failed to connect to ldap service, please check ldap status, error: %v", err)
-			}
-		}
-	} else {
-		klog.Warning("ks-controller-manager starts without ldap provided, it will not sync user into ldap")
-	}
-
-	var s3Client s3.Interface
 	if s.S3Options != nil && len(s.S3Options.Endpoint) != 0 {
-		s3Client, err = s3.NewS3Client(s.S3Options)
+		_, err = s3.NewS3Client(s.S3Options)
 		if err != nil {
 			return fmt.Errorf("failed to connect to s3, please check s3 service status, error: %v", err)
 		}
@@ -212,131 +214,13 @@ func run(s *options.KubeSphereControllerManagerOptions, ctx context.Context) err
 	// register common meta types into schemas.
 	metav1.AddToGroupVersion(mgr.GetScheme(), metav1.SchemeGroupVersion)

-	kubeconfigClient := kubeconfig.NewOperator(kubernetesClient.Kubernetes(),
-		informerFactory.KubernetesSharedInformerFactory().Core().V1().ConfigMaps().Lister(),
-		kubernetesClient.Config())
-	userController := user.Reconciler{
-		MultiClusterEnabled:     s.MultiClusterOptions.Enable,
-		MaxConcurrentReconciles: 4,
-		LdapClient:              ldapClient,
-		DevopsClient:            devopsClient,
-		KubeconfigClient:        kubeconfigClient,
-		AuthenticationOptions:   s.AuthenticationOptions,
-	}
-
-	if err = userController.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create user controller: %v", err)
-	}
-
-	workspaceTemplateReconciler := &workspacetemplate.Reconciler{MultiClusterEnabled: s.MultiClusterOptions.Enable}
-	if err = workspaceTemplateReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create workspace template controller: %v", err)
-	}
-
-	workspaceReconciler := &workspace.Reconciler{}
-	if err = workspaceReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create workspace controller: %v", err)
-	}
-
-	workspaceRoleReconciler := &workspacerole.Reconciler{MultiClusterEnabled: s.MultiClusterOptions.Enable}
-	if err = workspaceRoleReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create workspace role controller: %v", err)
-	}
-
-	workspaceRoleBindingReconciler := &workspacerolebinding.Reconciler{MultiClusterEnabled: s.MultiClusterOptions.Enable}
-	if err = workspaceRoleBindingReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create workspace role binding controller: %v", err)
-	}
-
-	namespaceReconciler := &namespace.Reconciler{}
-	if err = namespaceReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create namespace controller: %v", err)
-	}
-
-	err = helmrepo.Add(mgr)
-	if err != nil {
-		klog.Fatal("Unable to create helm repo controller")
-	}
-
-	err = helmcategory.Add(mgr)
-	if err != nil {
-		klog.Fatal("Unable to create helm category controller")
-	}
-
-	var opS3Client s3.Interface
-	if !s.OpenPitrixOptions.AppStoreConfIsEmpty() {
-		opS3Client, err = s3.NewS3Client(s.OpenPitrixOptions.S3Options)
-		if err != nil {
-			klog.Fatalf("failed to connect to s3, please check openpitrix s3 service status, error: %v", err)
-		}
-		err = (&helmapplication.ReconcileHelmApplication{}).SetupWithManager(mgr)
-		if err != nil {
-			klog.Fatalf("Unable to create helm application controller, error: %s", err)
-		}
-
-		err = (&helmapplication.ReconcileHelmApplicationVersion{}).SetupWithManager(mgr)
-		if err != nil {
-			klog.Fatalf("Unable to create helm application version controller, error: %s ", err)
-		}
-	}
-
-	err = (&helmrelease.ReconcileHelmRelease{
-		// nil interface is valid value.
-		StorageClient:      opS3Client,
-		KsFactory:          informerFactory.KubeSphereSharedInformerFactory(),
-		MultiClusterEnable: s.MultiClusterOptions.Enable,
-		WaitTime:           s.OpenPitrixOptions.ReleaseControllerOptions.WaitTime,
-		MaxConcurrent:      s.OpenPitrixOptions.ReleaseControllerOptions.MaxConcurrent,
-		StopChan:           ctx.Done(),
-	}).SetupWithManager(mgr)
-
-	if err != nil {
-		klog.Fatalf("Unable to create helm release controller, error: %s", err)
-	}
-
-	selector, _ := labels.Parse(s.ApplicationSelector)
-	applicationReconciler := &application.ApplicationReconciler{
-		Scheme:              mgr.GetScheme(),
-		Client:              mgr.GetClient(),
-		Mapper:              mgr.GetRESTMapper(),
-		ApplicationSelector: selector,
-	}
-	if err = applicationReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create application controller: %v", err)
-	}
-
-	saReconciler := &serviceaccount.Reconciler{}
-	if err = saReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create ServiceAccount controller: %v", err)
-	}
-
-	resourceQuotaReconciler := quota.Reconciler{}
-	if err := resourceQuotaReconciler.SetupWithManager(mgr, quota.DefaultMaxConcurrentReconciles, quota.DefaultResyncPeriod, informerFactory.KubernetesSharedInformerFactory()); err != nil {
-		klog.Fatalf("Unable to create ResourceQuota controller: %v", err)
-	}
-
-	helmReconciler := helm.Reconciler{}
-	if !s.GatewayOptions.IsEmpty() {
-		helmReconciler.WatchFiles = append(helmReconciler.WatchFiles, s.GatewayOptions.WatchesPath)
-	}
-	if err := helmReconciler.SetupWithManager(mgr); err != nil {
-		klog.Fatalf("Unable to create helm controller: %v", err)
-	}
-
 	// TODO(jeff): refactor config with CRD
-	servicemeshEnabled := s.ServiceMeshOptions != nil && len(s.ServiceMeshOptions.IstioPilotHost) != 0
-	if err = addControllers(mgr,
+	// install all controllers
+	if err = addAllControllers(mgr,
 		kubernetesClient,
 		informerFactory,
-		devopsClient,
-		s3Client,
-		ldapClient,
-		s.KubernetesOptions,
-		s.AuthenticationOptions,
-		s.MultiClusterOptions,
-		s.NetworkOptions,
-		servicemeshEnabled,
-		s.AuthenticationOptions.KubectlImage, ctx.Done()); err != nil {
+		s,
+		ctx.Done()); err != nil {
 		klog.Fatalf("unable to register controllers to the manager: %v", err)
 	}

@@ -352,6 +236,7 @@ func run(s *options.KubeSphereControllerManagerOptions, ctx context.Context) err
 	hookServer.Register("/validate-email-iam-kubesphere-io-v1alpha2", &webhook.Admission{Handler: &user.EmailValidator{Client: mgr.GetClient()}})
 	hookServer.Register("/validate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.ValidatingHandler{C: mgr.GetClient()}})
 	hookServer.Register("/mutate-network-kubesphere-io-v1alpha1", &webhook.Admission{Handler: &webhooks.MutatingHandler{C: mgr.GetClient()}})
+	hookServer.Register("/persistentvolumeclaims", &webhook.Admission{Handler: &webhooks.AccessorHandler{C: mgr.GetClient()}})

 	resourceQuotaAdmission, err := quota.NewResourceQuotaAdmission(mgr.GetClient(), mgr.GetScheme())
 	if err != nil {
--- a/cmd/ks-apiserver/app/options/options.go
+++ b/cmd/ks-apiserver/app/options/options.go
@@ -21,6 +21,9 @@ import (
 	"flag"
 	"fmt"

+	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
+	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
+
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"

 	"k8s.io/client-go/kubernetes/scheme"
@@ -59,6 +62,9 @@ type ServerRunOptions struct {

 	//
 	DebugMode bool
+
+	// Enable gops or not.
+	GOPSEnabled bool
 }

 func NewServerRunOptions() *ServerRunOptions {
@@ -73,6 +79,8 @@ func NewServerRunOptions() *ServerRunOptions {
 func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	fs := fss.FlagSet("generic")
 	fs.BoolVar(&s.DebugMode, "debug", false, "Don't enable this if you don't know what it means.")
+	fs.BoolVar(&s.GOPSEnabled, "gops", false, "Whether to enable gops or not. When enabled this option, "+
+		"ks-apiserver will listen on a random port on 127.0.0.1, then you can use the gops tool to list and diagnose the ks-apiserver currently running.")
 	s.GenericServerRunOptions.AddFlags(fs, s.GenericServerRunOptions)
 	s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"), s.KubernetesOptions)
 	s.AuthenticationOptions.AddFlags(fss.FlagSet("authentication"), s.AuthenticationOptions)
@@ -209,6 +217,13 @@ func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIS
 		apiServer.AlertingClient = alertingClient
 	}

+	if s.Config.MultiClusterOptions.Enable {
+		cc := clusterclient.NewClusterClient(informerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters())
+		apiServer.ClusterClient = cc
+	}
+
+	apiServer.OpenpitrixClient = openpitrixv1.NewOpenpitrixClient(informerFactory, apiServer.KubernetesClient.KubeSphere(), s.OpenPitrixOptions, apiServer.ClusterClient, stopCh)
+
 	server := &http.Server{
 		Addr: fmt.Sprintf(":%d", s.GenericServerRunOptions.InsecurePort),
 	}
--- a/cmd/ks-apiserver/app/server.go
+++ b/cmd/ks-apiserver/app/server.go
@@ -19,7 +19,9 @@ package app
 import (
 	"context"
 	"fmt"
+	"net/http"

+	"github.com/google/gops/agent"
 	"github.com/spf13/cobra"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -57,7 +59,15 @@ cluster's shared state through which all other components interact.`,
 				return utilerrors.NewAggregate(errs)
 			}

-			return Run(s, signals.SetupSignalHandler())
+			if s.GOPSEnabled {
+				// Add agent to report additional information such as the current stack trace, Go version, memory stats, etc.
+				// Bind to a random port on address 127.0.0.1.
+				if err := agent.Listen(agent.Options{}); err != nil {
+					klog.Fatal(err)
+				}
+			}
+
+			return Run(s, apiserverconfig.WatchConfigChange(), signals.SetupSignalHandler())
 		},
 		SilenceUsage: true,
 	}
@@ -88,8 +98,41 @@ cluster's shared state through which all other components interact.`,
 	return cmd
 }

-func Run(s *options.ServerRunOptions, ctx context.Context) error {
+func Run(s *options.ServerRunOptions, configCh <-chan apiserverconfig.Config, ctx context.Context) error {
+	ictx, cancelFunc := context.WithCancel(context.TODO())
+	errCh := make(chan error)
+	defer close(errCh)
+	go func() {
+		if err := run(s, ictx); err != nil {
+			errCh <- err
+		}
+	}()

+	// The ctx (signals.SetupSignalHandler()) is to control the entire program life cycle,
+	// The ictx(internal context)  is created here to control the life cycle of the ks-apiserver(http server, sharedInformer etc.)
+	// when config change, stop server and renew context, start new server
+	for {
+		select {
+		case <-ctx.Done():
+			cancelFunc()
+			return nil
+		case cfg := <-configCh:
+			cancelFunc()
+			s.Config = &cfg
+			ictx, cancelFunc = context.WithCancel(context.TODO())
+			go func() {
+				if err := run(s, ictx); err != nil {
+					errCh <- err
+				}
+			}()
+		case err := <-errCh:
+			cancelFunc()
+			return err
+		}
+	}
+}
+
+func run(s *options.ServerRunOptions, ctx context.Context) error {
 	apiserver, err := s.NewAPIServer(ctx.Done())
 	if err != nil {
 		return err
@@ -100,5 +143,9 @@ func Run(s *options.ServerRunOptions, ctx context.Context) error {
 		return err
 	}

-	return apiserver.Run(ctx)
+	err = apiserver.Run(ctx)
+	if err == http.ErrServerClosed {
+		return nil
+	}
+	return err
 }
--- a/config/crds/application.kubesphere.io_helmapplications.yaml
+++ b/config/crds/application.kubesphere.io_helmapplications.yaml
@@ -37,10 +37,14 @@ spec:
        description: HelmApplication is the Schema for the helmapplications API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -73,10 +77,13 @@ spec:
            description: HelmApplicationStatus defines the observed state of HelmApplication
            properties:
              latestVersion:
-                description: If this application belong to appStore, latestVersion is the the latest version of the active application version. otherwise latestVersion is the latest version of all application version
+                description: If this application belong to appStore, latestVersion
+                  is the the latest version of the active application version. otherwise
+                  latestVersion is the latest version of all application version
                type: string
              state:
-                description: 'the state of the helm application: draft, submitted, passed, rejected, suspended, active'
+                description: 'the state of the helm application: draft, submitted,
+                  passed, rejected, suspended, active'
                type: string
              statusTime:
                format: date-time
--- a/config/crds/application.kubesphere.io_helmapplicationversions.yaml
+++ b/config/crds/application.kubesphere.io_helmapplicationversions.yaml
@@ -31,13 +31,18 @@ spec:
    name: v1alpha1
    schema:
      openAPIV3Schema:
-        description: HelmApplicationVersion is the Schema for the helmapplicationversions API
+        description: HelmApplicationVersion is the Schema for the helmapplicationversions
+          API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -47,13 +52,15 @@ spec:
              annotations:
                additionalProperties:
                  type: string
-                description: Annotations are additional mappings uninterpreted by Helm, made available for inspection by other applications.
+                description: Annotations are additional mappings uninterpreted by
+                  Helm, made available for inspection by other applications.
                type: object
              apiVersion:
                description: The API Version of this chart.
                type: string
              appVersion:
-                description: The version of the application enclosed inside of this chart.
+                description: The version of the application enclosed inside of this
+                  chart.
                type: string
              condition:
                description: The condition to check to enable chart
@@ -72,30 +79,39 @@ spec:
              dependencies:
                description: Dependencies are a list of dependencies for a chart.
                items:
-                  description: Dependency describes a chart upon which another chart depends. Dependencies can be used to express developer intent, or to capture the state of a chart.
+                  description: Dependency describes a chart upon which another chart
+                    depends. Dependencies can be used to express developer intent,
+                    or to capture the state of a chart.
                  properties:
                    alias:
                      description: Alias usable alias to be used for the chart
                      type: string
                    condition:
-                      description: A yaml path that resolves to a boolean, used for enabling/disabling charts (e.g. subchart1.enabled )
+                      description: A yaml path that resolves to a boolean, used for
+                        enabling/disabling charts (e.g. subchart1.enabled )
                      type: string
                    enabled:
                      description: Enabled bool determines if chart should be loaded
                      type: boolean
                    name:
-                      description: Name is the name of the dependency. This must mach the name in the dependency's Chart.yaml.
+                      description: Name is the name of the dependency. This must mach
+                        the name in the dependency's Chart.yaml.
                      type: string
                    repository:
-                      description: The URL to the repository. Appending `index.yaml` to this string should result in a URL that can be used to fetch the repository index.
+                      description: The URL to the repository. Appending `index.yaml`
+                        to this string should result in a URL that can be used to
+                        fetch the repository index.
                      type: string
                    tags:
-                      description: Tags can be used to group charts for enabling/disabling together
+                      description: Tags can be used to group charts for enabling/disabling
+                        together
                      items:
                        type: string
                      type: array
                    version:
-                      description: Version is the version (range) of this chart. A lock file will always produce a single version, while a dependency may contain a semantic version range.
+                      description: Version is the version (range) of this chart. A
+                        lock file will always produce a single version, while a dependency
+                        may contain a semantic version range.
                      type: string
                  required:
                  - name
@@ -112,7 +128,8 @@ spec:
                description: chart digest
                type: string
              home:
-                description: The URL to a relevant project page, git repo, or contact person
+                description: The URL to a relevant project page, git repo, or contact
+                  person
                type: string
              icon:
                description: The URL to an icon file.
@@ -123,21 +140,25 @@ spec:
                  type: string
                type: array
              kubeVersion:
-                description: KubeVersion is a SemVer constraint specifying the version of Kubernetes required.
+                description: KubeVersion is a SemVer constraint specifying the version
+                  of Kubernetes required.
                type: string
              maintainers:
-                description: A list of name and URL/email address combinations for the maintainer(s)
+                description: A list of name and URL/email address combinations for
+                  the maintainer(s)
                items:
                  description: Maintainer describes a Chart maintainer.
                  properties:
                    email:
-                      description: Email is an optional email address to contact the named maintainer
+                      description: Email is an optional email address to contact the
+                        named maintainer
                      type: string
                    name:
                      description: Name is a user name or organization name
                      type: string
                    url:
-                      description: URL is an optional URL to an address for the named maintainer
+                      description: URL is an optional URL to an address for the named
+                        maintainer
                      type: string
                  type: object
                type: array
@@ -165,7 +186,8 @@ spec:
                type: string
            type: object
          status:
-            description: HelmApplicationVersionStatus defines the observed state of HelmApplicationVersion
+            description: HelmApplicationVersionStatus defines the observed state of
+              HelmApplicationVersion
            properties:
              audit:
                items:
@@ -179,7 +201,8 @@ spec:
                    operatorType:
                      type: string
                    state:
-                      description: 'audit state: submitted, passed, draft, active, rejected, suspended'
+                      description: 'audit state: submitted, passed, draft, active,
+                        rejected, suspended'
                      type: string
                    time:
                      description: audit time
--- a/config/crds/application.kubesphere.io_helmcategories.yaml
+++ b/config/crds/application.kubesphere.io_helmcategories.yaml
@@ -34,10 +34,14 @@ spec:
        description: HelmCategory is the Schema for the helmcategories API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/application.kubesphere.io_helmreleases.yaml
+++ b/config/crds/application.kubesphere.io_helmreleases.yaml
@@ -43,10 +43,14 @@ spec:
        description: HelmRelease is the Schema for the helmreleases API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -66,7 +70,8 @@ spec:
                description: The name of the chart which will be installed.
                type: string
              chartVersion:
-                description: Specify the exact chart version to install. If this is not specified, the latest version is installed
+                description: Specify the exact chart version to install. If this is
+                  not specified, the latest version is installed
                type: string
              description:
                description: Message got from frontend
@@ -82,7 +87,9 @@ spec:
                format: byte
                type: string
              version:
-                description: expected release version, when this version is not equal status.version, the release need upgrade this filed should be modified when any filed of the spec modified.
+                description: expected release version, when this version is not equal
+                  status.version, the release need upgrade this filed should be modified
+                  when any filed of the spec modified.
                type: integer
            required:
            - chartName
@@ -94,7 +101,8 @@ spec:
            description: HelmReleaseStatus defines the observed state of HelmRelease
            properties:
              deployStatus:
-                description: deploy status list of history, which will store at most 10 state
+                description: deploy status list of history, which will store at most
+                  10 state
                items:
                  properties:
                    deployTime:
@@ -102,7 +110,8 @@ spec:
                      format: date-time
                      type: string
                    message:
-                      description: A human readable message indicating details about why the release is in this state.
+                      description: A human readable message indicating details about
+                        why the release is in this state.
                      type: string
                    state:
                      description: current state of the release
@@ -121,7 +130,8 @@ spec:
                format: date-time
                type: string
              message:
-                description: A human readable message indicating details about why the release is in this state.
+                description: A human readable message indicating details about why
+                  the release is in this state.
                type: string
              state:
                description: current state
--- a/config/crds/application.kubesphere.io_helmrepos.yaml
+++ b/config/crds/application.kubesphere.io_helmrepos.yaml
@@ -40,10 +40,14 @@ spec:
        description: HelmRepo is the Schema for the helmrepoes API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -56,13 +60,16 @@ spec:
                  accessKeyID:
                    type: string
                  caFile:
-                    description: verify certificates of HTTPS-enabled servers using this CA bundle
+                    description: verify certificates of HTTPS-enabled servers using
+                      this CA bundle
                    type: string
                  certFile:
-                    description: identify HTTPS client using this SSL certificate file
+                    description: identify HTTPS client using this SSL certificate
+                      file
                    type: string
                  insecureSkipTLSVerify:
-                    description: skip tls certificate checks for the repository, default is ture
+                    description: skip tls certificate checks for the repository, default
+                      is ture
                    type: boolean
                  keyFile:
                    description: identify HTTPS client using this SSL key file
@@ -83,13 +90,16 @@ spec:
                description: name of the repo
                type: string
              syncPeriod:
-                description: sync period in seconds, no sync when SyncPeriod=0, the minimum SyncPeriod is 180s
+                description: sync period in seconds, no sync when SyncPeriod=0, the
+                  minimum SyncPeriod is 180s
                type: integer
              url:
                description: helm repo url
                type: string
              version:
-                description: expected repo version, when this version is not equal status.version, the repo need upgrade this filed should be modified when any filed of the spec modified.
+                description: expected repo version, when this version is not equal
+                  status.version, the repo need upgrade this filed should be modified
+                  when any filed of the spec modified.
                type: integer
            required:
            - name
@@ -109,14 +119,17 @@ spec:
                description: current state of the repo, successful, failed or syncing
                type: string
              syncState:
-                description: sync state list of history, which will store at most 10 state
+                description: sync state list of history, which will store at most
+                  10 state
                items:
                  properties:
                    message:
-                      description: A human readable message indicating details about why the repo is in this state.
+                      description: A human readable message indicating details about
+                        why the repo is in this state.
                      type: string
                    state:
-                      description: 'last sync state, valid state are: "failed", "success", and ""'
+                      description: 'last sync state, valid state are: "failed", "success",
+                        and ""'
                      type: string
                    syncTime:
                      format: date-time
@@ -126,7 +139,8 @@ spec:
                  type: object
                type: array
              version:
-                description: if status.version!=spec.Version, we need sync the repo now
+                description: if status.version!=spec.Version, we need sync the repo
+                  now
                type: integer
            type: object
        type: object
--- a/config/crds/cluster.kubesphere.io_clusters.yaml
+++ b/config/crds/cluster.kubesphere.io_clusters.yaml
@@ -35,10 +35,14 @@ spec:
        description: Cluster is the schema for the clusters API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -47,32 +51,58 @@ spec:
              connection:
                description: Connection holds info to connect to the member cluster
                properties:
+                  externalKubernetesAPIEndpoint:
+                    description: External Kubernetes API Server endpoint Will be populated
+                      by ks-apiserver if connection type is proxy and ExternalKubeAPIEnabled
+                      is true.
+                    type: string
                  kubeconfig:
-                    description: KubeConfig content used to connect to cluster api server Should provide this field explicitly if connection type is direct. Will be populated by ks-proxy if connection type is proxy.
+                    description: KubeConfig content used to connect to cluster api
+                      server Should provide this field explicitly if connection type
+                      is direct. Will be populated by ks-proxy if connection type
+                      is proxy.
                    format: byte
                    type: string
                  kubernetesAPIEndpoint:
-                    description: 'Kubernetes API Server endpoint. Example: https://10.10.0.1:6443 Should provide this field explicitly if connection type is direct. Will be populated by ks-apiserver if connection type is proxy.'
+                    description: 'Kubernetes API Server endpoint. Example: https://10.10.0.1:6443
+                      Should provide this field explicitly if connection type is direct.
+                      Will be populated by ks-apiserver if connection type is proxy.'
                    type: string
                  kubernetesAPIServerPort:
-                    description: KubeAPIServerPort is the port which listens for forwarding kube-apiserver traffic Only applicable when connection type is proxy.
+                    description: KubeAPIServerPort is the port which listens for forwarding
+                      kube-apiserver traffic Only applicable when connection type
+                      is proxy.
                    type: integer
                  kubesphereAPIEndpoint:
-                    description: 'KubeSphere API Server endpoint. Example: http://10.10.0.11:8080 Should provide this field explicitly if connection type is direct. Will be populated by ks-apiserver if connection type is proxy.'
+                    description: 'KubeSphere API Server endpoint. Example: http://10.10.0.11:8080
+                      Should provide this field explicitly if connection type is direct.
+                      Will be populated by ks-apiserver if connection type is proxy.'
                    type: string
                  kubesphereAPIServerPort:
-                    description: KubeSphereAPIServerPort is the port which listens for forwarding kubesphere apigateway traffic Only applicable when connection type is proxy.
+                    description: KubeSphereAPIServerPort is the port which listens
+                      for forwarding kubesphere apigateway traffic Only applicable
+                      when connection type is proxy.
                    type: integer
                  token:
-                    description: Token used by agents of member cluster to connect to host cluster proxy. This field is populated by apiserver only if connection type is proxy.
+                    description: Token used by agents of member cluster to connect
+                      to host cluster proxy. This field is populated by apiserver
+                      only if connection type is proxy.
                    type: string
                  type:
-                    description: type defines how host cluster will connect to host cluster ConnectionTypeDirect means direct connection, this requires   kubeconfig and kubesphere apiserver endpoint provided ConnectionTypeProxy means using kubesphere proxy, no kubeconfig   or kubesphere apiserver endpoint required
+                    description: type defines how host cluster will connect to host
+                      cluster ConnectionTypeDirect means direct connection, this requires   kubeconfig
+                      and kubesphere apiserver endpoint provided ConnectionTypeProxy
+                      means using kubesphere proxy, no kubeconfig   or kubesphere
+                      apiserver endpoint required
                    type: string
                type: object
              enable:
                description: Desired state of the cluster
                type: boolean
+              externalKubeAPIEnabled:
+                description: ExternalKubeAPIEnabled export kubeapiserver to public
+                  use a lb type service if connection type is proxy
+                type: boolean
              joinFederation:
                description: Join cluster as a kubefed cluster
                type: boolean
@@ -83,11 +113,13 @@ spec:
          status:
            properties:
              conditions:
-                description: Represents the latest available observations of a cluster's current state.
+                description: Represents the latest available observations of a cluster's
+                  current state.
                items:
                  properties:
                    lastTransitionTime:
-                      description: Last time the condition transitioned from one status to another.
+                      description: Last time the condition transitioned from one status
+                        to another.
                      format: date-time
                      type: string
                    lastUpdateTime:
@@ -95,7 +127,8 @@ spec:
                      format: date-time
                      type: string
                    message:
-                      description: A human readable message indicating details about the transition.
+                      description: A human readable message indicating details about
+                        the transition.
                      type: string
                    reason:
                      description: The reason for the condition's last transition.
@@ -114,22 +147,33 @@ spec:
              configz:
                additionalProperties:
                  type: boolean
-                description: Configz is status of components enabled in the member cluster. This is synchronized with member cluster every amount of time, like 5 minutes.
+                description: Configz is status of components enabled in the member
+                  cluster. This is synchronized with member cluster every amount of
+                  time, like 5 minutes.
                type: object
              kubeSphereVersion:
-                description: GitVersion of the /kapis/version api response, this field is populated by cluster controller
+                description: GitVersion of the /kapis/version api response, this field
+                  is populated by cluster controller
                type: string
              kubernetesVersion:
-                description: GitVersion of the kubernetes cluster, this field is populated by cluster controller
+                description: GitVersion of the kubernetes cluster, this field is populated
+                  by cluster controller
                type: string
              nodeCount:
-                description: Count of the kubernetes cluster nodes This field may not reflect the instant status of the cluster.
+                description: Count of the kubernetes cluster nodes This field may
+                  not reflect the instant status of the cluster.
                type: integer
              region:
-                description: Region is the name of the region in which all of the nodes in the cluster exist.  e.g. 'us-east1'.
+                description: Region is the name of the region in which all of the
+                  nodes in the cluster exist.  e.g. 'us-east1'.
+                type: string
+              uid:
+                description: UID is the kube-system namespace UID of the cluster,
+                  which represents the unique ID of the cluster.
                type: string
              zones:
-                description: Zones are the names of availability zones in which the nodes of the cluster exist, e.g. 'us-east1-a'.
+                description: Zones are the names of availability zones in which the
+                  nodes of the cluster exist, e.g. 'us-east1-a'.
                items:
                  type: string
                type: array
--- a/config/crds/gateway.kubesphere.io_gateways.yaml
+++ b/config/crds/gateway.kubesphere.io_gateways.yaml
@@ -66,6 +66,33 @@ spec:
                  replicas:
                    format: int32
                    type: integer
+                  resources:
+                    description: ResourceRequirements describes the compute resource
+                      requirements.
+                    properties:
+                      limits:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Limits describes the maximum amount of compute
+                          resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                      requests:
+                        additionalProperties:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                        description: 'Requests describes the minimum amount of compute
+                          resources required. If Requests is omitted for a container,
+                          it defaults to Limits if that is explicitly specified, otherwise
+                          to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/'
+                        type: object
+                    type: object
                type: object
              service:
                properties:
--- a/config/crds/iam.kubesphere.io_federatedrolebindings.yaml
+++ b/config/crds/iam.kubesphere.io_federatedrolebindings.yaml
@@ -0,0 +1,128 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  creationTimestamp: null
+  name: federatedrolebindings.iam.kubesphere.io
+spec:
+  group: iam.kubesphere.io
+  names:
+    kind: FederatedRoleBinding
+    listKind: FederatedRoleBindingList
+    plural: federatedrolebindings
+    singular: federatedrolebinding
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              placement:
+                properties:
+                  clusterSelector:
+                    properties:
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        type: object
+                    type: object
+                  clusters:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                type: object
+              template:
+                properties:
+                  metadata:
+                    type: object
+                  roleRef:
+                    description: RoleRef contains information that points to the role
+                      being used
+                    properties:
+                      apiGroup:
+                        description: APIGroup is the group for the resource being
+                          referenced
+                        type: string
+                      kind:
+                        description: Kind is the type of resource being referenced
+                        type: string
+                      name:
+                        description: Name is the name of resource being referenced
+                        type: string
+                    required:
+                    - apiGroup
+                    - kind
+                    - name
+                    type: object
+                  subjects:
+                    items:
+                      description: Subject contains a reference to the object or user
+                        identities a role binding applies to.  This can either hold
+                        a direct API object reference, or a value for non-objects
+                        such as user and group names.
+                      properties:
+                        apiGroup:
+                          description: APIGroup holds the API group of the referenced
+                            subject. Defaults to "" for ServiceAccount subjects. Defaults
+                            to "rbac.authorization.k8s.io" for User and Group subjects.
+                          type: string
+                        kind:
+                          description: Kind of object being referenced. Values defined
+                            by this API group are "User", "Group", and "ServiceAccount".
+                            If the Authorizer does not recognized the kind value,
+                            the Authorizer should report an error.
+                          type: string
+                        name:
+                          description: Name of the object being referenced.
+                          type: string
+                        namespace:
+                          description: Namespace of the referenced object.  If the
+                            object kind is non-namespace, such as "User" or "Group",
+                            and this value is not empty the Authorizer should report
+                            an error.
+                          type: string
+                      required:
+                      - kind
+                      - name
+                      type: object
+                    type: array
+                required:
+                - roleRef
+                type: object
+            required:
+            - placement
+            - template
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/iam.kubesphere.io_federatedroles.yaml
+++ b/config/crds/iam.kubesphere.io_federatedroles.yaml
@@ -0,0 +1,125 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  creationTimestamp: null
+  name: federatedroles.iam.kubesphere.io
+spec:
+  group: iam.kubesphere.io
+  names:
+    kind: FederatedRole
+    listKind: FederatedRoleList
+    plural: federatedroles
+    singular: federatedrole
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              placement:
+                properties:
+                  clusterSelector:
+                    properties:
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        type: object
+                    type: object
+                  clusters:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                type: object
+              template:
+                properties:
+                  metadata:
+                    type: object
+                  rules:
+                    items:
+                      description: PolicyRule holds information that describes a policy
+                        rule, but does not contain information about who the rule
+                        applies to or which namespace the rule applies to.
+                      properties:
+                        apiGroups:
+                          description: APIGroups is the name of the APIGroup that
+                            contains the resources.  If multiple API groups are specified,
+                            any action requested against one of the enumerated resources
+                            in any API group will be allowed.
+                          items:
+                            type: string
+                          type: array
+                        nonResourceURLs:
+                          description: NonResourceURLs is a set of partial urls that
+                            a user should have access to.  *s are allowed, but only
+                            as the full, final step in the path Since non-resource
+                            URLs are not namespaced, this field is only applicable
+                            for ClusterRoles referenced from a ClusterRoleBinding.
+                            Rules can either apply to API resources (such as "pods"
+                            or "secrets") or non-resource URL paths (such as "/api"),  but
+                            not both.
+                          items:
+                            type: string
+                          type: array
+                        resourceNames:
+                          description: ResourceNames is an optional white list of
+                            names that the rule applies to.  An empty set means that
+                            everything is allowed.
+                          items:
+                            type: string
+                          type: array
+                        resources:
+                          description: Resources is a list of resources this rule
+                            applies to.  ResourceAll represents all resources.
+                          items:
+                            type: string
+                          type: array
+                        verbs:
+                          description: Verbs is a list of Verbs that apply to ALL
+                            the ResourceKinds and AttributeRestrictions contained
+                            in this rule.  VerbAll represents all kinds.
+                          items:
+                            type: string
+                          type: array
+                      required:
+                      - verbs
+                      type: object
+                    type: array
+                type: object
+            required:
+            - placement
+            - template
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/iam.kubesphere.io_federatedusers.yaml
+++ b/config/crds/iam.kubesphere.io_federatedusers.yaml
@@ -0,0 +1,139 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  creationTimestamp: null
+  name: federatedusers.iam.kubesphere.io
+spec:
+  group: iam.kubesphere.io
+  names:
+    kind: FederatedUser
+    listKind: FederatedUserList
+    plural: federatedusers
+    singular: federateduser
+  scope: Namespaced
+  versions:
+  - name: v1alpha2
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              placement:
+                properties:
+                  clusterSelector:
+                    properties:
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        type: object
+                    type: object
+                  clusters:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                      required:
+                      - name
+                      type: object
+                    type: array
+                type: object
+              template:
+                properties:
+                  metadata:
+                    type: object
+                  spec:
+                    description: UserSpec defines the desired state of User
+                    properties:
+                      description:
+                        description: Description of the user.
+                        type: string
+                      displayName:
+                        type: string
+                      email:
+                        description: Unique email address(https://www.ietf.org/rfc/rfc5322.txt).
+                        type: string
+                      groups:
+                        items:
+                          type: string
+                        type: array
+                      lang:
+                        description: The preferred written or spoken language for
+                          the user.
+                        type: string
+                      password:
+                        description: 'password will be encrypted by mutating admission
+                          webhook Password pattern is tricky here. The rule is simple:
+                          length between [6,64], at least one uppercase letter, one
+                          lowercase letter, one digit. The regexp in console(javascript)
+                          is quite straightforward: ^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)[^]{6,64}$
+                          But in Go, we don''t have ?= (back tracking) capability
+                          in regexp (also in CRD validation pattern) So we adopted
+                          an alternative scheme to achieve. Use 6 different regexp
+                          to combine to achieve the same effect. These six schemes
+                          enumerate the arrangement of numbers, uppercase letters,
+                          and lowercase letters that appear for the first time. -
+                          ^(.*[a-z].*[A-Z].*[0-9].*)$ stands for lowercase letter
+                          comes first, then followed by an uppercase letter, then
+                          a digit. - ^(.*[a-z].*[0-9].*[A-Z].*)$ stands for lowercase
+                          letter comes first, then followed by a digit, then an uppercase
+                          leeter. - ^(.*[A-Z].*[a-z].*[0-9].*)$ ... - ^(.*[A-Z].*[0-9].*[a-z].*)$
+                          ... - ^(.*[0-9].*[a-z].*[A-Z].*)$ ... - ^(.*[0-9].*[A-Z].*[a-z].*)$
+                          ... Last but not least, the bcrypt string is also included
+                          to match the encrypted password. ^(\$2[ayb]\$.{56})$'
+                        maxLength: 64
+                        minLength: 6
+                        pattern: ^(.*[a-z].*[A-Z].*[0-9].*)$|^(.*[a-z].*[0-9].*[A-Z].*)$|^(.*[A-Z].*[a-z].*[0-9].*)$|^(.*[A-Z].*[0-9].*[a-z].*)$|^(.*[0-9].*[a-z].*[A-Z].*)$|^(.*[0-9].*[A-Z].*[a-z].*)$|^(\$2[ayb]\$.{56})$
+                        type: string
+                    required:
+                    - email
+                    type: object
+                  status:
+                    description: UserStatus defines the observed state of User
+                    properties:
+                      lastLoginTime:
+                        description: Last login attempt timestamp
+                        format: date-time
+                        type: string
+                      lastTransitionTime:
+                        format: date-time
+                        type: string
+                      reason:
+                        type: string
+                      state:
+                        description: The user status
+                        type: string
+                    type: object
+                required:
+                - spec
+                type: object
+            required:
+            - placement
+            - template
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/iam.kubesphere.io_globalrolebindings.yaml
+++ b/config/crds/iam.kubesphere.io_globalrolebindings.yaml
@@ -24,15 +24,20 @@ spec:
        description: GlobalRoleBinding is the Schema for the globalrolebindings API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          roleRef:
-            description: RoleRef can only reference a GlobalRole. If the RoleRef cannot be resolved, the Authorizer must return an error.
+            description: RoleRef can only reference a GlobalRole. If the RoleRef cannot
+              be resolved, the Authorizer must return an error.
            properties:
              apiGroup:
                description: APIGroup is the group for the resource being referenced
@@ -49,21 +54,31 @@ spec:
            - name
            type: object
          subjects:
-            description: Subjects holds references to the objects the role applies to.
+            description: Subjects holds references to the objects the role applies
+              to.
            items:
-              description: Subject contains a reference to the object or user identities a role binding applies to.  This can either hold a direct API object reference, or a value for non-objects such as user and group names.
+              description: Subject contains a reference to the object or user identities
+                a role binding applies to.  This can either hold a direct API object
+                reference, or a value for non-objects such as user and group names.
              properties:
                apiGroup:
-                  description: APIGroup holds the API group of the referenced subject. Defaults to "" for ServiceAccount subjects. Defaults to "rbac.authorization.k8s.io" for User and Group subjects.
+                  description: APIGroup holds the API group of the referenced subject.
+                    Defaults to "" for ServiceAccount subjects. Defaults to "rbac.authorization.k8s.io"
+                    for User and Group subjects.
                  type: string
                kind:
-                  description: Kind of object being referenced. Values defined by this API group are "User", "Group", and "ServiceAccount". If the Authorizer does not recognized the kind value, the Authorizer should report an error.
+                  description: Kind of object being referenced. Values defined by
+                    this API group are "User", "Group", and "ServiceAccount". If the
+                    Authorizer does not recognized the kind value, the Authorizer
+                    should report an error.
                  type: string
                name:
                  description: Name of the object being referenced.
                  type: string
                namespace:
-                  description: Namespace of the referenced object.  If the object kind is non-namespace, such as "User" or "Group", and this value is not empty the Authorizer should report an error.
+                  description: Namespace of the referenced object.  If the object
+                    kind is non-namespace, such as "User" or "Group", and this value
+                    is not empty the Authorizer should report an error.
                  type: string
              required:
              - kind
--- a/config/crds/iam.kubesphere.io_globalroles.yaml
+++ b/config/crds/iam.kubesphere.io_globalroles.yaml
@@ -23,40 +23,59 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          rules:
            description: Rules holds all the PolicyRules for this GlobalRole
            items:
-              description: PolicyRule holds information that describes a policy rule, but does not contain information about who the rule applies to or which namespace the rule applies to.
+              description: PolicyRule holds information that describes a policy rule,
+                but does not contain information about who the rule applies to or
+                which namespace the rule applies to.
              properties:
                apiGroups:
-                  description: APIGroups is the name of the APIGroup that contains the resources.  If multiple API groups are specified, any action requested against one of the enumerated resources in any API group will be allowed.
+                  description: APIGroups is the name of the APIGroup that contains
+                    the resources.  If multiple API groups are specified, any action
+                    requested against one of the enumerated resources in any API group
+                    will be allowed.
                  items:
                    type: string
                  type: array
                nonResourceURLs:
-                  description: NonResourceURLs is a set of partial urls that a user should have access to.  *s are allowed, but only as the full, final step in the path Since non-resource URLs are not namespaced, this field is only applicable for ClusterRoles referenced from a ClusterRoleBinding. Rules can either apply to API resources (such as "pods" or "secrets") or non-resource URL paths (such as "/api"),  but not both.
+                  description: NonResourceURLs is a set of partial urls that a user
+                    should have access to.  *s are allowed, but only as the full,
+                    final step in the path Since non-resource URLs are not namespaced,
+                    this field is only applicable for ClusterRoles referenced from
+                    a ClusterRoleBinding. Rules can either apply to API resources
+                    (such as "pods" or "secrets") or non-resource URL paths (such
+                    as "/api"),  but not both.
                  items:
                    type: string
                  type: array
                resourceNames:
-                  description: ResourceNames is an optional white list of names that the rule applies to.  An empty set means that everything is allowed.
+                  description: ResourceNames is an optional white list of names that
+                    the rule applies to.  An empty set means that everything is allowed.
                  items:
                    type: string
                  type: array
                resources:
-                  description: Resources is a list of resources this rule applies to.  ResourceAll represents all resources.
+                  description: Resources is a list of resources this rule applies
+                    to.  ResourceAll represents all resources.
                  items:
                    type: string
                  type: array
                verbs:
-                  description: Verbs is a list of Verbs that apply to ALL the ResourceKinds and AttributeRestrictions contained in this rule.  VerbAll represents all kinds.
+                  description: Verbs is a list of Verbs that apply to ALL the ResourceKinds
+                    and AttributeRestrictions contained in this rule.  VerbAll represents
+                    all kinds.
                  items:
                    type: string
                  type: array
--- a/config/crds/iam.kubesphere.io_groupbindings.yaml
+++ b/config/crds/iam.kubesphere.io_groupbindings.yaml
@@ -31,7 +31,9 @@ spec:
        description: GroupBinding is the Schema for the groupbindings API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          groupRef:
            description: GroupRef defines the desired relation of GroupBinding
@@ -44,7 +46,9 @@ spec:
                type: string
            type: object
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/iam.kubesphere.io_groups.yaml
+++ b/config/crds/iam.kubesphere.io_groups.yaml
@@ -28,10 +28,14 @@ spec:
        description: Group is the Schema for the groups API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/iam.kubesphere.io_loginrecords.yaml
+++ b/config/crds/iam.kubesphere.io_loginrecords.yaml
@@ -42,10 +42,14 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/iam.kubesphere.io_rolebases.yaml
+++ b/config/crds/iam.kubesphere.io_rolebases.yaml
@@ -23,10 +23,14 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/iam.kubesphere.io_users.yaml
+++ b/config/crds/iam.kubesphere.io_users.yaml
@@ -31,10 +31,14 @@ spec:
        description: User is the Schema for the users API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -57,7 +61,26 @@ spec:
                description: The preferred written or spoken language for the user.
                type: string
              password:
-                description: password will be encrypted by mutating admission webhook
+                description: 'password will be encrypted by mutating admission webhook
+                  Password pattern is tricky here. The rule is simple: length between
+                  [6,64], at least one uppercase letter, one lowercase letter, one
+                  digit. The regexp in console(javascript) is quite straightforward:
+                  ^(?=.*[a-z])(?=.*[A-Z])(?=.*\d)[^]{6,64}$ But in Go, we don''t have
+                  ?= (back tracking) capability in regexp (also in CRD validation
+                  pattern) So we adopted an alternative scheme to achieve. Use 6 different
+                  regexp to combine to achieve the same effect. These six schemes
+                  enumerate the arrangement of numbers, uppercase letters, and lowercase
+                  letters that appear for the first time. - ^(.*[a-z].*[A-Z].*[0-9].*)$
+                  stands for lowercase letter comes first, then followed by an uppercase
+                  letter, then a digit. - ^(.*[a-z].*[0-9].*[A-Z].*)$ stands for lowercase
+                  letter comes first, then followed by a digit, then an uppercase
+                  leeter. - ^(.*[A-Z].*[a-z].*[0-9].*)$ ... - ^(.*[A-Z].*[0-9].*[a-z].*)$
+                  ... - ^(.*[0-9].*[a-z].*[A-Z].*)$ ... - ^(.*[0-9].*[A-Z].*[a-z].*)$
+                  ... Last but not least, the bcrypt string is also included to match
+                  the encrypted password. ^(\$2[ayb]\$.{56})$'
+                maxLength: 64
+                minLength: 6
+                pattern: ^(.*[a-z].*[A-Z].*[0-9].*)$|^(.*[a-z].*[0-9].*[A-Z].*)$|^(.*[A-Z].*[a-z].*[0-9].*)$|^(.*[A-Z].*[0-9].*[a-z].*)$|^(.*[0-9].*[a-z].*[A-Z].*)$|^(.*[0-9].*[A-Z].*[a-z].*)$|^(\$2[ayb]\$.{56})$
                type: string
            required:
            - email
--- a/config/crds/iam.kubesphere.io_workspacerolebindings.yaml
+++ b/config/crds/iam.kubesphere.io_workspacerolebindings.yaml
@@ -25,18 +25,24 @@ spec:
    name: v1alpha2
    schema:
      openAPIV3Schema:
-        description: WorkspaceRoleBinding is the Schema for the workspacerolebindings API
+        description: WorkspaceRoleBinding is the Schema for the workspacerolebindings
+          API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          roleRef:
-            description: RoleRef can only reference a WorkspaceRole. If the RoleRef cannot be resolved, the Authorizer must return an error.
+            description: RoleRef can only reference a WorkspaceRole. If the RoleRef
+              cannot be resolved, the Authorizer must return an error.
            properties:
              apiGroup:
                description: APIGroup is the group for the resource being referenced
@@ -53,21 +59,31 @@ spec:
            - name
            type: object
          subjects:
-            description: Subjects holds references to the objects the role applies to.
+            description: Subjects holds references to the objects the role applies
+              to.
            items:
-              description: Subject contains a reference to the object or user identities a role binding applies to.  This can either hold a direct API object reference, or a value for non-objects such as user and group names.
+              description: Subject contains a reference to the object or user identities
+                a role binding applies to.  This can either hold a direct API object
+                reference, or a value for non-objects such as user and group names.
              properties:
                apiGroup:
-                  description: APIGroup holds the API group of the referenced subject. Defaults to "" for ServiceAccount subjects. Defaults to "rbac.authorization.k8s.io" for User and Group subjects.
+                  description: APIGroup holds the API group of the referenced subject.
+                    Defaults to "" for ServiceAccount subjects. Defaults to "rbac.authorization.k8s.io"
+                    for User and Group subjects.
                  type: string
                kind:
-                  description: Kind of object being referenced. Values defined by this API group are "User", "Group", and "ServiceAccount". If the Authorizer does not recognized the kind value, the Authorizer should report an error.
+                  description: Kind of object being referenced. Values defined by
+                    this API group are "User", "Group", and "ServiceAccount". If the
+                    Authorizer does not recognized the kind value, the Authorizer
+                    should report an error.
                  type: string
                name:
                  description: Name of the object being referenced.
                  type: string
                namespace:
-                  description: Namespace of the referenced object.  If the object kind is non-namespace, such as "User" or "Group", and this value is not empty the Authorizer should report an error.
+                  description: Namespace of the referenced object.  If the object
+                    kind is non-namespace, such as "User" or "Group", and this value
+                    is not empty the Authorizer should report an error.
                  type: string
              required:
              - kind
--- a/config/crds/iam.kubesphere.io_workspaceroles.yaml
+++ b/config/crds/iam.kubesphere.io_workspaceroles.yaml
@@ -30,40 +30,59 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          rules:
            description: Rules holds all the PolicyRules for this WorkspaceRole
            items:
-              description: PolicyRule holds information that describes a policy rule, but does not contain information about who the rule applies to or which namespace the rule applies to.
+              description: PolicyRule holds information that describes a policy rule,
+                but does not contain information about who the rule applies to or
+                which namespace the rule applies to.
              properties:
                apiGroups:
-                  description: APIGroups is the name of the APIGroup that contains the resources.  If multiple API groups are specified, any action requested against one of the enumerated resources in any API group will be allowed.
+                  description: APIGroups is the name of the APIGroup that contains
+                    the resources.  If multiple API groups are specified, any action
+                    requested against one of the enumerated resources in any API group
+                    will be allowed.
                  items:
                    type: string
                  type: array
                nonResourceURLs:
-                  description: NonResourceURLs is a set of partial urls that a user should have access to.  *s are allowed, but only as the full, final step in the path Since non-resource URLs are not namespaced, this field is only applicable for ClusterRoles referenced from a ClusterRoleBinding. Rules can either apply to API resources (such as "pods" or "secrets") or non-resource URL paths (such as "/api"),  but not both.
+                  description: NonResourceURLs is a set of partial urls that a user
+                    should have access to.  *s are allowed, but only as the full,
+                    final step in the path Since non-resource URLs are not namespaced,
+                    this field is only applicable for ClusterRoles referenced from
+                    a ClusterRoleBinding. Rules can either apply to API resources
+                    (such as "pods" or "secrets") or non-resource URL paths (such
+                    as "/api"),  but not both.
                  items:
                    type: string
                  type: array
                resourceNames:
-                  description: ResourceNames is an optional white list of names that the rule applies to.  An empty set means that everything is allowed.
+                  description: ResourceNames is an optional white list of names that
+                    the rule applies to.  An empty set means that everything is allowed.
                  items:
                    type: string
                  type: array
                resources:
-                  description: Resources is a list of resources this rule applies to.  ResourceAll represents all resources.
+                  description: Resources is a list of resources this rule applies
+                    to.  ResourceAll represents all resources.
                  items:
                    type: string
                  type: array
                verbs:
-                  description: Verbs is a list of Verbs that apply to ALL the ResourceKinds and AttributeRestrictions contained in this rule.  VerbAll represents all kinds.
+                  description: Verbs is a list of Verbs that apply to ALL the ResourceKinds
+                    and AttributeRestrictions contained in this rule.  VerbAll represents
+                    all kinds.
                  items:
                    type: string
                  type: array
--- a/config/crds/network.kubesphere.io_ipamblocks.yaml
+++ b/config/crds/network.kubesphere.io_ipamblocks.yaml
@@ -21,10 +21,14 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/network.kubesphere.io_ipamhandles.yaml
+++ b/config/crds/network.kubesphere.io_ipamhandles.yaml
@@ -21,10 +21,14 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/network.kubesphere.io_ippools.yaml
+++ b/config/crds/network.kubesphere.io_ippools.yaml
@@ -21,23 +21,29 @@ spec:
      openAPIV3Schema:
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
            properties:
              blockSize:
-                description: The block size to use for IP address assignments from this pool. Defaults to 26 for IPv4 and 112 for IPv6.
+                description: The block size to use for IP address assignments from
+                  this pool. Defaults to 26 for IPv4 and 112 for IPv6.
                type: integer
              cidr:
                description: The pool CIDR.
                type: string
              disabled:
-                description: When disabled is true, IPAM will not assign addresses from this pool.
+                description: When disabled is true, IPAM will not assign addresses
+                  from this pool.
                type: boolean
              dns:
                description: DNS contains values interesting for DNS resolvers
--- a/config/crds/network.kubesphere.io_namespacenetworkpolicies.yaml
+++ b/config/crds/network.kubesphere.io_namespacenetworkpolicies.yaml
@@ -19,59 +19,109 @@ spec:
    - nsnp
    singular: namespacenetworkpolicy
  scope: Namespaced
-  preserveUnknownFields: false
  versions:
  - name: v1alpha1
    schema:
      openAPIV3Schema:
-        description: NamespaceNetworkPolicy is the Schema for the namespacenetworkpolicies API
+        description: NamespaceNetworkPolicy is the Schema for the namespacenetworkpolicies
+          API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
          spec:
-            description: NamespaceNetworkPolicySpec provides the specification of a NamespaceNetworkPolicy
+            description: NamespaceNetworkPolicySpec provides the specification of
+              a NamespaceNetworkPolicy
            properties:
              egress:
-                description: List of egress rules to be applied to the selected pods. Outgoing traffic is allowed if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic matches at least one egress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy limits all outgoing traffic (and serves solely to ensure that the pods it selects are isolated by default). This field is beta-level in 1.8
+                description: List of egress rules to be applied to the selected pods.
+                  Outgoing traffic is allowed if there are no NetworkPolicies selecting
+                  the pod (and cluster policy otherwise allows the traffic), OR if
+                  the traffic matches at least one egress rule across all of the NetworkPolicy
+                  objects whose podSelector matches the pod. If this field is empty
+                  then this NetworkPolicy limits all outgoing traffic (and serves
+                  solely to ensure that the pods it selects are isolated by default).
+                  This field is beta-level in 1.8
                items:
-                  description: NetworkPolicyEgressRule describes a particular set of traffic that is allowed out of pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and to. This type is beta-level in 1.8
+                  description: NetworkPolicyEgressRule describes a particular set
+                    of traffic that is allowed out of pods matched by a NetworkPolicySpec's
+                    podSelector. The traffic must match both ports and to. This type
+                    is beta-level in 1.8
                  properties:
                    ports:
-                      description: List of destination ports for outgoing traffic. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.
+                      description: List of destination ports for outgoing traffic.
+                        Each item in this list is combined using a logical OR. If
+                        this field is empty or missing, this rule matches all ports
+                        (traffic not restricted by port). If this field is present
+                        and contains at least one item, then this rule allows traffic
+                        only if the traffic matches at least one port in the list.
                      items:
-                        description: NetworkPolicyPort describes a port to allow traffic on
+                        description: NetworkPolicyPort describes a port to allow traffic
+                          on
                        properties:
+                          endPort:
+                            description: If set, indicates that the range of ports
+                              from port to endPort, inclusive, should be allowed by
+                              the policy. This field cannot be defined if the port
+                              field is not defined or if the port field is defined
+                              as a named (string) port. The endPort must be equal
+                              or greater than port. This feature is in Alpha state
+                              and should be enabled using the Feature Gate "NetworkPolicyEndPort".
+                            format: int32
+                            type: integer
                          port:
                            anyOf:
                            - type: integer
                            - type: string
-                            description: The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.
+                            description: The port on the given protocol. This can
+                              either be a numerical or named port on a pod. If this
+                              field is not provided, this matches all port names and
+                              numbers. If present, only traffic on the specified protocol
+                              AND port will be matched.
                            x-kubernetes-int-or-string: true
                          protocol:
                            default: TCP
-                            description: The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.
+                            description: The protocol (TCP, UDP, or SCTP) which traffic
+                              must match. If not specified, this field defaults to
+                              TCP.
                            type: string
                        type: object
                      type: array
                    to:
-                      description: List of destinations for outgoing traffic of pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all destinations (traffic not restricted by destination). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the to list.
+                      description: List of destinations for outgoing traffic of pods
+                        selected for this rule. Items in this list are combined using
+                        a logical OR operation. If this field is empty or missing,
+                        this rule matches all destinations (traffic not restricted
+                        by destination). If this field is present and contains at
+                        least one item, this rule allows traffic only if the traffic
+                        matches at least one item in the to list.
                      items:
-                        description: NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed
+                        description: NetworkPolicyPeer describes a peer to allow traffic
+                          from. Only certain combinations of fields are allowed
                        properties:
                          ipBlock:
-                            description: IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.
+                            description: IPBlock defines policy on a particular IPBlock.
+                              If this field is set then neither of the other fields
+                              can be.
                            properties:
                              cidr:
-                                description: CIDR is a string representing the IP Block Valid examples are "192.168.1.1/24" or "2001:db9::/64"
+                                description: CIDR is a string representing the IP
+                                  Block Valid examples are "192.168.1.1/24" or "2001:db9::/64"
                                type: string
                              except:
-                                description: Except is a slice of CIDRs that should not be included within an IP Block Valid examples are "192.168.1.1/24" or "2001:db9::/64" Except values will be rejected if they are outside the CIDR range
+                                description: Except is a slice of CIDRs that should
+                                  not be included within an IP Block Valid examples
+                                  are "192.168.1.1/24" or "2001:db9::/64" Except values
+                                  will be rejected if they are outside the CIDR range
                                items:
                                  type: string
                                type: array
@@ -100,23 +150,45 @@ spec:
                  type: object
                type: array
              ingress:
-                description: List of ingress rules to be applied to the selected pods. Traffic is allowed to a pod if there are no NetworkPolicies selecting the pod (and cluster policy otherwise allows the traffic), OR if the traffic source is the pod's local node, OR if the traffic matches at least one ingress rule across all of the NetworkPolicy objects whose podSelector matches the pod. If this field is empty then this NetworkPolicy does not allow any traffic (and serves solely to ensure that the pods it selects are isolated by default)
+                description: List of ingress rules to be applied to the selected pods.
+                  Traffic is allowed to a pod if there are no NetworkPolicies selecting
+                  the pod (and cluster policy otherwise allows the traffic), OR if
+                  the traffic source is the pod's local node, OR if the traffic matches
+                  at least one ingress rule across all of the NetworkPolicy objects
+                  whose podSelector matches the pod. If this field is empty then this
+                  NetworkPolicy does not allow any traffic (and serves solely to ensure
+                  that the pods it selects are isolated by default)
                items:
-                  description: NetworkPolicyIngressRule describes a particular set of traffic that is allowed to the pods matched by a NetworkPolicySpec's podSelector. The traffic must match both ports and from.
+                  description: NetworkPolicyIngressRule describes a particular set
+                    of traffic that is allowed to the pods matched by a NetworkPolicySpec's
+                    podSelector. The traffic must match both ports and from.
                  properties:
                    from:
-                      description: List of sources which should be able to access the pods selected for this rule. Items in this list are combined using a logical OR operation. If this field is empty or missing, this rule matches all sources (traffic not restricted by source). If this field is present and contains at least one item, this rule allows traffic only if the traffic matches at least one item in the from list.
+                      description: List of sources which should be able to access
+                        the pods selected for this rule. Items in this list are combined
+                        using a logical OR operation. If this field is empty or missing,
+                        this rule matches all sources (traffic not restricted by source).
+                        If this field is present and contains at least one item, this
+                        rule allows traffic only if the traffic matches at least one
+                        item in the from list.
                      items:
-                        description: NetworkPolicyPeer describes a peer to allow traffic from. Only certain combinations of fields are allowed
+                        description: NetworkPolicyPeer describes a peer to allow traffic
+                          from. Only certain combinations of fields are allowed
                        properties:
                          ipBlock:
-                            description: IPBlock defines policy on a particular IPBlock. If this field is set then neither of the other fields can be.
+                            description: IPBlock defines policy on a particular IPBlock.
+                              If this field is set then neither of the other fields
+                              can be.
                            properties:
                              cidr:
-                                description: CIDR is a string representing the IP Block Valid examples are "192.168.1.1/24" or "2001:db9::/64"
+                                description: CIDR is a string representing the IP
+                                  Block Valid examples are "192.168.1.1/24" or "2001:db9::/64"
                                type: string
                              except:
-                                description: Except is a slice of CIDRs that should not be included within an IP Block Valid examples are "192.168.1.1/24" or "2001:db9::/64" Except values will be rejected if they are outside the CIDR range
+                                description: Except is a slice of CIDRs that should
+                                  not be included within an IP Block Valid examples
+                                  are "192.168.1.1/24" or "2001:db9::/64" Except values
+                                  will be rejected if they are outside the CIDR range
                                items:
                                  type: string
                                type: array
@@ -143,28 +215,63 @@ spec:
                        type: object
                      type: array
                    ports:
-                      description: List of ports which should be made accessible on the pods selected for this rule. Each item in this list is combined using a logical OR. If this field is empty or missing, this rule matches all ports (traffic not restricted by port). If this field is present and contains at least one item, then this rule allows traffic only if the traffic matches at least one port in the list.
+                      description: List of ports which should be made accessible on
+                        the pods selected for this rule. Each item in this list is
+                        combined using a logical OR. If this field is empty or missing,
+                        this rule matches all ports (traffic not restricted by port).
+                        If this field is present and contains at least one item, then
+                        this rule allows traffic only if the traffic matches at least
+                        one port in the list.
                      items:
-                        description: NetworkPolicyPort describes a port to allow traffic on
+                        description: NetworkPolicyPort describes a port to allow traffic
+                          on
                        properties:
+                          endPort:
+                            description: If set, indicates that the range of ports
+                              from port to endPort, inclusive, should be allowed by
+                              the policy. This field cannot be defined if the port
+                              field is not defined or if the port field is defined
+                              as a named (string) port. The endPort must be equal
+                              or greater than port. This feature is in Alpha state
+                              and should be enabled using the Feature Gate "NetworkPolicyEndPort".
+                            format: int32
+                            type: integer
                          port:
                            anyOf:
                            - type: integer
                            - type: string
-                            description: The port on the given protocol. This can either be a numerical or named port on a pod. If this field is not provided, this matches all port names and numbers.
+                            description: The port on the given protocol. This can
+                              either be a numerical or named port on a pod. If this
+                              field is not provided, this matches all port names and
+                              numbers. If present, only traffic on the specified protocol
+                              AND port will be matched.
                            x-kubernetes-int-or-string: true
                          protocol:
                            default: TCP
-                            description: The protocol (TCP, UDP, or SCTP) which traffic must match. If not specified, this field defaults to TCP.
+                            description: The protocol (TCP, UDP, or SCTP) which traffic
+                              must match. If not specified, this field defaults to
+                              TCP.
                            type: string
                        type: object
                      type: array
                  type: object
                type: array
              policyTypes:
-                description: List of rule types that the NetworkPolicy relates to. Valid options are "Ingress", "Egress", or "Ingress,Egress". If this field is not specified, it will default based on the existence of Ingress or Egress rules; policies that contain an Egress section are assumed to affect Egress, and all policies (whether or not they contain an Ingress section) are assumed to affect Ingress. If you want to write an egress-only policy, you must explicitly specify policyTypes [ "Egress" ]. Likewise, if you want to write a policy that specifies that no egress is allowed, you must specify a policyTypes value that include "Egress" (since such a policy would not include an Egress section and would otherwise default to just [ "Ingress" ]). This field is beta-level in 1.8
+                description: List of rule types that the NetworkPolicy relates to.
+                  Valid options are "Ingress", "Egress", or "Ingress,Egress". If this
+                  field is not specified, it will default based on the existence of
+                  Ingress or Egress rules; policies that contain an Egress section
+                  are assumed to affect Egress, and all policies (whether or not they
+                  contain an Ingress section) are assumed to affect Ingress. If you
+                  want to write an egress-only policy, you must explicitly specify
+                  policyTypes [ "Egress" ]. Likewise, if you want to write a policy
+                  that specifies that no egress is allowed, you must specify a policyTypes
+                  value that include "Egress" (since such a policy would not include
+                  an Egress section and would otherwise default to just [ "Ingress"
+                  ]). This field is beta-level in 1.8
                items:
-                  description: Policy Type string describes the NetworkPolicy type This type is beta-level in 1.8
+                  description: PolicyType string describes the NetworkPolicy type
+                    This type is beta-level in 1.8
                  type: string
                type: array
            type: object
--- a/config/crds/quota.kubesphere.io_resourcequotas.yaml
+++ b/config/crds/quota.kubesphere.io_resourcequotas.yaml
@@ -21,13 +21,18 @@ spec:
  - name: v1alpha2
    schema:
      openAPIV3Schema:
-        description: WorkspaceResourceQuota sets aggregate quota restrictions enforced per workspace
+        description: WorkspaceResourceQuota sets aggregate quota restrictions enforced
+          per workspace
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -44,24 +49,39 @@ spec:
                      - type: string
                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                      x-kubernetes-int-or-string: true
-                    description: 'hard is the set of desired hard limits for each named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
+                    description: 'hard is the set of desired hard limits for each
+                      named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
                    type: object
                  scopeSelector:
-                    description: scopeSelector is also a collection of filters like scopes that must match each object tracked by a quota but expressed using ScopeSelectorOperator in combination with possible values. For a resource to match, both scopes AND scopeSelector (if specified in spec), must be matched.
+                    description: scopeSelector is also a collection of filters like
+                      scopes that must match each object tracked by a quota but expressed
+                      using ScopeSelectorOperator in combination with possible values.
+                      For a resource to match, both scopes AND scopeSelector (if specified
+                      in spec), must be matched.
                    properties:
                      matchExpressions:
-                        description: A list of scope selector requirements by scope of the resources.
+                        description: A list of scope selector requirements by scope
+                          of the resources.
                        items:
-                          description: A scoped-resource selector requirement is a selector that contains values, a scope name, and an operator that relates the scope name and values.
+                          description: A scoped-resource selector requirement is a
+                            selector that contains values, a scope name, and an operator
+                            that relates the scope name and values.
                          properties:
                            operator:
-                              description: Represents a scope's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist.
+                              description: Represents a scope's relationship to a
+                                set of values. Valid operators are In, NotIn, Exists,
+                                DoesNotExist.
                              type: string
                            scopeName:
-                              description: The name of the scope that the selector applies to.
+                              description: The name of the scope that the selector
+                                applies to.
                              type: string
                            values:
-                              description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                              description: An array of string values. If the operator
+                                is In or NotIn, the values array must be non-empty.
+                                If the operator is Exists or DoesNotExist, the values
+                                array must be empty. This array is replaced during
+                                a strategic merge patch.
                              items:
                                type: string
                              type: array
@@ -72,9 +92,12 @@ spec:
                        type: array
                    type: object
                  scopes:
-                    description: A collection of filters that must match each object tracked by a quota. If not specified, the quota matches all objects.
+                    description: A collection of filters that must match each object
+                      tracked by a quota. If not specified, the quota matches all
+                      objects.
                    items:
-                      description: A ResourceQuotaScope defines a filter that must match each object tracked by a quota
+                      description: A ResourceQuotaScope defines a filter that must
+                        match each object tracked by a quota
                      type: string
                    type: array
                type: object
@@ -88,12 +111,14 @@ spec:
            - selector
            type: object
          status:
-            description: Status defines the actual enforced quota and its current usage
+            description: Status defines the actual enforced quota and its current
+              usage
            properties:
              namespaces:
                description: Namespaces slices the usage by project.
                items:
-                  description: ResourceQuotaStatusByNamespace gives status for a particular project
+                  description: ResourceQuotaStatusByNamespace gives status for a particular
+                    project
                  properties:
                    hard:
                      additionalProperties:
@@ -102,7 +127,8 @@ spec:
                        - type: string
                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                        x-kubernetes-int-or-string: true
-                      description: 'Hard is the set of enforced hard limits for each named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
+                      description: 'Hard is the set of enforced hard limits for each
+                        named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
                      type: object
                    namespace:
                      description: Namespace the project this status applies to
@@ -114,14 +140,16 @@ spec:
                        - type: string
                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                        x-kubernetes-int-or-string: true
-                      description: Used is the current observed total usage of the resource in the namespace.
+                      description: Used is the current observed total usage of the
+                        resource in the namespace.
                      type: object
                  required:
                  - namespace
                  type: object
                type: array
              total:
-                description: Total defines the actual enforced quota and its current usage across all projects
+                description: Total defines the actual enforced quota and its current
+                  usage across all projects
                properties:
                  hard:
                    additionalProperties:
@@ -130,7 +158,8 @@ spec:
                      - type: string
                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                      x-kubernetes-int-or-string: true
-                    description: 'Hard is the set of enforced hard limits for each named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
+                    description: 'Hard is the set of enforced hard limits for each
+                      named resource. More info: https://kubernetes.io/docs/concepts/policy/resource-quotas/'
                    type: object
                  used:
                    additionalProperties:
@@ -139,7 +168,8 @@ spec:
                      - type: string
                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                      x-kubernetes-int-or-string: true
-                    description: Used is the current observed total usage of the resource in the namespace.
+                    description: Used is the current observed total usage of the resource
+                      in the namespace.
                    type: object
                type: object
            required:
--- a/config/crds/storage.kubesphere.io_provisionercapabilities.yaml
+++ b/config/crds/storage.kubesphere.io_provisionercapabilities.yaml
@@ -0,0 +1,120 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  creationTimestamp: null
+  name: provisionercapabilities.storage.kubesphere.io
+spec:
+  group: storage.kubesphere.io
+  names:
+    kind: ProvisionerCapability
+    listKind: ProvisionerCapabilityList
+    plural: provisionercapabilities
+    singular: provisionercapability
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .spec.pluginInfo.name
+      name: Provisioner
+      type: string
+    - jsonPath: .spec.features.volume.expandMode
+      name: Expand
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: ProvisionerCapability is the schema for the provisionercapability
+          API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ProvisionerCapabilitySpec defines the desired state of ProvisionerCapability
+            properties:
+              features:
+                description: CapabilityFeatures describe storage features
+                properties:
+                  snapshot:
+                    description: SnapshotFeature describe snapshot features
+                    properties:
+                      create:
+                        type: boolean
+                      list:
+                        type: boolean
+                    required:
+                    - create
+                    - list
+                    type: object
+                  topology:
+                    type: boolean
+                  volume:
+                    description: VolumeFeature describe volume features
+                    properties:
+                      attach:
+                        type: boolean
+                      clone:
+                        type: boolean
+                      create:
+                        type: boolean
+                      expandMode:
+                        type: string
+                      list:
+                        type: boolean
+                      stats:
+                        type: boolean
+                    required:
+                    - attach
+                    - clone
+                    - create
+                    - expandMode
+                    - list
+                    - stats
+                    type: object
+                required:
+                - snapshot
+                - topology
+                - volume
+                type: object
+              pluginInfo:
+                description: PluginInfo describes plugin info
+                properties:
+                  name:
+                    type: string
+                  version:
+                    type: string
+                required:
+                - name
+                - version
+                type: object
+            required:
+            - features
+            - pluginInfo
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/storage.kubesphere.io_storageclasscapabilities.yaml
+++ b/config/crds/storage.kubesphere.io_storageclasscapabilities.yaml
@@ -0,0 +1,120 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: (devel)
+  creationTimestamp: null
+  name: storageclasscapabilities.storage.kubesphere.io
+spec:
+  group: storage.kubesphere.io
+  names:
+    kind: StorageClassCapability
+    listKind: StorageClassCapabilityList
+    plural: storageclasscapabilities
+    singular: storageclasscapability
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .spec.provisioner
+      name: Provisioner
+      type: string
+    - jsonPath: .spec.features.volume.create
+      name: Volume
+      type: boolean
+    - jsonPath: .spec.features.volume.expandMode
+      name: Expand
+      type: string
+    - jsonPath: .spec.features.volume.clone
+      name: Clone
+      type: boolean
+    - jsonPath: .spec.features.snapshot.create
+      name: Snapshot
+      type: boolean
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: StorageClassCapability is the Schema for the storage class capability
+          API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: StorageClassCapabilitySpec defines the desired state of StorageClassCapability
+            properties:
+              features:
+                description: CapabilityFeatures describe storage features
+                properties:
+                  snapshot:
+                    description: SnapshotFeature describe snapshot features
+                    properties:
+                      create:
+                        type: boolean
+                      list:
+                        type: boolean
+                    required:
+                    - create
+                    - list
+                    type: object
+                  topology:
+                    type: boolean
+                  volume:
+                    description: VolumeFeature describe volume features
+                    properties:
+                      attach:
+                        type: boolean
+                      clone:
+                        type: boolean
+                      create:
+                        type: boolean
+                      expandMode:
+                        type: string
+                      list:
+                        type: boolean
+                      stats:
+                        type: boolean
+                    required:
+                    - attach
+                    - clone
+                    - create
+                    - expandMode
+                    - list
+                    - stats
+                    type: object
+                required:
+                - snapshot
+                - topology
+                - volume
+                type: object
+              provisioner:
+                type: string
+            required:
+            - features
+            - provisioner
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/storage.kubesphere.io_storageclasseraccessor.yaml
+++ b/config/crds/storage.kubesphere.io_storageclasseraccessor.yaml
@@ -0,0 +1,180 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  creationTimestamp: null
+  name: accessors.storage.kubesphere.io
+spec:
+  group: storage.kubesphere.io
+  names:
+    kind: Accessor
+    listKind: AccessorList
+    plural: accessors
+    singular: accessor
+  scope: Cluster
+  versions:
+    - additionalPrinterColumns:
+        - jsonPath: .spec.storageClassName
+          name: StorageClass
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: Accessor is the Schema for the accessors API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: AccessorSpec defines the desired state of Accessor
+              properties:
+                namespaceSelector:
+                  properties:
+                    fieldSelector:
+                      items:
+                        properties:
+                          fieldExpressions:
+                            items:
+                              properties:
+                                field:
+                                  enum:
+                                    - Name
+                                    - Status
+                                  type: string
+                                operator:
+                                  enum:
+                                    - In
+                                    - NotIn
+                                  type: string
+                                values:
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                                - field
+                                - operator
+                                - values
+                              type: object
+                            type: array
+                        required:
+                          - fieldExpressions
+                        type: object
+                      type: array
+                    labelSelector:
+                      items:
+                        properties:
+                          matchExpressions:
+                            items:
+                              properties:
+                                key:
+                                  type: string
+                                operator:
+                                  enum:
+                                    - In
+                                    - NotIn
+                                  type: string
+                                values:
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                                - key
+                                - operator
+                                - values
+                              type: object
+                            type: array
+                        required:
+                          - matchExpressions
+                        type: object
+                      type: array
+                  type: object
+                storageClassName:
+                  type: string
+                workspaceSelector:
+                  properties:
+                    fieldSelector:
+                      items:
+                        properties:
+                          fieldExpressions:
+                            items:
+                              properties:
+                                field:
+                                  enum:
+                                    - Name
+                                    - Status
+                                  type: string
+                                operator:
+                                  enum:
+                                    - In
+                                    - NotIn
+                                  type: string
+                                values:
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                                - field
+                                - operator
+                                - values
+                              type: object
+                            type: array
+                        required:
+                          - fieldExpressions
+                        type: object
+                      type: array
+                    labelSelector:
+                      items:
+                        properties:
+                          matchExpressions:
+                            items:
+                              properties:
+                                key:
+                                  type: string
+                                operator:
+                                  enum:
+                                    - In
+                                    - NotIn
+                                  type: string
+                                values:
+                                  items:
+                                    type: string
+                                  type: array
+                              required:
+                                - key
+                                - operator
+                                - values
+                              type: object
+                            type: array
+                        required:
+                          - matchExpressions
+                        type: object
+                      type: array
+                  type: object
+              required:
+                - storageClassName
+              type: object
+          type: object
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
--- a/config/crds/tenant.kubesphere.io_workspaces.yaml
+++ b/config/crds/tenant.kubesphere.io_workspaces.yaml
@@ -24,10 +24,14 @@ spec:
        description: Workspace is the Schema for the workspaces API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
--- a/config/crds/tenant.kubesphere.io_workspacetemplates.yaml
+++ b/config/crds/tenant.kubesphere.io_workspacetemplates.yaml
@@ -24,10 +24,14 @@ spec:
        description: WorkspaceTemplate is the Schema for the workspacetemplates API
        properties:
          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
            type: string
          kind:
-            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
            type: string
          metadata:
            type: object
@@ -59,21 +63,34 @@ spec:
              placement:
                properties:
                  clusterSelector:
-                    description: A label selector is a label query over a set of resources. The result of matchLabels and matchExpressions are ANDed. An empty label selector matches all objects. A null label selector matches no objects.
+                    description: A label selector is a label query over a set of resources.
+                      The result of matchLabels and matchExpressions are ANDed. An
+                      empty label selector matches all objects. A null label selector
+                      matches no objects.
                    properties:
                      matchExpressions:
-                        description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
+                        description: matchExpressions is a list of label selector
+                          requirements. The requirements are ANDed.
                        items:
-                          description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
+                          description: A label selector requirement is a selector
+                            that contains values, a key, and an operator that relates
+                            the key and values.
                          properties:
                            key:
-                              description: key is the label key that the selector applies to.
+                              description: key is the label key that the selector
+                                applies to.
                              type: string
                            operator:
-                              description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
+                              description: operator represents a key's relationship
+                                to a set of values. Valid operators are In, NotIn,
+                                Exists and DoesNotExist.
                              type: string
                            values:
-                              description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
+                              description: values is an array of string values. If
+                                the operator is In or NotIn, the values array must
+                                be non-empty. If the operator is Exists or DoesNotExist,
+                                the values array must be empty. This array is replaced
+                                during a strategic merge patch.
                              items:
                                type: string
                              type: array
@@ -85,7 +102,11 @@ spec:
                      matchLabels:
                        additionalProperties:
                          type: string
-                        description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
+                        description: matchLabels is a map of {key,value} pairs. A
+                          single {key,value} in the matchLabels map is equivalent
+                          to an element of matchExpressions, whose key field is "key",
+                          the operator is "In", and the values array contains only
+                          "value". The requirements are ANDed.
                        type: object
                    type: object
                  clusters:
--- a/config/gateway/templates/nginx-ingress.yaml
+++ b/config/gateway/templates/nginx-ingress.yaml
@@ -11,7 +11,7 @@ spec:
    {{- with .Values.controller.image }}
    {{- toYaml . | nindent 6 }}
    {{- end }}
-
+    watchIngressWithoutClass: true
    publishService:
      enabled: {{ eq .Values.service.type "LoadBalancer" }}

@@ -192,13 +192,7 @@ spec:
    # ref: https://github.com/kubernetes/ingress-nginx/issues/4735#issuecomment-551204903
    # Ideally, there should be no limits.
    # https://engineering.indeedblog.com/blog/2019/12/cpu-throttling-regression-fix/
-    resources:
-    #  limits:
-    #    cpu: 100m
-    #    memory: 90Mi
-      requests:
-        cpu: 100m
-        memory: 90Mi
+    resources: {{ toYaml .Values.deployment.resources |  nindent 6 }}

    # Mutually exclusive with keda autoscaling
    autoscaling:
--- a/config/gateway/values.yaml
+++ b/config/gateway/values.yaml
@@ -12,8 +12,9 @@ controller:
    namespace: ""  # defaults to .Release.Namespace
  image:
    repository: kubesphere/nginx-ingress-controller
-    tag: "v0.48.1"
+    tag: "v1.1.0"
    pullPolicy: IfNotPresent
+    digest: ""


 service:
@@ -25,4 +26,12 @@ service:
 deployment:
  annotations: {}
  replicas: 1
+  resources:
+#    limits:
+#      cpu: 100m
+#      memory: 90Mi
+    requests:
+      cpu: 100m
+      memory: 90Mi
+
  
--- a/config/ks-core/Chart.yaml
+++ b/config/ks-core/Chart.yaml
@@ -7,9 +7,9 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.1.1

 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
-appVersion: "v3.1.0"
+appVersion: "v3.2.1"
--- a/config/ks-core/templates/ks-apiserver.yml
+++ b/config/ks-core/templates/ks-apiserver.yml
@@ -41,8 +41,6 @@ spec:
        resources:
          {{- toYaml .Values.apiserver.resources | nindent 12 }}
        volumeMounts:
-        - mountPath: /etc/kubesphere/ingress-controller
-          name: ks-router-config
        - mountPath: /etc/kubesphere/
          name: kubesphere-config
        - mountPath: /etc/localtime
@@ -77,10 +75,6 @@ spec:
        {{- toYaml . | nindent 8 }}
      {{- end }}
      volumes:
-      - configMap:
-          defaultMode: 420
-          name: ks-router-config
-        name: ks-router-config
      - configMap:
          defaultMode: 420
          name: kubesphere-config
--- a/config/ks-core/templates/ks-router-cm.yaml
+++ b/config/ks-core/templates/ks-router-cm.yaml
@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: ks-router-config
-data:
-  ingress-controller-svc.yaml: |+
-{{- include "ingress-controller-svc.yaml" . }}
-  ingress-controller.yaml: |
-{{- include "ingress-controller.yaml" . }}
--- a/config/ks-core/templates/ks-router-config.tpl
+++ b/config/ks-core/templates/ks-router-config.tpl
@@ -1,96 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-
-{{- define "ingress-controller.yaml" }}
-    apiVersion: apps/v1
-    kind: Deployment
-    metadata:
-      name: ks-router
-    spec:
-      replicas: 1
-      selector:
-        matchLabels:
-          app: kubesphere
-          component: ks-router
-          tier: backend
-      template:
-        metadata:
-          labels:
-            app: kubesphere
-            component: ks-router
-            tier: backend
-          annotations:
-            prometheus.io/port: '10254'
-            prometheus.io/scrape: 'true'
-        spec:
-          serviceAccountName: kubesphere-router-serviceaccount
-          containers:
-            - name: nginx-ingress-controller
-              image: {{ .Values.image.nginx_ingress_controller_repo }}:{{ .Values.image.nginx_ingress_controller_tag | default .Chart.AppVersion}}
-              args:
-                - /nginx-ingress-controller
-                - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
-                - --annotations-prefix=nginx.ingress.kubernetes.io
-                - --update-status
-                - --update-status-on-shutdown
-              env:
-                - name: POD_NAME
-                  valueFrom:
-                    fieldRef:
-                      fieldPath: metadata.name
-                - name: POD_NAMESPACE
-                  valueFrom:
-                    fieldRef:
-                      fieldPath: metadata.namespace
-              ports:
-              - name: http
-                containerPort: 80
-              - name: https
-                containerPort: 443
-              livenessProbe:
-                failureThreshold: 3
-                httpGet:
-                  path: /healthz
-                  port: 10254
-                  scheme: HTTP
-                initialDelaySeconds: 10
-                periodSeconds: 10
-                successThreshold: 1
-                timeoutSeconds: 1
-              readinessProbe:
-                failureThreshold: 3
-                httpGet:
-                  path: /healthz
-                  port: 10254
-                  scheme: HTTP
-                periodSeconds: 10
-                successThreshold: 1
-                timeoutSeconds: 1
-              securityContext:
-                runAsNonRoot: false
-{{- end }}
-
-{{- define "ingress-controller-svc.yaml" }}
-    apiVersion: v1
-    kind: Service
-    metadata:
-      name: kubesphere-router-gateway
-      labels:
-        app: kubesphere
-        component: ks-router
-        tier: backend
-    spec:
-      selector:
-        app: kubesphere
-        component: ks-router
-        tier: backend
-      type: LoadBalancer
-      ports:
-        - name: http
-          protocol: TCP
-          port: 80
-          targetPort: 80
-        - name: https
-          protocol: TCP
-          port: 443
-          targetPort: 443
-{{- end }}
--- a/config/ks-core/templates/webhook.yaml
+++ b/config/ks-core/templates/webhook.yaml
@@ -120,3 +120,37 @@ webhooks:
          - pods
        scope: '*'
    sideEffects: None
+
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: storageclass-accessor.storage.kubesphere.io
+webhooks:
+  - admissionReviewVersions:
+      - v1beta1
+    clientConfig:
+      caBundle: {{ b64enc $ca.Cert | quote }}
+      service:
+        name: ks-controller-manager
+        namespace: {{ .Release.Namespace }}
+        path: /persistentvolumeclaims
+        port: 443
+    failurePolicy: Ignore
+    matchPolicy: Exact
+    name: storageclass-accessor.storage.kubesphere.io
+    namespaceSelector: {}
+    objectSelector: {}
+    rules:
+      - apiGroups:
+          - '*'
+        apiVersions:
+          - '*'
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - persistentvolumeclaims
+        scope: '*'
+    sideEffects: None
--- a/config/ks-core/values.yaml
+++ b/config/ks-core/values.yaml
@@ -16,7 +16,7 @@ image:
  ks_kubectl_tag: ""

  nginx_ingress_controller_repo: kubesphere/nginx-ingress-controller
-  nginx_ingress_controller_tag: "v0.35.0"
+  nginx_ingress_controller_tag: "v1.1.0"
  defaultbackend_repo: "mirrorgooglecontainers/defaultbackend-amd64"
  defaultbackend_tag: "1.4"

--- a/doc.go
+++ b/doc.go
@@ -1,6 +0,0 @@
-// Copyright 2017 The OpenPitrix Authors. All rights reserved.
-// Use of this source code is governed by a Apache license
-// that can be found in the LICENSE file.
-
-// Package openpitrix provides the best Paas and Iaas platform.
-package kubesphere
--- a/docs/images/kubesphere-icon.gif
+++ b/docs/images/kubesphere-icon.gif
--- a/go.mod
+++ b/go.mod
@@ -36,6 +36,7 @@ require (
 	github.com/evanphx/json-patch v4.11.0+incompatible
 	github.com/fatih/structs v1.1.0
 	github.com/form3tech-oss/jwt-go v3.2.2+incompatible
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/garyburd/redigo v1.6.0 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-ldap/ldap v3.0.3+incompatible
@@ -45,12 +46,11 @@ require (
 	github.com/go-openapi/strfmt v0.19.5
 	github.com/go-openapi/validate v0.19.8
 	github.com/go-redis/redis v6.15.2+incompatible
-	github.com/go-sql-driver/mysql v1.5.0
-	github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6
 	github.com/gofrs/uuid v3.2.0+incompatible // indirect
 	github.com/golang/example v0.0.0-20170904185048-46695d81d1fa
 	github.com/google/go-cmp v0.5.6
 	github.com/google/go-containerregistry v0.6.0
+	github.com/google/gops v0.3.23
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/handlers v1.4.0 // indirect
 	github.com/gorilla/websocket v1.4.2
@@ -60,7 +60,9 @@ require (
 	github.com/jszwec/csvutil v1.5.0
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
+	github.com/kubesphere/pvc-autoresizer v0.1.1
 	github.com/kubesphere/sonargo v0.0.2
+	github.com/kubesphere/storageclass-accessor v0.2.0
 	github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
 	github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
 	github.com/mattn/go-runewidth v0.0.4 // indirect
@@ -80,11 +82,13 @@ require (
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.26.0
 	github.com/prometheus/prometheus v1.8.2-0.20200907175821-8219b442c864
+	github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7 // indirect
+	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
 	github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009
 	github.com/speps/go-hashids v2.0.0+incompatible
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5
-	github.com/spf13/viper v1.4.0
+	github.com/spf13/viper v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/xanzy/ssh-agent v0.2.1 // indirect
 	github.com/xenolf/lego v0.3.2-0.20160613233155-a9d8cec0e656 // indirect
@@ -107,16 +111,16 @@ require (
 	istio.io/api v0.0.0-20201113182140-d4b7e3fc2b44
 	istio.io/client-go v0.0.0-20201113183938-0734e976e785
 	istio.io/gogo-genproto v0.0.0-20201113182723-5b8563d8a012 // indirect
-	k8s.io/api v0.21.4
+	k8s.io/api v0.22.1
 	k8s.io/apiextensions-apiserver v0.21.4
-	k8s.io/apimachinery v0.21.4
+	k8s.io/apimachinery v0.22.1
 	k8s.io/apiserver v0.21.2
 	k8s.io/cli-runtime v0.21.2
 	k8s.io/client-go v12.0.0+incompatible
 	k8s.io/code-generator v0.21.2
 	k8s.io/component-base v0.21.4
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.8.0
+	k8s.io/klog/v2 v2.9.0
 	k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e
 	k8s.io/kubectl v0.21.2
 	k8s.io/metrics v0.21.2
@@ -126,7 +130,7 @@ require (
 	kubesphere.io/monitoring-dashboard v0.2.2
 	rsc.io/letsencrypt v0.0.1 // indirect
 	sigs.k8s.io/application v0.8.4-0.20201016185654-c8e2959e57a0
-	sigs.k8s.io/controller-runtime v0.9.8-0.20211019125639-aa2b3e68a52d
+	sigs.k8s.io/controller-runtime v0.10.0
 	sigs.k8s.io/controller-tools v0.6.2
 	sigs.k8s.io/kubefed v0.8.1
 	sigs.k8s.io/kustomize/api v0.8.8
@@ -257,6 +261,7 @@ replace (
 	github.com/coreos/pkg => github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
 	github.com/cortexproject/cortex => github.com/cortexproject/cortex v1.3.1-0.20200901115931-255ff3306960
 	github.com/cpuguy83/go-md2man => github.com/cpuguy83/go-md2man v1.0.10
+	github.com/cpuguy83/go-md2man/v2 => github.com/cpuguy83/go-md2man/v2 v2.0.0
 	github.com/creack/pty => github.com/creack/pty v1.1.7
 	github.com/cyphar/filepath-securejoin => github.com/cyphar/filepath-securejoin v0.2.2
 	github.com/cznic/b => github.com/cznic/b v0.0.0-20180115125044-35e9bbe41f07
@@ -374,7 +379,6 @@ replace (
 	github.com/gobwas/pool => github.com/gobwas/pool v0.2.0
 	github.com/gobwas/ws => github.com/gobwas/ws v1.0.2
 	github.com/gocql/gocql => github.com/gocql/gocql v0.0.0-20200526081602-cd04bd7f22a7
-	github.com/gocraft/dbr => github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6
 	github.com/godbus/dbus => github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968
 	github.com/godror/godror => github.com/godror/godror v0.13.3
 	github.com/gofrs/flock => github.com/gofrs/flock v0.7.1
@@ -500,9 +504,10 @@ replace (
 	github.com/kr/pty => github.com/kr/pty v1.1.5
 	github.com/kr/text => github.com/kr/text v0.1.0
 	github.com/kshvakov/clickhouse => github.com/kshvakov/clickhouse v1.3.5
-	github.com/kubernetes-csi/external-snapshotter/client/v3 => github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0
 	github.com/kubernetes-csi/external-snapshotter/client/v4 => github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0
+	github.com/kubesphere/pvc-autoresizer => github.com/kubesphere/pvc-autoresizer v0.1.1
 	github.com/kubesphere/sonargo => github.com/kubesphere/sonargo v0.0.2
+	github.com/kubesphere/storageclass-accessor => github.com/kubesphere/storageclass-accessor v0.2.0
 	github.com/kylelemons/go-gypsy => github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28
 	github.com/kylelemons/godebug => github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb
 	github.com/lann/builder => github.com/lann/builder v0.0.0-20180802200727-47ae307949d0
@@ -651,6 +656,7 @@ replace (
 	github.com/sergi/go-diff => github.com/sergi/go-diff v1.0.0
 	github.com/shopspring/decimal => github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
 	github.com/shurcooL/httpfs => github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
+	github.com/shurcooL/sanitized_anchor_name => github.com/shurcooL/sanitized_anchor_name v1.0.0
 	github.com/shurcooL/vfsgen => github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd
 	github.com/siebenmann/go-kstat => github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745
 	github.com/sirupsen/logrus => github.com/sirupsen/logrus v1.4.2
@@ -664,7 +670,7 @@ replace (
 	github.com/speps/go-hashids => github.com/speps/go-hashids v2.0.0+incompatible
 	github.com/spf13/afero => github.com/spf13/afero v1.2.2
 	github.com/spf13/cast => github.com/spf13/cast v1.3.0
-	github.com/spf13/cobra => github.com/spf13/cobra v0.0.5
+	github.com/spf13/cobra => github.com/spf13/cobra v1.2.1
 	github.com/spf13/jwalterweatherman => github.com/spf13/jwalterweatherman v1.0.0
 	github.com/spf13/pflag => github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper => github.com/spf13/viper v1.4.0
--- a/go.sum
+++ b/go.sum
@@ -59,6 +59,7 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/O
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8=
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
 github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
@@ -165,6 +166,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cortexproject/cortex v1.3.1-0.20200901115931-255ff3306960/go.mod h1:ub8BpRZrRa02BOM8NJTnI2YklxW/mGhEkJDrhsDfcfg=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg=
@@ -286,6 +288,8 @@ github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
 github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
 github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-ole/go-ole v1.2.6-0.20210915003542-8b1f7f90f6b1/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-openapi/analysis v0.19.10 h1:5BHISBAXOc/aJK25irLZnx2D3s6WyYaY9D4gmuz9fdE=
 github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ=
 github.com/go-openapi/errors v0.19.4 h1:fSGwO1tSYHFu70NKaWJt5Qh0qoBRtCm/mXS1yhf+0W0=
@@ -347,8 +351,6 @@ github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm
 github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo=
 github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/gocql/gocql v0.0.0-20200526081602-cd04bd7f22a7/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
-github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 h1:kumyNm8Vr8cbVm/aLQYTbDE3SKCbbn5HEVoDp/Dyyfc=
-github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6/go.mod h1:K/9g3pPouf13kP5K7pdriQEJAy272R9yXuWuDIEWJTM=
 github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg=
 github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
@@ -389,6 +391,8 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
 github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gops v0.3.23 h1:OjsHRINl5FiIyTc8jivIg4UN0GY6Nh32SL8KRbl8GQo=
+github.com/google/gops v0.3.23/go.mod h1:7diIdLsqpCihPSX3fQagksT/Ku/y4RL9LHTlKyEUDl8=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20200417002340-c6e0a841f49a/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@@ -502,6 +506,7 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
 github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/keybase/go-ps v0.0.0-20190827175125-91aafc93ba19/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
@@ -523,8 +528,12 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
+github.com/kubesphere/pvc-autoresizer v0.1.1 h1:Q0VrvLfTiE1f38EvmFpJdBevwN21X7BrgQgKrssqKQw=
+github.com/kubesphere/pvc-autoresizer v0.1.1/go.mod h1:88qz9L1Ov2bvw7L/i5mUT8g5DvBwRCZ60JA2d1WLgB0=
 github.com/kubesphere/sonargo v0.0.2 h1:hsSRE3sv3mkPcUAeSABdp7rtfcNW2zzeHXzFa01CTkU=
 github.com/kubesphere/sonargo v0.0.2/go.mod h1:ww8n9ANlDXhX5PBZ18iaRnCgEkXN0GMml3/KZXOZ11w=
+github.com/kubesphere/storageclass-accessor v0.2.0 h1:rnzKafhneo8160dh6REm3z1yAEaQWz1x/Lwi3QFVLWE=
+github.com/kubesphere/storageclass-accessor v0.2.0/go.mod h1:jqZ3tCiw09yOiPkZ3rDmf6QIpbZJx55McnyRaS0ayCY=
 github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c=
 github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 h1:SOEGU9fKiNWd/HOJuq6+3iTQz8KNCLtVX6idSoTLdUw=
@@ -738,8 +747,12 @@ github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfP
 github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/gopsutil/v3 v3.21.9/go.mod h1:YWp/H8Qs5fVmf17v7JNZzA0mPJ+mS2e9JdiUF9LlKzQ=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
 github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw=
 github.com/siebenmann/go-kstat v0.0.0-20160321171754-d34789b79745/go.mod h1:G81aIFAMS9ECrwBYR9YxhlPjWgrItd+Kje78O6+uqm8=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
@@ -760,8 +773,8 @@ github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
+github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -780,6 +793,8 @@ github.com/thanos-io/thanos v0.13.1-0.20200910143741-e0b7f7b32e9c/go.mod h1:1Ize
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
+github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
@@ -987,6 +1002,7 @@ k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 kubesphere.io/monitoring-dashboard v0.2.2 h1:aniATtXLgRAAvKOjd2UxWWHMh4/T7a0HoQ9bd+/bGcA=
 kubesphere.io/monitoring-dashboard v0.2.2/go.mod h1:ksDjmOuoN0C0GuYp0s5X3186cPgk2asLUaO1WlEKISY=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
 rsc.io/letsencrypt v0.0.1 h1:DV0d09Ne9E7UUa9ZqWktZ9L2VmybgTgfq7xlfFR/bbU=
 rsc.io/letsencrypt v0.0.1/go.mod h1:buyQKZ6IXrRnB7TdkHP0RyEybLx18HHyOSoTyoOLqNY=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
--- a/hack/generate_manifests.sh
+++ b/hack/generate_manifests.sh
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -ex
+
+CRD_OPTIONS="$1"
+PKGS="$2"
+IFS=" " read -r -a PKGS <<< "${PKGS}"
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+cd "${KUBE_ROOT}" || exit
+
+for PKG in "${PKGS[@]}"; do
+  echo "Generating manifests for ${PKG}"
+  if [[ "$PKG" =~ /\*$ ]]; then
+    PKG=${PKG%??}
+    DIR=$(go list -e -test=false -export=false -deps=false -find=false -tags ignore_autogenerated -f "{{.Dir}}" "kubesphere.io/api/${PKG}")
+    # shellcheck disable=SC2010
+    ls -1 -F "${DIR}" | grep '/$' | xargs -n 1 -I{} go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/"${PKG}"/{} rbac:roleName=controller-perms "${CRD_OPTIONS}" output:crd:artifacts:config=config/crds
+  else
+    go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go object:headerFile=./hack/boilerplate.go.txt paths=kubesphere.io/api/"${PKG}" rbac:roleName=controller-perms "${CRD_OPTIONS}" output:crd:artifacts:config=config/crds
+  fi
+done
--- a/hack/update-gofmt.sh
+++ b/hack/update-gofmt.sh
@@ -39,6 +39,7 @@ find_files() {
        -o -wholename '*/third_party/*' \
        -o -wholename '*/vendor/*' \
        -o -wholename './staging/src/kubesphere.io/client-go/*vendor/*' \
+        -o -wholename './staging/src/kubesphere.io/api/*/zz_generated.deepcopy.go' \
      \) -prune \
    \) -name '*.go'
 }
--- a/hack/update-licenses.sh
+++ b/hack/update-licenses.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright 2022 The KubeSphere Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+if ! command -v license-eye &> /dev/null
+then
+    # Ensure that we find the binaries we build before anything else.
+    export GOBIN="${KUBE_OUTPUT_BINPATH}"
+    PATH="${GOBIN}:${PATH}"
+
+    # Explicitly opt into go modules, even though we're inside a GOPATH directory
+    export GO111MODULE=on
+    # Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+    export GOFLAGS=
+
+    # Install skywalking-eyes
+    echo 'installing skywalking-eyes '
+    pushd "${KUBE_ROOT}/hack/tools" >/dev/null
+      go install github.com/apache/skywalking-eyes/cmd/license-eye@v0.2.0
+    popd >/dev/null
+fi
+
+cd "${KUBE_ROOT}"
+
+echo 'running skywalking-eyes fix '
+license-eye header fix
+exit 0
--- a/hack/verify-all.sh
+++ b/hack/verify-all.sh
@@ -28,6 +28,7 @@ EXCLUDED_PATTERNS=(
  "verify-*-dockerized.sh"       # Don't run any scripts that intended to be run dockerized
  "verify-govet-levee.sh"        # Do not run levee analysis by default while KEP-1933 implementation is in alpha.
  "verify-golangci-lint.sh"      # Experimental - intended to be run by hand periodically
+  "verify-licenses.sh"
  )

 while IFS='' read -r line; do EXCLUDED_CHECKS+=("$line"); done < <(ls "${EXCLUDED_PATTERNS[@]/#/${KUBE_ROOT}/hack/}" 2>/dev/null || true)
--- a/hack/verify-gofmt.sh
+++ b/hack/verify-gofmt.sh
@@ -44,6 +44,7 @@ find_files() {
        -o -wholename '*/third_party/*' \
        -o -wholename '*/vendor/*' \
        -o -wholename './staging/src/kubesphere.io/client-go/*vendor/*' \
+        -o -wholename './staging/src/kubesphere.io/api/*/zz_generated.deepcopy.go' \
        -o -wholename '*/bindata.go' \
      \) -prune \
    \) -name '*.go'
--- a/hack/verify-licenses.sh
+++ b/hack/verify-licenses.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# Copyright 2022 The KubeSphere Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+if ! command -v license-eye &> /dev/null
+then
+    # Ensure that we find the binaries we build before anything else.
+    export GOBIN="${KUBE_OUTPUT_BINPATH}"
+    PATH="${GOBIN}:${PATH}"
+
+    # Explicitly opt into go modules, even though we're inside a GOPATH directory
+    export GO111MODULE=on
+    # Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+    export GOFLAGS=
+
+    # Install skywalking-eyes
+    echo 'installing skywalking-eyes '
+    pushd "${KUBE_ROOT}/hack/tools" >/dev/null
+      go install github.com/apache/skywalking-eyes/cmd/license-eye@v0.2.0
+    popd >/dev/null
+fi
+
+cd "${KUBE_ROOT}"
+
+echo 'running skywalking-eyes check '
+license-eye header check
+exit 0
--- a/pkg/api/cluster/v1alpha1/types.go
+++ b/pkg/api/cluster/v1alpha1/types.go
@@ -0,0 +1,19 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+package v1alpha1
+
+type UpdateClusterRequest struct {
+	KubeConfig []byte `json:"kubeconfig"`
+}
--- a/pkg/api/metering/v1alpha1/types.go
+++ b/pkg/api/metering/v1alpha1/types.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 package v1alpha1

 import (
--- a/pkg/apis/addtoscheme_dashboard_v1alpha1.go
+++ b/pkg/apis/addtoscheme_dashboard_v1alpha1.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 package apis

 import monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
--- a/pkg/apis/addtoscheme_dashboard_v1alpha2.go
+++ b/pkg/apis/addtoscheme_dashboard_v1alpha2.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 package apis

 import monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
--- a/pkg/apiserver/apiserver.go
+++ b/pkg/apiserver/apiserver.go
@@ -22,35 +22,29 @@ import (
 	"fmt"
 	"net/http"
 	rt "runtime"
+	"strconv"
+	"sync"
 	"time"

-	"kubesphere.io/kubesphere/pkg/utils/iputil"
-
-	"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
-
-	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
-
-	"kubesphere.io/api/notification/v2beta1"
-
-	openpitrixv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v2alpha1"
-
-	"strconv"
-
 	"github.com/emicklei/go-restful"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	unionauth "k8s.io/apiserver/pkg/authentication/request/union"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/klog"
-	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
-	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
-
 	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
 	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
 	notificationv2beta1 "kubesphere.io/api/notification/v2beta1"
 	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
 	typesv1beta1 "kubesphere.io/api/types/v1beta1"
+	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"

 	audit "kubesphere.io/kubesphere/pkg/apiserver/auditing"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/basic"
@@ -58,6 +52,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/anonymous"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/bearertoken"
+	"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
+	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/path"
@@ -72,8 +68,9 @@ import (
 	alertingv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v2alpha1"
 	clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
 	configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
-	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
-	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
+	"kubesphere.io/kubesphere/pkg/kapis/crd"
+	kapisdevops "kubesphere.io/kubesphere/pkg/kapis/devops"
+	edgeruntimev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/edgeruntime/v1alpha1"
 	gatewayv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha1"
 	iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
 	kubeedgev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/kubeedge/v1alpha1"
@@ -85,17 +82,20 @@ import (
 	notificationkapisv2beta2 "kubesphere.io/kubesphere/pkg/kapis/notification/v2beta2"
 	"kubesphere.io/kubesphere/pkg/kapis/oauth"
 	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
+	openpitrixv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v2alpha1"
 	operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
 	resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2"
 	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha3"
 	servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/servicemesh/metrics/v1alpha2"
 	tenantv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha2"
+	tenantv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha3"
 	terminalv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/terminal/v1alpha2"
 	"kubesphere.io/kubesphere/pkg/kapis/version"
 	"kubesphere.io/kubesphere/pkg/models/auth"
 	"kubesphere.io/kubesphere/pkg/models/iam/am"
 	"kubesphere.io/kubesphere/pkg/models/iam/group"
 	"kubesphere.io/kubesphere/pkg/models/iam/im"
+	"kubesphere.io/kubesphere/pkg/models/openpitrix"
 	"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/loginrecord"
 	"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/user"
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
@@ -108,9 +108,13 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
+	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
+	"kubesphere.io/kubesphere/pkg/utils/iputil"
 	"kubesphere.io/kubesphere/pkg/utils/metrics"
 )

+var initMetrics sync.Once
+
 type APIServer struct {
 	// number of kubesphere apiserver
 	ServerCount int
@@ -159,6 +163,10 @@ type APIServer struct {

 	// controller-runtime client
 	RuntimeClient runtimeclient.Client
+
+	ClusterClient clusterclient.ClusterClients
+
+	OpenpitrixClient openpitrix.Interface
 }

 func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
@@ -169,7 +177,8 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 		logStackOnRecover(panicReason, httpWriter)
 	})

-	s.installKubeSphereAPIs()
+	s.installKubeSphereAPIs(stopCh)
+	s.installCRDAPIs()
 	s.installMetricsAPI()
 	s.container.Filter(monitorRequest)

@@ -196,14 +205,14 @@ func monitorRequest(r *restful.Request, response *restful.Response, chain *restf
 }

 func (s *APIServer) installMetricsAPI() {
-	registerMetrics()
+	initMetrics.Do(registerMetrics)
 	metrics.Defaults.Install(s.container)
 }

 // Install all kubesphere api groups
 // Installation happens before all informers start to cache objects, so
 //   any attempt to list objects using listers will get empty results.
-func (s *APIServer) installKubeSphereAPIs() {
+func (s *APIServer) installKubeSphereAPIs(stopCh <-chan struct{}) {
 	imOperator := im.NewOperator(s.KubernetesClient.KubeSphere(),
 		user.New(s.InformerFactory.KubeSphereSharedInformerFactory(),
 			s.InformerFactory.KubernetesSharedInformerFactory()),
@@ -217,17 +226,20 @@ func (s *APIServer) installKubeSphereAPIs() {

 	urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
 	urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory, s.RuntimeCache))
-	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.MetricsClient, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions))
-	urlruntime.Must(meteringv1alpha1.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.RuntimeCache, s.Config.MeteringOptions, nil))
-	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions))
+	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.MetricsClient, s.InformerFactory, s.OpenpitrixClient, s.RuntimeClient))
+	urlruntime.Must(meteringv1alpha1.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient, s.RuntimeClient))
+	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions, s.OpenpitrixClient))
 	urlruntime.Must(openpitrixv2alpha1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.KubeSphere(), s.Config.OpenPitrixOptions))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory,
 		s.KubernetesClient.Master()))
 	urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
-		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, rbacAuthorizer, s.MonitoringClient, s.RuntimeCache, s.Config.MeteringOptions))
-	urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), rbacAuthorizer, s.KubernetesClient.Config()))
+		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, imOperator, rbacAuthorizer, s.MonitoringClient, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient))
+	urlruntime.Must(tenantv1alpha3.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
+		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient, s.AuditingClient, amOperator, imOperator, rbacAuthorizer, s.MonitoringClient, s.RuntimeCache, s.Config.MeteringOptions, s.OpenpitrixClient))
+	urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), rbacAuthorizer, s.KubernetesClient.Config(), s.Config.TerminalOptions))
 	urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
+		s.KubernetesClient.KubeSphere(),
 		s.InformerFactory.KubernetesSharedInformerFactory(),
 		s.InformerFactory.KubeSphereSharedInformerFactory(),
 		s.Config.MultiClusterOptions.ProxyPublishService,
@@ -246,20 +258,28 @@ func (s *APIServer) installKubeSphereAPIs() {
 		s.Config.AuthenticationOptions))
 	urlruntime.Must(servicemeshv1alpha2.AddToContainer(s.Config.ServiceMeshOptions, s.container, s.KubernetesClient.Kubernetes(), s.CacheClient))
 	urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
-	urlruntime.Must(devopsv1alpha2.AddToContainer(s.container, s.Config.DevopsOptions.Endpoint))
-	urlruntime.Must(devopsv1alpha3.AddToContainer(s.container, s.Config.DevopsOptions.Endpoint))
+	urlruntime.Must(kapisdevops.AddToContainer(s.container, s.Config.DevopsOptions.Endpoint))
 	urlruntime.Must(notificationv1.AddToContainer(s.container, s.Config.NotificationOptions.Endpoint))
 	urlruntime.Must(alertingv1.AddToContainer(s.container, s.Config.AlertingOptions.Endpoint))
 	urlruntime.Must(alertingv2alpha1.AddToContainer(s.container, s.InformerFactory,
 		s.KubernetesClient.Prometheus(), s.AlertingClient, s.Config.AlertingOptions))
-	urlruntime.Must(version.AddToContainer(s.container, s.KubernetesClient.Discovery()))
+	urlruntime.Must(version.AddToContainer(s.container, s.KubernetesClient.Kubernetes().Discovery()))
 	urlruntime.Must(kubeedgev1alpha1.AddToContainer(s.container, s.Config.KubeEdgeOptions.Endpoint))
+	urlruntime.Must(edgeruntimev1alpha1.AddToContainer(s.container, s.Config.EdgeRuntimeOptions.Endpoint))
 	urlruntime.Must(notificationkapisv2beta1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
 		s.KubernetesClient.KubeSphere()))
 	urlruntime.Must(notificationkapisv2beta2.AddToContainer(s.container, s.Config.NotificationOptions))
 	urlruntime.Must(gatewayv1alpha1.AddToContainer(s.container, s.Config.GatewayOptions, s.RuntimeCache, s.RuntimeClient, s.InformerFactory, s.KubernetesClient.Kubernetes(), s.LoggingClient))
 }

+// installCRDAPIs Install CRDs to the KAPIs with List and Get options
+func (s *APIServer) installCRDAPIs() {
+	crds := &extv1.CustomResourceDefinitionList{}
+	// TODO Maybe we need a better label name
+	urlruntime.Must(s.RuntimeClient.List(context.TODO(), crds, runtimeclient.MatchingLabels{"kubesphere.io/resource-served": "true"}))
+	urlruntime.Must(crd.AddToContainer(s.container, s.RuntimeClient, s.RuntimeCache, crds))
+}
+
 func (s *APIServer) Run(ctx context.Context) (err error) {

 	err = s.waitForResourceSync(ctx)
@@ -298,8 +318,8 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 			tenantv1alpha2.Resource(clusterv1alpha1.ResourcesPluralCluster),
 			clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralCluster),
 			resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralCluster),
-			notificationv2beta1.Resource(v2beta1.ResourcesPluralConfig),
-			notificationv2beta1.Resource(v2beta1.ResourcesPluralReceiver),
+			notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralConfig),
+			notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralReceiver),
 		},
 	}

@@ -329,7 +349,7 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {

 	handler = filters.WithAuthorization(handler, authorizers)
 	if s.Config.MultiClusterOptions.Enable {
-		clusterDispatcher := dispatch.NewClusterDispatch(s.InformerFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters())
+		clusterDispatcher := dispatch.NewClusterDispatch(s.ClusterClient)
 		handler = filters.WithMultipleClusterDispatcher(handler, clusterDispatcher)
 	}

@@ -352,215 +372,233 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 	s.Server.Handler = handler
 }

+func isResourceExists(apiResources []v1.APIResource, resource schema.GroupVersionResource) bool {
+	for _, apiResource := range apiResources {
+		if apiResource.Name == resource.Resource {
+			return true
+		}
+	}
+	return false
+}
+
+type informerForResourceFunc func(resource schema.GroupVersionResource) (interface{}, error)
+
+func waitForCacheSync(discoveryClient discovery.DiscoveryInterface, sharedInformerFactory informers.GenericInformerFactory, informerForResourceFunc informerForResourceFunc, GVRs map[schema.GroupVersion][]string, stopCh <-chan struct{}) error {
+	for groupVersion, resourceNames := range GVRs {
+		var apiResourceList *v1.APIResourceList
+		var err error
+		err = retry.OnError(retry.DefaultRetry, func(err error) bool {
+			return !errors.IsNotFound(err)
+		}, func() error {
+			apiResourceList, err = discoveryClient.ServerResourcesForGroupVersion(groupVersion.String())
+			return err
+		})
+		if err != nil {
+			return fmt.Errorf("failed to fetch group version resources %s: %s", groupVersion, err)
+		}
+		for _, resourceName := range resourceNames {
+			groupVersionResource := groupVersion.WithResource(resourceName)
+			if !isResourceExists(apiResourceList.APIResources, groupVersionResource) {
+				klog.Warningf("resource %s not exists in the cluster", groupVersionResource)
+			} else {
+				// reflect.ValueOf(sharedInformerFactory).MethodByName("ForResource").Call([]reflect.Value{reflect.ValueOf(groupVersionResource)})
+				if _, err = informerForResourceFunc(groupVersionResource); err != nil {
+					return fmt.Errorf("failed to create informer for %s: %s", groupVersionResource, err)
+				}
+			}
+		}
+	}
+	sharedInformerFactory.Start(stopCh)
+	sharedInformerFactory.WaitForCacheSync(stopCh)
+	return nil
+}
+
 func (s *APIServer) waitForResourceSync(ctx context.Context) error {
 	klog.V(0).Info("Start cache objects")

 	stopCh := ctx.Done()
+	// resources we have to create informer first
+	k8sGVRs := map[schema.GroupVersion][]string{
+		{Group: "", Version: "v1"}: {
+			"namespaces",
+			"nodes",
+			"resourcequotas",
+			"pods",
+			"services",
+			"persistentvolumeclaims",
+			"persistentvolumes",
+			"secrets",
+			"configmaps",
+			"serviceaccounts",
+		},
+		{Group: "rbac.authorization.k8s.io", Version: "v1"}: {
+			"roles",
+			"rolebindings",
+			"clusterroles",
+			"clusterrolebindings",
+		},
+		{Group: "apps", Version: "v1"}: {
+			"deployments",
+			"daemonsets",
+			"replicasets",
+			"statefulsets",
+			"controllerrevisions",
+		},
+		{Group: "storage.k8s.io", Version: "v1"}: {
+			"storageclasses",
+		},
+		{Group: "batch", Version: "v1"}: {
+			"jobs",
+		},
+		{Group: "batch", Version: "v1beta1"}: {
+			"cronjobs",
+		},
+		{Group: "networking.k8s.io", Version: "v1"}: {
+			"ingresses",
+			"networkpolicies",
+		},
+		{Group: "autoscaling", Version: "v2beta2"}: {
+			"horizontalpodautoscalers",
+		},
+	}

-	discoveryClient := s.KubernetesClient.Kubernetes().Discovery()
-	_, apiResourcesList, err := discoveryClient.ServerGroupsAndResources()
-	if err != nil {
+	if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
+		s.InformerFactory.KubernetesSharedInformerFactory(),
+		func(resource schema.GroupVersionResource) (interface{}, error) {
+			return s.InformerFactory.KubernetesSharedInformerFactory().ForResource(resource)
+		},
+		k8sGVRs, stopCh); err != nil {
 		return err
 	}

-	isResourceExists := func(resource schema.GroupVersionResource) bool {
-		for _, apiResource := range apiResourcesList {
-			if apiResource.GroupVersion == resource.GroupVersion().String() {
-				for _, rsc := range apiResource.APIResources {
-					if rsc.Name == resource.Resource {
-						return true
-					}
-				}
-			}
-		}
-		return false
-	}
-
-	// resources we have to create informer first
-	k8sGVRs := []schema.GroupVersionResource{
-		{Group: "", Version: "v1", Resource: "namespaces"},
-		{Group: "", Version: "v1", Resource: "nodes"},
-		{Group: "", Version: "v1", Resource: "resourcequotas"},
-		{Group: "", Version: "v1", Resource: "pods"},
-		{Group: "", Version: "v1", Resource: "services"},
-		{Group: "", Version: "v1", Resource: "persistentvolumeclaims"},
-		{Group: "", Version: "v1", Resource: "persistentvolumes"},
-		{Group: "", Version: "v1", Resource: "secrets"},
-		{Group: "", Version: "v1", Resource: "configmaps"},
-		{Group: "", Version: "v1", Resource: "serviceaccounts"},
-
-		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"},
-		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"},
-		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"},
-		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"},
-		{Group: "apps", Version: "v1", Resource: "deployments"},
-		{Group: "apps", Version: "v1", Resource: "daemonsets"},
-		{Group: "apps", Version: "v1", Resource: "replicasets"},
-		{Group: "apps", Version: "v1", Resource: "statefulsets"},
-		{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
-		{Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"},
-		{Group: "batch", Version: "v1", Resource: "jobs"},
-		{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
-		{Group: "networking.k8s.io", Version: "v1", Resource: "ingresses"},
-		{Group: "autoscaling", Version: "v2beta2", Resource: "horizontalpodautoscalers"},
-		{Group: "networking.k8s.io", Version: "v1", Resource: "networkpolicies"},
-	}
-
-	for _, gvr := range k8sGVRs {
-		if !isResourceExists(gvr) {
-			klog.Warningf("resource %s not exists in the cluster", gvr)
-		} else {
-			_, err := s.InformerFactory.KubernetesSharedInformerFactory().ForResource(gvr)
-			if err != nil {
-				klog.Errorf("cannot create informer for %s", gvr)
-				return err
-			}
-		}
-	}
-
-	s.InformerFactory.KubernetesSharedInformerFactory().Start(stopCh)
-	s.InformerFactory.KubernetesSharedInformerFactory().WaitForCacheSync(stopCh)
-
-	ksInformerFactory := s.InformerFactory.KubeSphereSharedInformerFactory()
-
-	ksGVRs := []schema.GroupVersionResource{
-		{Group: "tenant.kubesphere.io", Version: "v1alpha1", Resource: "workspaces"},
-		{Group: "tenant.kubesphere.io", Version: "v1alpha2", Resource: "workspacetemplates"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "users"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "globalroles"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "globalrolebindings"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "groups"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "groupbindings"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "workspaceroles"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "workspacerolebindings"},
-		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "loginrecords"},
-		{Group: "cluster.kubesphere.io", Version: "v1alpha1", Resource: "clusters"},
-		{Group: "network.kubesphere.io", Version: "v1alpha1", Resource: "ippools"},
-		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: v2beta1.ResourcesPluralConfig},
-		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: v2beta1.ResourcesPluralReceiver},
-	}
-
-	devopsGVRs := []schema.GroupVersionResource{
-		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibinaries"},
-		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuildertemplates"},
-		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2iruns"},
-		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuilders"},
-		{Group: "devops.kubesphere.io", Version: "v1alpha3", Resource: "devopsprojects"},
-		{Group: "devops.kubesphere.io", Version: "v1alpha3", Resource: "pipelines"},
-	}
-
-	servicemeshGVRs := []schema.GroupVersionResource{
-		{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "strategies"},
-		{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "servicepolicies"},
-	}
-
-	// federated resources on cached in multi cluster setup
-	federatedResourceGVRs := []schema.GroupVersionResource{
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedClusterRole),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedClusterRoleBindingBinding),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedNamespace),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedService),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedDeployment),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedSecret),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedConfigmap),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedStatefulSet),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedIngress),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedResourceQuota),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedPersistentVolumeClaim),
-		typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedApplication),
+	ksGVRs := map[schema.GroupVersion][]string{
+		{Group: "tenant.kubesphere.io", Version: "v1alpha1"}: {
+			"workspaces",
+		},
+		{Group: "tenant.kubesphere.io", Version: "v1alpha2"}: {
+			"workspacetemplates",
+		},
+		{Group: "iam.kubesphere.io", Version: "v1alpha2"}: {
+			"users",
+			"globalroles",
+			"globalrolebindings",
+			"groups",
+			"groupbindings",
+			"workspaceroles",
+			"workspacerolebindings",
+			"loginrecords",
+		},
+		{Group: "cluster.kubesphere.io", Version: "v1alpha1"}: {
+			"clusters",
+		},
+		{Group: "network.kubesphere.io", Version: "v1alpha1"}: {
+			"ippools",
+		},
+		{Group: "notification.kubesphere.io", Version: "v2beta1"}: {
+			notificationv2beta1.ResourcesPluralConfig,
+			notificationv2beta1.ResourcesPluralReceiver,
+		},
 	}

 	// skip caching devops resources if devops not enabled
 	if s.DevopsClient != nil {
-		ksGVRs = append(ksGVRs, devopsGVRs...)
+		ksGVRs[schema.GroupVersion{Group: "devops.kubesphere.io", Version: "v1alpha1"}] = []string{
+			"s2ibinaries",
+			"s2ibuildertemplates",
+			"s2iruns",
+			"s2ibuilders",
+		}
+		ksGVRs[schema.GroupVersion{Group: "devops.kubesphere.io", Version: "v1alpha3"}] = []string{
+			"devopsprojects",
+			"pipelines",
+		}
 	}

 	// skip caching servicemesh resources if servicemesh not enabled
 	if s.KubernetesClient.Istio() != nil {
-		ksGVRs = append(ksGVRs, servicemeshGVRs...)
+		ksGVRs[schema.GroupVersion{Group: "servicemesh.kubesphere.io", Version: "v1alpha2"}] = []string{
+			"strategies",
+			"servicepolicies",
+		}
 	}

+	// federated resources on cached in multi cluster setup
 	if s.Config.MultiClusterOptions.Enable {
-		ksGVRs = append(ksGVRs, federatedResourceGVRs...)
-	}
-
-	for _, gvr := range ksGVRs {
-		if !isResourceExists(gvr) {
-			klog.Warningf("resource %s not exists in the cluster", gvr)
-		} else {
-			_, err = ksInformerFactory.ForResource(gvr)
-			if err != nil {
-				return err
-			}
+		ksGVRs[typesv1beta1.SchemeGroupVersion] = []string{
+			typesv1beta1.ResourcePluralFederatedClusterRole,
+			typesv1beta1.ResourcePluralFederatedClusterRoleBindingBinding,
+			typesv1beta1.ResourcePluralFederatedNamespace,
+			typesv1beta1.ResourcePluralFederatedService,
+			typesv1beta1.ResourcePluralFederatedDeployment,
+			typesv1beta1.ResourcePluralFederatedSecret,
+			typesv1beta1.ResourcePluralFederatedConfigmap,
+			typesv1beta1.ResourcePluralFederatedStatefulSet,
+			typesv1beta1.ResourcePluralFederatedIngress,
+			typesv1beta1.ResourcePluralFederatedPersistentVolumeClaim,
+			typesv1beta1.ResourcePluralFederatedApplication,
 		}
 	}

-	ksInformerFactory.Start(stopCh)
-	ksInformerFactory.WaitForCacheSync(stopCh)
-
-	snapshotInformerFactory := s.InformerFactory.SnapshotSharedInformerFactory()
-	snapshotGVRs := []schema.GroupVersionResource{
-		{Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshotclasses"},
-		{Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshots"},
-		{Group: "snapshot.storage.k8s.io", Version: "v1", Resource: "volumesnapshotcontents"},
-	}
-	for _, gvr := range snapshotGVRs {
-		if !isResourceExists(gvr) {
-			klog.Warningf("resource %s not exists in the cluster", gvr)
-		} else {
-			_, err = snapshotInformerFactory.ForResource(gvr)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	snapshotInformerFactory.Start(stopCh)
-	snapshotInformerFactory.WaitForCacheSync(stopCh)
-
-	apiextensionsInformerFactory := s.InformerFactory.ApiExtensionSharedInformerFactory()
-	apiextensionsGVRs := []schema.GroupVersionResource{
-		{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"},
+	if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
+		s.InformerFactory.KubeSphereSharedInformerFactory(),
+		func(resource schema.GroupVersionResource) (interface{}, error) {
+			return s.InformerFactory.KubeSphereSharedInformerFactory().ForResource(resource)
+		},
+		ksGVRs, stopCh); err != nil {
+		return err
 	}

-	for _, gvr := range apiextensionsGVRs {
-		if !isResourceExists(gvr) {
-			klog.Warningf("resource %s not exists in the cluster", gvr)
-		} else {
-			_, err = apiextensionsInformerFactory.ForResource(gvr)
-			if err != nil {
-				return err
-			}
-		}
+	snapshotGVRs := map[schema.GroupVersion][]string{
+		{Group: "snapshot.storage.k8s.io", Version: "v1"}: {
+			"volumesnapshots",
+			"volumesnapshotcontents",
+			"volumesnapshotclasses",
+		},
+	}
+
+	if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
+		s.InformerFactory.SnapshotSharedInformerFactory(), func(resource schema.GroupVersionResource) (interface{}, error) {
+			return s.InformerFactory.SnapshotSharedInformerFactory().ForResource(resource)
+		},
+		snapshotGVRs, stopCh); err != nil {
+		return err
+	}
+
+	apiextensionsGVRs := map[schema.GroupVersion][]string{
+		{Group: "apiextensions.k8s.io", Version: "v1"}: {
+			"customresourcedefinitions",
+		},
+	}
+
+	if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
+		s.InformerFactory.ApiExtensionSharedInformerFactory(), func(resource schema.GroupVersionResource) (interface{}, error) {
+			return s.InformerFactory.ApiExtensionSharedInformerFactory().ForResource(resource)
+		},
+		apiextensionsGVRs, stopCh); err != nil {
+		return err
 	}
-	apiextensionsInformerFactory.Start(stopCh)
-	apiextensionsInformerFactory.WaitForCacheSync(stopCh)

 	if promFactory := s.InformerFactory.PrometheusSharedInformerFactory(); promFactory != nil {
-		prometheusGVRs := []schema.GroupVersionResource{
-			{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheuses"},
-			{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"},
-			{Group: "monitoring.coreos.com", Version: "v1", Resource: "thanosrulers"},
+		prometheusGVRs := map[schema.GroupVersion][]string{
+			{Group: "monitoring.coreos.com", Version: "v1"}: {
+				"prometheuses",
+				"prometheusrules",
+				"thanosrulers",
+			},
 		}
-		for _, gvr := range prometheusGVRs {
-			if isResourceExists(gvr) {
-				_, err = promFactory.ForResource(gvr)
-				if err != nil {
-					return err
-				}
-			} else {
-				klog.Warningf("resource %s not exists in the cluster", gvr)
-			}
+		if err := waitForCacheSync(s.KubernetesClient.Kubernetes().Discovery(),
+			promFactory, func(resource schema.GroupVersionResource) (interface{}, error) {
+				return promFactory.ForResource(resource)
+			},
+			prometheusGVRs, stopCh); err != nil {
+			return err
 		}
-		promFactory.Start(stopCh)
-		promFactory.WaitForCacheSync(stopCh)
 	}

-	// controller runtime cache for resources
 	go s.RuntimeCache.Start(ctx)
 	s.RuntimeCache.WaitForCacheSync(ctx)

 	klog.V(0).Info("Finished caching objects")
-
 	return nil

 }
--- a/pkg/apiserver/auditing/backend.go
+++ b/pkg/apiserver/auditing/backend.go
@@ -36,7 +36,7 @@ const (
 	DefaultSendersNum    = 100
 	DefaultBatchSize     = 100
 	DefaultBatchInterval = time.Second * 3
-	WebhookURL           = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:443/audit/webhook/event"
+	WebhookURL           = "https://kube-auditing-webhook-svc.kubesphere-logging-system.svc:6443/audit/webhook/event"
 )

 type Backend struct {
--- a/pkg/apiserver/authentication/authenticators/jwt/jwt.go
+++ b/pkg/apiserver/authentication/authenticators/jwt/jwt.go
@@ -60,15 +60,19 @@ func (t *tokenAuthenticator) AuthenticateToken(ctx context.Context, token string
 		}, true, nil
 	}

-	u, err := t.userLister.Get(verified.User.GetName())
+	userInfo, err := t.userLister.Get(verified.User.GetName())
 	if err != nil {
 		return nil, false, err
 	}

+	// AuthLimitExceeded state should be ignored
+	if userInfo.Status.State == iamv1alpha2.UserDisabled {
+		return nil, false, auth.AccountIsNotActiveError
+	}
 	return &authenticator.Response{
 		User: &user.DefaultInfo{
-			Name:   u.GetName(),
-			Groups: append(u.Spec.Groups, user.AllAuthenticated),
+			Name:   userInfo.GetName(),
+			Groups: append(userInfo.Spec.Groups, user.AllAuthenticated),
 		},
 	}, true, nil
 }
--- a/pkg/apiserver/authentication/oauth/options.go
+++ b/pkg/apiserver/authentication/oauth/options.go
@@ -63,10 +63,10 @@ type Options struct {
 	Issuer string `json:"issuer,omitempty" yaml:"issuer,omitempty"`

 	// RSA private key file used to sign the id token
-	SignKey string `json:"signKey,omitempty" yaml:"signKey"`
+	SignKey string `json:"signKey,omitempty" yaml:"signKey,omitempty"`

 	// Raw RSA private key. Base64 encoded PEM file
-	SignKeyData string `json:"-,omitempty" yaml:"signKeyData"`
+	SignKeyData string `json:"-,omitempty" yaml:"signKeyData,omitempty"`

 	// Register identity providers.
 	IdentityProviders []IdentityProviderOptions `json:"identityProviders,omitempty" yaml:"identityProviders,omitempty"`
@@ -191,7 +191,7 @@ type Token struct {
 type Client struct {
 	// The name of the OAuth client is used as the client_id parameter when making requests to <master>/oauth/authorize
 	// and <master>/oauth/token.
-	Name string `json:"name" yaml:"name,omitempty"`
+	Name string `json:"name,omitempty" yaml:"name,omitempty"`

 	// Secret is the unique secret associated with a client
 	Secret string `json:"-" yaml:"secret,omitempty"`
--- a/pkg/apiserver/authentication/options.go
+++ b/pkg/apiserver/authentication/options.go
@@ -51,7 +51,7 @@ type Options struct {
 	LoginHistoryRetentionPeriod time.Duration `json:"loginHistoryRetentionPeriod" yaml:"loginHistoryRetentionPeriod"`
 	// retention login history, records beyond this amount will be deleted
 	// LoginHistoryMaximumEntries restricts for all kubesphere accounts and must be greater than AuthenticateRateLimiterMaxTries
-	LoginHistoryMaximumEntries int `json:"loginHistoryMaximumEntries" yaml:"loginHistoryMaximumEntries"`
+	LoginHistoryMaximumEntries int `json:"loginHistoryMaximumEntries,omitempty" yaml:"loginHistoryMaximumEntries,omitempty"`
 	// allow multiple users login from different location at the same time
 	MultipleLogin bool `json:"multipleLogin" yaml:"multipleLogin"`
 	// secret to sign jwt token
--- a/pkg/apiserver/authorization/options.go
+++ b/pkg/apiserver/authorization/options.go
@@ -28,7 +28,7 @@ import (
 )

 type Options struct {
-	Mode string `json:"mode"  yaml:"mode"`
+	Mode string `json:"mode" yaml:"mode"`
 }

 func NewOptions() *Options {
--- a/pkg/apiserver/config/config.go
+++ b/pkg/apiserver/config/config.go
@@ -20,18 +20,25 @@ import (
 	"fmt"
 	"reflect"
 	"strings"
+	"sync"

-	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
-	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
-
+	"github.com/fsnotify/fsnotify"
 	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog"

 	networkv1alpha1 "kubesphere.io/api/network/v1alpha1"

+	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
+	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/models/terminal"
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
 	"kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
+	"kubesphere.io/kubesphere/pkg/simple/client/edgeruntime"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
 	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
 	"kubesphere.io/kubesphere/pkg/simple/client/gpu"
@@ -77,6 +84,11 @@ import (
 // mysql-host is missing in command line flags, all other mysql command line flags
 // will be ignored.

+var (
+	// singleton instance of config package
+	_config = defaultConfig()
+)
+
 const (
 	// DefaultConfigurationName is the default name of configuration
 	defaultConfigurationName = "kubesphere"
@@ -85,6 +97,61 @@ const (
 	defaultConfigurationPath = "/etc/kubesphere"
 )

+type config struct {
+	cfg         *Config
+	cfgChangeCh chan Config
+	watchOnce   sync.Once
+	loadOnce    sync.Once
+}
+
+func (c *config) watchConfig() <-chan Config {
+	c.watchOnce.Do(func() {
+		viper.WatchConfig()
+		viper.OnConfigChange(func(in fsnotify.Event) {
+			cfg := New()
+			if err := viper.Unmarshal(cfg); err != nil {
+				klog.Warning("config reload error", err)
+			} else {
+				c.cfgChangeCh <- *cfg
+			}
+		})
+	})
+	return c.cfgChangeCh
+}
+
+func (c *config) loadFromDisk() (*Config, error) {
+	var err error
+	c.loadOnce.Do(func() {
+		if err = viper.ReadInConfig(); err != nil {
+			if _, ok := err.(viper.ConfigFileNotFoundError); !ok {
+				err = fmt.Errorf("error parsing configuration file %s", err)
+			}
+		}
+		err = viper.Unmarshal(c.cfg)
+	})
+	return c.cfg, err
+}
+
+func defaultConfig() *config {
+	viper.SetConfigName(defaultConfigurationName)
+	viper.AddConfigPath(defaultConfigurationPath)
+
+	// Load from current working directory, only used for debugging
+	viper.AddConfigPath(".")
+
+	// Load from Environment variables
+	viper.SetEnvPrefix("kubesphere")
+	viper.AutomaticEnv()
+	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
+
+	return &config{
+		cfg:         New(),
+		cfgChangeCh: make(chan Config),
+		watchOnce:   sync.Once{},
+		loadOnce:    sync.Once{},
+	}
+}
+
 // Config defines everything needed for apiserver to deal with external services
 type Config struct {
 	DevopsOptions         *jenkins.Options        `json:"devops,omitempty" yaml:"devops,omitempty" mapstructure:"devops"`
@@ -106,9 +173,11 @@ type Config struct {
 	AlertingOptions       *alerting.Options       `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"`
 	NotificationOptions   *notification.Options   `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"`
 	KubeEdgeOptions       *kubeedge.Options       `json:"kubeedge,omitempty" yaml:"kubeedge,omitempty" mapstructure:"kubeedge"`
+	EdgeRuntimeOptions    *edgeruntime.Options    `json:"edgeruntime,omitempty" yaml:"edgeruntime,omitempty" mapstructure:"edgeruntime"`
 	MeteringOptions       *metering.Options       `json:"metering,omitempty" yaml:"metering,omitempty" mapstructure:"metering"`
 	GatewayOptions        *gateway.Options        `json:"gateway,omitempty" yaml:"gateway,omitempty" mapstructure:"gateway"`
 	GPUOptions            *gpu.Options            `json:"gpu,omitempty" yaml:"gpu,omitempty" mapstructure:"gpu"`
+	TerminalOptions       *terminal.Options       `json:"terminal,omitempty" yaml:"terminal,omitempty" mapstructure:"terminal"`
 }

 // newConfig creates a default non-empty Config
@@ -133,41 +202,23 @@ func New() *Config {
 		EventsOptions:         events.NewEventsOptions(),
 		AuditingOptions:       auditing.NewAuditingOptions(),
 		KubeEdgeOptions:       kubeedge.NewKubeEdgeOptions(),
+		EdgeRuntimeOptions:    edgeruntime.NewEdgeRuntimeOptions(),
 		MeteringOptions:       metering.NewMeteringOptions(),
 		GatewayOptions:        gateway.NewGatewayOptions(),
 		GPUOptions:            gpu.NewGPUOptions(),
+		TerminalOptions:       terminal.NewTerminalOptions(),
 	}
 }

 // TryLoadFromDisk loads configuration from default location after server startup
 // return nil error if configuration file not exists
 func TryLoadFromDisk() (*Config, error) {
-	viper.SetConfigName(defaultConfigurationName)
-	viper.AddConfigPath(defaultConfigurationPath)
+	return _config.loadFromDisk()
+}

-	// Load from current working directory, only used for debugging
-	viper.AddConfigPath(".")
-
-	// Load from Environment variables
-	viper.SetEnvPrefix("kubesphere")
-	viper.AutomaticEnv()
-	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
-
-	if err := viper.ReadInConfig(); err != nil {
-		if _, ok := err.(viper.ConfigFileNotFoundError); ok {
-			return nil, err
-		} else {
-			return nil, fmt.Errorf("error parsing configuration file %s", err)
-		}
-	}
-
-	conf := New()
-
-	if err := viper.Unmarshal(conf); err != nil {
-		return nil, err
-	}
-
-	return conf, nil
+// WatchConfigChange return config change channel
+func WatchConfigChange() <-chan Config {
+	return _config.watchConfig()
 }

 // convertToMap simply converts config to map[string]bool
@@ -304,7 +355,25 @@ func (conf *Config) stripEmptyOptions() {
 		conf.KubeEdgeOptions = nil
 	}

+	if conf.EdgeRuntimeOptions != nil && conf.EdgeRuntimeOptions.Endpoint == "" {
+		conf.EdgeRuntimeOptions = nil
+	}
+
 	if conf.GPUOptions != nil && len(conf.GPUOptions.Kinds) == 0 {
 		conf.GPUOptions = nil
 	}
 }
+
+// GetFromConfigMap returns KubeSphere ruuning config by the given ConfigMap.
+func GetFromConfigMap(cm *corev1.ConfigMap) (*Config, error) {
+	c := &Config{}
+	value, ok := cm.Data[constants.KubeSphereConfigMapDataKey]
+	if !ok {
+		return nil, fmt.Errorf("failed to get configmap kubesphere.yaml value")
+	}
+
+	if err := yaml.Unmarshal([]byte(value), c); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal value from configmap. err: %s", err)
+	}
+	return c, nil
+}
--- a/pkg/apiserver/config/config_test.go
+++ b/pkg/apiserver/config/config_test.go
@@ -23,19 +23,20 @@ import (
 	"testing"
 	"time"

-	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
-	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
-
 	"github.com/google/go-cmp/cmp"
 	"gopkg.in/yaml.v2"

 	networkv1alpha1 "kubesphere.io/api/network/v1alpha1"

+	"kubesphere.io/kubesphere/pkg/apiserver/authentication"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth"
+	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
+	"kubesphere.io/kubesphere/pkg/models/terminal"
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
 	"kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
+	"kubesphere.io/kubesphere/pkg/simple/client/edgeruntime"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
 	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
 	"kubesphere.io/kubesphere/pkg/simple/client/gpu"
@@ -55,7 +56,6 @@ import (
 )

 func newTestConfig() (*Config, error) {
-
 	var conf = &Config{
 		DevopsOptions: &jenkins.Options{
 			Host:           "http://ks-devops.kubesphere-devops-system.svc",
@@ -84,6 +84,9 @@ func newTestConfig() (*Config, error) {
 			ManagerPassword: "P@88w0rd",
 			UserSearchBase:  "ou=Users,dc=example,dc=org",
 			GroupSearchBase: "ou=Groups,dc=example,dc=org",
+			InitialCap:      10,
+			MaxCap:          100,
+			PoolName:        "ldap",
 		},
 		RedisOptions: &cache.Options{
 			Host:     "localhost",
@@ -93,7 +96,7 @@ func newTestConfig() (*Config, error) {
 		},
 		S3Options: &s3.Options{
 			Endpoint:        "http://minio.openpitrix-system.svc",
-			Region:          "",
+			Region:          "us-east-1",
 			DisableSSL:      false,
 			ForcePathStyle:  false,
 			AccessKeyID:     "ABCDEFGHIJKLMN",
@@ -148,6 +151,7 @@ func newTestConfig() (*Config, error) {
 			AuthenticateRateLimiterMaxTries: 5,
 			AuthenticateRateLimiterDuration: 30 * time.Minute,
 			JwtSecret:                       "xxxxxx",
+			LoginHistoryMaximumEntries:      100,
 			MultipleLogin:                   false,
 			OAuthOptions: &oauth.Options{
 				Issuer:            oauth.DefaultIssuer,
@@ -164,9 +168,7 @@ func newTestConfig() (*Config, error) {
 				AccessTokenInactivityTimeout: 0,
 			},
 		},
-		MultiClusterOptions: &multicluster.Options{
-			Enable: false,
-		},
+		MultiClusterOptions: multicluster.NewOptions(),
 		EventsOptions: &events.Options{
 			Host:        "http://elasticsearch-logging-data.kubesphere-logging-system.svc:9200",
 			IndexPrefix: "ks-logstash-events",
@@ -180,6 +182,9 @@ func newTestConfig() (*Config, error) {
 		KubeEdgeOptions: &kubeedge.Options{
 			Endpoint: "http://edge-watcher.kubeedge.svc/api/",
 		},
+		EdgeRuntimeOptions: &edgeruntime.Options{
+			Endpoint: "http://edgeservice.kubeedge.svc/api/",
+		},
 		MeteringOptions: &metering.Options{
 			RetentionDay: "7d",
 		},
@@ -190,6 +195,10 @@ func newTestConfig() (*Config, error) {
 		GPUOptions: &gpu.Options{
 			Kinds: []gpu.GPUKind{},
 		},
+		TerminalOptions: &terminal.Options{
+			Image:   "alpine:3.15",
+			Timeout: 600,
+		},
 	}
 	return conf, nil
 }
@@ -271,6 +280,7 @@ func TestStripEmptyOptions(t *testing.T) {
 	config.EventsOptions = &events.Options{Host: ""}
 	config.AuditingOptions = &auditing.Options{Host: ""}
 	config.KubeEdgeOptions = &kubeedge.Options{Endpoint: ""}
+	config.EdgeRuntimeOptions = &edgeruntime.Options{Endpoint: ""}

 	config.stripEmptyOptions()

@@ -288,7 +298,8 @@ func TestStripEmptyOptions(t *testing.T) {
 		config.MultiClusterOptions != nil ||
 		config.EventsOptions != nil ||
 		config.AuditingOptions != nil ||
-		config.KubeEdgeOptions != nil {
+		config.KubeEdgeOptions != nil ||
+		config.EdgeRuntimeOptions != nil {
 		t.Fatal("config stripEmptyOptions failed")
 	}
 }
--- a/pkg/apiserver/dispatch/dispatch.go
+++ b/pkg/apiserver/dispatch/dispatch.go
@@ -30,7 +30,6 @@ import (
 	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"

 	"kubesphere.io/kubesphere/pkg/apiserver/request"
-	clusterinformer "kubesphere.io/kubesphere/pkg/client/informers/externalversions/cluster/v1alpha1"
 	"kubesphere.io/kubesphere/pkg/utils/clusterclient"
 )

@@ -47,8 +46,8 @@ type clusterDispatch struct {
 	clusterclient.ClusterClients
 }

-func NewClusterDispatch(clusterInformer clusterinformer.ClusterInformer) Dispatcher {
-	return &clusterDispatch{clusterclient.NewClusterClient(clusterInformer)}
+func NewClusterDispatch(cc clusterclient.ClusterClients) Dispatcher {
+	return &clusterDispatch{cc}
 }

 // Dispatch dispatch requests to designated cluster
--- a/pkg/apiserver/metric.go
+++ b/pkg/apiserver/metric.go
@@ -1,3 +1,17 @@
+// Copyright 2022 The KubeSphere Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 package apiserver

 import (
--- a/pkg/client/clientset/versioned/fake/register.go
+++ b/pkg/client/clientset/versioned/fake/register.go
@@ -42,7 +42,7 @@ import (

 var scheme = runtime.NewScheme()
 var codecs = serializer.NewCodecFactory(scheme)
-var parameterCodec = runtime.NewParameterCodec(scheme)
+
 var localSchemeBuilder = runtime.SchemeBuilder{
 	applicationv1alpha1.AddToScheme,
 	auditingv1alpha1.AddToScheme,
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_provisionercapability.go
@@ -0,0 +1,122 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+	v1alpha1 "kubesphere.io/api/storage/v1alpha1"
+)
+
+// FakeProvisionerCapabilities implements ProvisionerCapabilityInterface
+type FakeProvisionerCapabilities struct {
+	Fake *FakeStorageV1alpha1
+}
+
+var provisionercapabilitiesResource = schema.GroupVersionResource{Group: "storage.kubesphere.io", Version: "v1alpha1", Resource: "provisionercapabilities"}
+
+var provisionercapabilitiesKind = schema.GroupVersionKind{Group: "storage.kubesphere.io", Version: "v1alpha1", Kind: "ProvisionerCapability"}
+
+// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any.
+func (c *FakeProvisionerCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootGetAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors.
+func (c *FakeProvisionerCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootListAction(provisionercapabilitiesResource, provisionercapabilitiesKind, opts), &v1alpha1.ProvisionerCapabilityList{})
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1alpha1.ProvisionerCapabilityList{ListMeta: obj.(*v1alpha1.ProvisionerCapabilityList).ListMeta}
+	for _, item := range obj.(*v1alpha1.ProvisionerCapabilityList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested provisionerCapabilities.
+func (c *FakeProvisionerCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewRootWatchAction(provisionercapabilitiesResource, opts))
+}
+
+// Create takes the representation of a provisionerCapability and creates it.  Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *FakeProvisionerCapabilities) Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootCreateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *FakeProvisionerCapabilities) Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateAction(provisionercapabilitiesResource, provisionerCapability), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
+
+// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs.
+func (c *FakeProvisionerCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewRootDeleteAction(provisionercapabilitiesResource, name), &v1alpha1.ProvisionerCapability{})
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeProvisionerCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewRootDeleteCollectionAction(provisionercapabilitiesResource, listOpts)
+
+	_, err := c.Fake.Invokes(action, &v1alpha1.ProvisionerCapabilityList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched provisionerCapability.
+func (c *FakeProvisionerCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootPatchSubresourceAction(provisionercapabilitiesResource, name, pt, data, subresources...), &v1alpha1.ProvisionerCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.ProvisionerCapability), err
+}
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storage_client.go
@@ -21,12 +21,21 @@ package fake
 import (
 	rest "k8s.io/client-go/rest"
 	testing "k8s.io/client-go/testing"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/client/clientset/versioned/typed/storage/v1alpha1"
 )

 type FakeStorageV1alpha1 struct {
 	*testing.Fake
 }

+func (c *FakeStorageV1alpha1) ProvisionerCapabilities() v1alpha1.ProvisionerCapabilityInterface {
+	return &FakeProvisionerCapabilities{c}
+}
+
+func (c *FakeStorageV1alpha1) StorageClassCapabilities() v1alpha1.StorageClassCapabilityInterface {
+	return &FakeStorageClassCapabilities{c}
+}
+
 // RESTClient returns a RESTClient that is used to communicate
 // with API server by this client implementation.
 func (c *FakeStorageV1alpha1) RESTClient() rest.Interface {
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/fake/fake_storageclasscapability.go
@@ -0,0 +1,122 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package fake
+
+import (
+	"context"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	labels "k8s.io/apimachinery/pkg/labels"
+	schema "k8s.io/apimachinery/pkg/runtime/schema"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	testing "k8s.io/client-go/testing"
+	v1alpha1 "kubesphere.io/api/storage/v1alpha1"
+)
+
+// FakeStorageClassCapabilities implements StorageClassCapabilityInterface
+type FakeStorageClassCapabilities struct {
+	Fake *FakeStorageV1alpha1
+}
+
+var storageclasscapabilitiesResource = schema.GroupVersionResource{Group: "storage.kubesphere.io", Version: "v1alpha1", Resource: "storageclasscapabilities"}
+
+var storageclasscapabilitiesKind = schema.GroupVersionKind{Group: "storage.kubesphere.io", Version: "v1alpha1", Kind: "StorageClassCapability"}
+
+// Get takes name of the storageClassCapability, and returns the corresponding storageClassCapability object, and an error if there is any.
+func (c *FakeStorageClassCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootGetAction(storageclasscapabilitiesResource, name), &v1alpha1.StorageClassCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.StorageClassCapability), err
+}
+
+// List takes label and field selectors, and returns the list of StorageClassCapabilities that match those selectors.
+func (c *FakeStorageClassCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.StorageClassCapabilityList, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootListAction(storageclasscapabilitiesResource, storageclasscapabilitiesKind, opts), &v1alpha1.StorageClassCapabilityList{})
+	if obj == nil {
+		return nil, err
+	}
+
+	label, _, _ := testing.ExtractFromListOptions(opts)
+	if label == nil {
+		label = labels.Everything()
+	}
+	list := &v1alpha1.StorageClassCapabilityList{ListMeta: obj.(*v1alpha1.StorageClassCapabilityList).ListMeta}
+	for _, item := range obj.(*v1alpha1.StorageClassCapabilityList).Items {
+		if label.Matches(labels.Set(item.Labels)) {
+			list.Items = append(list.Items, item)
+		}
+	}
+	return list, err
+}
+
+// Watch returns a watch.Interface that watches the requested storageClassCapabilities.
+func (c *FakeStorageClassCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	return c.Fake.
+		InvokesWatch(testing.NewRootWatchAction(storageclasscapabilitiesResource, opts))
+}
+
+// Create takes the representation of a storageClassCapability and creates it.  Returns the server's representation of the storageClassCapability, and an error, if there is any.
+func (c *FakeStorageClassCapabilities) Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootCreateAction(storageclasscapabilitiesResource, storageClassCapability), &v1alpha1.StorageClassCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.StorageClassCapability), err
+}
+
+// Update takes the representation of a storageClassCapability and updates it. Returns the server's representation of the storageClassCapability, and an error, if there is any.
+func (c *FakeStorageClassCapabilities) Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootUpdateAction(storageclasscapabilitiesResource, storageClassCapability), &v1alpha1.StorageClassCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.StorageClassCapability), err
+}
+
+// Delete takes name of the storageClassCapability and deletes it. Returns an error if one occurs.
+func (c *FakeStorageClassCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	_, err := c.Fake.
+		Invokes(testing.NewRootDeleteAction(storageclasscapabilitiesResource, name), &v1alpha1.StorageClassCapability{})
+	return err
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *FakeStorageClassCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	action := testing.NewRootDeleteCollectionAction(storageclasscapabilitiesResource, listOpts)
+
+	_, err := c.Fake.Invokes(action, &v1alpha1.StorageClassCapabilityList{})
+	return err
+}
+
+// Patch applies the patch and returns the patched storageClassCapability.
+func (c *FakeStorageClassCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error) {
+	obj, err := c.Fake.
+		Invokes(testing.NewRootPatchSubresourceAction(storageclasscapabilitiesResource, name, pt, data, subresources...), &v1alpha1.StorageClassCapability{})
+	if obj == nil {
+		return nil, err
+	}
+	return obj.(*v1alpha1.StorageClassCapability), err
+}
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/provisionercapability.go
@@ -0,0 +1,168 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"context"
+	"time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+	v1alpha1 "kubesphere.io/api/storage/v1alpha1"
+	scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+)
+
+// ProvisionerCapabilitiesGetter has a method to return a ProvisionerCapabilityInterface.
+// A group's client should implement this interface.
+type ProvisionerCapabilitiesGetter interface {
+	ProvisionerCapabilities() ProvisionerCapabilityInterface
+}
+
+// ProvisionerCapabilityInterface has methods to work with ProvisionerCapability resources.
+type ProvisionerCapabilityInterface interface {
+	Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (*v1alpha1.ProvisionerCapability, error)
+	Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (*v1alpha1.ProvisionerCapability, error)
+	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.ProvisionerCapability, error)
+	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.ProvisionerCapabilityList, error)
+	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error)
+	ProvisionerCapabilityExpansion
+}
+
+// provisionerCapabilities implements ProvisionerCapabilityInterface
+type provisionerCapabilities struct {
+	client rest.Interface
+}
+
+// newProvisionerCapabilities returns a ProvisionerCapabilities
+func newProvisionerCapabilities(c *StorageV1alpha1Client) *provisionerCapabilities {
+	return &provisionerCapabilities{
+		client: c.RESTClient(),
+	}
+}
+
+// Get takes name of the provisionerCapability, and returns the corresponding provisionerCapability object, and an error if there is any.
+func (c *provisionerCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Get().
+		Resource("provisionercapabilities").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of ProvisionerCapabilities that match those selectors.
+func (c *provisionerCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.ProvisionerCapabilityList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1alpha1.ProvisionerCapabilityList{}
+	err = c.client.Get().
+		Resource("provisionercapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested provisionerCapabilities.
+func (c *provisionerCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Resource("provisionercapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch(ctx)
+}
+
+// Create takes the representation of a provisionerCapability and creates it.  Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *provisionerCapabilities) Create(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.CreateOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Post().
+		Resource("provisionercapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(provisionerCapability).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Update takes the representation of a provisionerCapability and updates it. Returns the server's representation of the provisionerCapability, and an error, if there is any.
+func (c *provisionerCapabilities) Update(ctx context.Context, provisionerCapability *v1alpha1.ProvisionerCapability, opts v1.UpdateOptions) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Put().
+		Resource("provisionercapabilities").
+		Name(provisionerCapability.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(provisionerCapability).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Delete takes name of the provisionerCapability and deletes it. Returns an error if one occurs.
+func (c *provisionerCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	return c.client.Delete().
+		Resource("provisionercapabilities").
+		Name(name).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *provisionerCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	var timeout time.Duration
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Resource("provisionercapabilities").
+		VersionedParams(&listOpts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// Patch applies the patch and returns the patched provisionerCapability.
+func (c *provisionerCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.ProvisionerCapability, err error) {
+	result = &v1alpha1.ProvisionerCapability{}
+	err = c.client.Patch(pt).
+		Resource("provisionercapabilities").
+		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(data).
+		Do(ctx).
+		Into(result)
+	return
+}
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/storage_client.go
@@ -26,6 +26,8 @@ import (

 type StorageV1alpha1Interface interface {
 	RESTClient() rest.Interface
+	ProvisionerCapabilitiesGetter
+	StorageClassCapabilitiesGetter
 }

 // StorageV1alpha1Client is used to interact with features provided by the storage.kubesphere.io group.
@@ -33,6 +35,14 @@ type StorageV1alpha1Client struct {
 	restClient rest.Interface
 }

+func (c *StorageV1alpha1Client) ProvisionerCapabilities() ProvisionerCapabilityInterface {
+	return newProvisionerCapabilities(c)
+}
+
+func (c *StorageV1alpha1Client) StorageClassCapabilities() StorageClassCapabilityInterface {
+	return newStorageClassCapabilities(c)
+}
+
 // NewForConfig creates a new StorageV1alpha1Client for the given config.
 func NewForConfig(c *rest.Config) (*StorageV1alpha1Client, error) {
 	config := *c
--- a/pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go
+++ b/pkg/client/clientset/versioned/typed/storage/v1alpha1/storageclasscapability.go
@@ -0,0 +1,168 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by client-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"context"
+	"time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	watch "k8s.io/apimachinery/pkg/watch"
+	rest "k8s.io/client-go/rest"
+	v1alpha1 "kubesphere.io/api/storage/v1alpha1"
+	scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+)
+
+// StorageClassCapabilitiesGetter has a method to return a StorageClassCapabilityInterface.
+// A group's client should implement this interface.
+type StorageClassCapabilitiesGetter interface {
+	StorageClassCapabilities() StorageClassCapabilityInterface
+}
+
+// StorageClassCapabilityInterface has methods to work with StorageClassCapability resources.
+type StorageClassCapabilityInterface interface {
+	Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (*v1alpha1.StorageClassCapability, error)
+	Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (*v1alpha1.StorageClassCapability, error)
+	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
+	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
+	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1alpha1.StorageClassCapability, error)
+	List(ctx context.Context, opts v1.ListOptions) (*v1alpha1.StorageClassCapabilityList, error)
+	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
+	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error)
+	StorageClassCapabilityExpansion
+}
+
+// storageClassCapabilities implements StorageClassCapabilityInterface
+type storageClassCapabilities struct {
+	client rest.Interface
+}
+
+// newStorageClassCapabilities returns a StorageClassCapabilities
+func newStorageClassCapabilities(c *StorageV1alpha1Client) *storageClassCapabilities {
+	return &storageClassCapabilities{
+		client: c.RESTClient(),
+	}
+}
+
+// Get takes name of the storageClassCapability, and returns the corresponding storageClassCapability object, and an error if there is any.
+func (c *storageClassCapabilities) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	result = &v1alpha1.StorageClassCapability{}
+	err = c.client.Get().
+		Resource("storageclasscapabilities").
+		Name(name).
+		VersionedParams(&options, scheme.ParameterCodec).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// List takes label and field selectors, and returns the list of StorageClassCapabilities that match those selectors.
+func (c *storageClassCapabilities) List(ctx context.Context, opts v1.ListOptions) (result *v1alpha1.StorageClassCapabilityList, err error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	result = &v1alpha1.StorageClassCapabilityList{}
+	err = c.client.Get().
+		Resource("storageclasscapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Watch returns a watch.Interface that watches the requested storageClassCapabilities.
+func (c *storageClassCapabilities) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
+	var timeout time.Duration
+	if opts.TimeoutSeconds != nil {
+		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
+	}
+	opts.Watch = true
+	return c.client.Get().
+		Resource("storageclasscapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Watch(ctx)
+}
+
+// Create takes the representation of a storageClassCapability and creates it.  Returns the server's representation of the storageClassCapability, and an error, if there is any.
+func (c *storageClassCapabilities) Create(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.CreateOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	result = &v1alpha1.StorageClassCapability{}
+	err = c.client.Post().
+		Resource("storageclasscapabilities").
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(storageClassCapability).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Update takes the representation of a storageClassCapability and updates it. Returns the server's representation of the storageClassCapability, and an error, if there is any.
+func (c *storageClassCapabilities) Update(ctx context.Context, storageClassCapability *v1alpha1.StorageClassCapability, opts v1.UpdateOptions) (result *v1alpha1.StorageClassCapability, err error) {
+	result = &v1alpha1.StorageClassCapability{}
+	err = c.client.Put().
+		Resource("storageclasscapabilities").
+		Name(storageClassCapability.Name).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(storageClassCapability).
+		Do(ctx).
+		Into(result)
+	return
+}
+
+// Delete takes name of the storageClassCapability and deletes it. Returns an error if one occurs.
+func (c *storageClassCapabilities) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
+	return c.client.Delete().
+		Resource("storageclasscapabilities").
+		Name(name).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// DeleteCollection deletes a collection of objects.
+func (c *storageClassCapabilities) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
+	var timeout time.Duration
+	if listOpts.TimeoutSeconds != nil {
+		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
+	}
+	return c.client.Delete().
+		Resource("storageclasscapabilities").
+		VersionedParams(&listOpts, scheme.ParameterCodec).
+		Timeout(timeout).
+		Body(&opts).
+		Do(ctx).
+		Error()
+}
+
+// Patch applies the patch and returns the patched storageClassCapability.
+func (c *storageClassCapabilities) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1alpha1.StorageClassCapability, err error) {
+	result = &v1alpha1.StorageClassCapability{}
+	err = c.client.Patch(pt).
+		Resource("storageclasscapabilities").
+		Name(name).
+		SubResource(subresources...).
+		VersionedParams(&opts, scheme.ParameterCodec).
+		Body(data).
+		Do(ctx).
+		Into(result)
+	return
+}
--- a/pkg/client/clientset/versioned/typed/types/v1beta1/fake/fake_federatedresourcequota.go
+++ b/pkg/client/clientset/versioned/typed/types/v1beta1/fake/fake_federatedresourcequota.go
@@ -1,142 +0,0 @@
-/*
-Copyright 2020 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by client-gen. DO NOT EDIT.
-
-package fake
-
-import (
-	"context"
-
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	labels "k8s.io/apimachinery/pkg/labels"
-	schema "k8s.io/apimachinery/pkg/runtime/schema"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	testing "k8s.io/client-go/testing"
-	v1beta1 "kubesphere.io/api/types/v1beta1"
-)
-
-// FakeFederatedResourceQuotas implements FederatedResourceQuotaInterface
-type FakeFederatedResourceQuotas struct {
-	Fake *FakeTypesV1beta1
-	ns   string
-}
-
-var federatedresourcequotasResource = schema.GroupVersionResource{Group: "types.kubefed.io", Version: "v1beta1", Resource: "federatedresourcequotas"}
-
-var federatedresourcequotasKind = schema.GroupVersionKind{Group: "types.kubefed.io", Version: "v1beta1", Kind: "FederatedResourceQuota"}
-
-// Get takes name of the federatedResourceQuota, and returns the corresponding federatedResourceQuota object, and an error if there is any.
-func (c *FakeFederatedResourceQuotas) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewGetAction(federatedresourcequotasResource, c.ns, name), &v1beta1.FederatedResourceQuota{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1beta1.FederatedResourceQuota), err
-}
-
-// List takes label and field selectors, and returns the list of FederatedResourceQuotas that match those selectors.
-func (c *FakeFederatedResourceQuotas) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.FederatedResourceQuotaList, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewListAction(federatedresourcequotasResource, federatedresourcequotasKind, c.ns, opts), &v1beta1.FederatedResourceQuotaList{})
-
-	if obj == nil {
-		return nil, err
-	}
-
-	label, _, _ := testing.ExtractFromListOptions(opts)
-	if label == nil {
-		label = labels.Everything()
-	}
-	list := &v1beta1.FederatedResourceQuotaList{ListMeta: obj.(*v1beta1.FederatedResourceQuotaList).ListMeta}
-	for _, item := range obj.(*v1beta1.FederatedResourceQuotaList).Items {
-		if label.Matches(labels.Set(item.Labels)) {
-			list.Items = append(list.Items, item)
-		}
-	}
-	return list, err
-}
-
-// Watch returns a watch.Interface that watches the requested federatedResourceQuotas.
-func (c *FakeFederatedResourceQuotas) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
-	return c.Fake.
-		InvokesWatch(testing.NewWatchAction(federatedresourcequotasResource, c.ns, opts))
-
-}
-
-// Create takes the representation of a federatedResourceQuota and creates it.  Returns the server's representation of the federatedResourceQuota, and an error, if there is any.
-func (c *FakeFederatedResourceQuotas) Create(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.CreateOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewCreateAction(federatedresourcequotasResource, c.ns, federatedResourceQuota), &v1beta1.FederatedResourceQuota{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1beta1.FederatedResourceQuota), err
-}
-
-// Update takes the representation of a federatedResourceQuota and updates it. Returns the server's representation of the federatedResourceQuota, and an error, if there is any.
-func (c *FakeFederatedResourceQuotas) Update(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewUpdateAction(federatedresourcequotasResource, c.ns, federatedResourceQuota), &v1beta1.FederatedResourceQuota{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1beta1.FederatedResourceQuota), err
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *FakeFederatedResourceQuotas) UpdateStatus(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (*v1beta1.FederatedResourceQuota, error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewUpdateSubresourceAction(federatedresourcequotasResource, "status", c.ns, federatedResourceQuota), &v1beta1.FederatedResourceQuota{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1beta1.FederatedResourceQuota), err
-}
-
-// Delete takes name of the federatedResourceQuota and deletes it. Returns an error if one occurs.
-func (c *FakeFederatedResourceQuotas) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
-	_, err := c.Fake.
-		Invokes(testing.NewDeleteAction(federatedresourcequotasResource, c.ns, name), &v1beta1.FederatedResourceQuota{})
-
-	return err
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *FakeFederatedResourceQuotas) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	action := testing.NewDeleteCollectionAction(federatedresourcequotasResource, c.ns, listOpts)
-
-	_, err := c.Fake.Invokes(action, &v1beta1.FederatedResourceQuotaList{})
-	return err
-}
-
-// Patch applies the patch and returns the patched federatedResourceQuota.
-func (c *FakeFederatedResourceQuotas) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FederatedResourceQuota, err error) {
-	obj, err := c.Fake.
-		Invokes(testing.NewPatchSubresourceAction(federatedresourcequotasResource, c.ns, name, pt, data, subresources...), &v1beta1.FederatedResourceQuota{})
-
-	if obj == nil {
-		return nil, err
-	}
-	return obj.(*v1beta1.FederatedResourceQuota), err
-}
--- a/pkg/client/clientset/versioned/typed/types/v1beta1/fake/fake_types_client.go
+++ b/pkg/client/clientset/versioned/typed/types/v1beta1/fake/fake_types_client.go
@@ -76,10 +76,6 @@ func (c *FakeTypesV1beta1) FederatedPersistentVolumeClaims(namespace string) v1b
 	return &FakeFederatedPersistentVolumeClaims{c, namespace}
 }

-func (c *FakeTypesV1beta1) FederatedResourceQuotas(namespace string) v1beta1.FederatedResourceQuotaInterface {
-	return &FakeFederatedResourceQuotas{c, namespace}
-}
-
 func (c *FakeTypesV1beta1) FederatedSecrets(namespace string) v1beta1.FederatedSecretInterface {
 	return &FakeFederatedSecrets{c, namespace}
 }
--- a/pkg/client/clientset/versioned/typed/types/v1beta1/federatedresourcequota.go
+++ b/pkg/client/clientset/versioned/typed/types/v1beta1/federatedresourcequota.go
@@ -1,195 +0,0 @@
-/*
-Copyright 2020 The KubeSphere Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by client-gen. DO NOT EDIT.
-
-package v1beta1
-
-import (
-	"context"
-	"time"
-
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	types "k8s.io/apimachinery/pkg/types"
-	watch "k8s.io/apimachinery/pkg/watch"
-	rest "k8s.io/client-go/rest"
-	v1beta1 "kubesphere.io/api/types/v1beta1"
-	scheme "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
-)
-
-// FederatedResourceQuotasGetter has a method to return a FederatedResourceQuotaInterface.
-// A group's client should implement this interface.
-type FederatedResourceQuotasGetter interface {
-	FederatedResourceQuotas(namespace string) FederatedResourceQuotaInterface
-}
-
-// FederatedResourceQuotaInterface has methods to work with FederatedResourceQuota resources.
-type FederatedResourceQuotaInterface interface {
-	Create(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.CreateOptions) (*v1beta1.FederatedResourceQuota, error)
-	Update(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (*v1beta1.FederatedResourceQuota, error)
-	UpdateStatus(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (*v1beta1.FederatedResourceQuota, error)
-	Delete(ctx context.Context, name string, opts v1.DeleteOptions) error
-	DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error
-	Get(ctx context.Context, name string, opts v1.GetOptions) (*v1beta1.FederatedResourceQuota, error)
-	List(ctx context.Context, opts v1.ListOptions) (*v1beta1.FederatedResourceQuotaList, error)
-	Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error)
-	Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FederatedResourceQuota, err error)
-	FederatedResourceQuotaExpansion
-}
-
-// federatedResourceQuotas implements FederatedResourceQuotaInterface
-type federatedResourceQuotas struct {
-	client rest.Interface
-	ns     string
-}
-
-// newFederatedResourceQuotas returns a FederatedResourceQuotas
-func newFederatedResourceQuotas(c *TypesV1beta1Client, namespace string) *federatedResourceQuotas {
-	return &federatedResourceQuotas{
-		client: c.RESTClient(),
-		ns:     namespace,
-	}
-}
-
-// Get takes name of the federatedResourceQuota, and returns the corresponding federatedResourceQuota object, and an error if there is any.
-func (c *federatedResourceQuotas) Get(ctx context.Context, name string, options v1.GetOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	result = &v1beta1.FederatedResourceQuota{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		Name(name).
-		VersionedParams(&options, scheme.ParameterCodec).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// List takes label and field selectors, and returns the list of FederatedResourceQuotas that match those selectors.
-func (c *federatedResourceQuotas) List(ctx context.Context, opts v1.ListOptions) (result *v1beta1.FederatedResourceQuotaList, err error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	result = &v1beta1.FederatedResourceQuotaList{}
-	err = c.client.Get().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Watch returns a watch.Interface that watches the requested federatedResourceQuotas.
-func (c *federatedResourceQuotas) Watch(ctx context.Context, opts v1.ListOptions) (watch.Interface, error) {
-	var timeout time.Duration
-	if opts.TimeoutSeconds != nil {
-		timeout = time.Duration(*opts.TimeoutSeconds) * time.Second
-	}
-	opts.Watch = true
-	return c.client.Get().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Watch(ctx)
-}
-
-// Create takes the representation of a federatedResourceQuota and creates it.  Returns the server's representation of the federatedResourceQuota, and an error, if there is any.
-func (c *federatedResourceQuotas) Create(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.CreateOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	result = &v1beta1.FederatedResourceQuota{}
-	err = c.client.Post().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(federatedResourceQuota).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Update takes the representation of a federatedResourceQuota and updates it. Returns the server's representation of the federatedResourceQuota, and an error, if there is any.
-func (c *federatedResourceQuotas) Update(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	result = &v1beta1.FederatedResourceQuota{}
-	err = c.client.Put().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		Name(federatedResourceQuota.Name).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(federatedResourceQuota).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// UpdateStatus was generated because the type contains a Status member.
-// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus().
-func (c *federatedResourceQuotas) UpdateStatus(ctx context.Context, federatedResourceQuota *v1beta1.FederatedResourceQuota, opts v1.UpdateOptions) (result *v1beta1.FederatedResourceQuota, err error) {
-	result = &v1beta1.FederatedResourceQuota{}
-	err = c.client.Put().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		Name(federatedResourceQuota.Name).
-		SubResource("status").
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(federatedResourceQuota).
-		Do(ctx).
-		Into(result)
-	return
-}
-
-// Delete takes name of the federatedResourceQuota and deletes it. Returns an error if one occurs.
-func (c *federatedResourceQuotas) Delete(ctx context.Context, name string, opts v1.DeleteOptions) error {
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		Name(name).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// DeleteCollection deletes a collection of objects.
-func (c *federatedResourceQuotas) DeleteCollection(ctx context.Context, opts v1.DeleteOptions, listOpts v1.ListOptions) error {
-	var timeout time.Duration
-	if listOpts.TimeoutSeconds != nil {
-		timeout = time.Duration(*listOpts.TimeoutSeconds) * time.Second
-	}
-	return c.client.Delete().
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		VersionedParams(&listOpts, scheme.ParameterCodec).
-		Timeout(timeout).
-		Body(&opts).
-		Do(ctx).
-		Error()
-}
-
-// Patch applies the patch and returns the patched federatedResourceQuota.
-func (c *federatedResourceQuotas) Patch(ctx context.Context, name string, pt types.PatchType, data []byte, opts v1.PatchOptions, subresources ...string) (result *v1beta1.FederatedResourceQuota, err error) {
-	result = &v1beta1.FederatedResourceQuota{}
-	err = c.client.Patch(pt).
-		Namespace(c.ns).
-		Resource("federatedresourcequotas").
-		Name(name).
-		SubResource(subresources...).
-		VersionedParams(&opts, scheme.ParameterCodec).
-		Body(data).
-		Do(ctx).
-		Into(result)
-	return
-}
--- a/pkg/client/clientset/versioned/typed/types/v1beta1/generated_expansion.go
+++ b/pkg/client/clientset/versioned/typed/types/v1beta1/generated_expansion.go
@@ -42,8 +42,6 @@ type FederatedNamespaceExpansion interface{}

 type FederatedPersistentVolumeClaimExpansion interface{}

-type FederatedResourceQuotaExpansion interface{}
-
 type FederatedSecretExpansion interface{}

 type FederatedServiceExpansion interface{}
--- a/pkg/client/clientset/versioned/typed/types/v1beta1/types_client.go
+++ b/pkg/client/clientset/versioned/typed/types/v1beta1/types_client.go
@@ -38,7 +38,6 @@ type TypesV1beta1Interface interface {
 	FederatedLimitRangesGetter
 	FederatedNamespacesGetter
 	FederatedPersistentVolumeClaimsGetter
-	FederatedResourceQuotasGetter
 	FederatedSecretsGetter
 	FederatedServicesGetter
 	FederatedStatefulSetsGetter
@@ -97,10 +96,6 @@ func (c *TypesV1beta1Client) FederatedPersistentVolumeClaims(namespace string) F
 	return newFederatedPersistentVolumeClaims(c, namespace)
 }

-func (c *TypesV1beta1Client) FederatedResourceQuotas(namespace string) FederatedResourceQuotaInterface {
-	return newFederatedResourceQuotas(c, namespace)
-}
-
 func (c *TypesV1beta1Client) FederatedSecrets(namespace string) FederatedSecretInterface {
 	return newFederatedSecrets(c, namespace)
 }
--- a/pkg/client/informers/externalversions/generic.go
+++ b/pkg/client/informers/externalversions/generic.go
@@ -33,6 +33,7 @@ import (
 	v2beta1 "kubesphere.io/api/notification/v2beta1"
 	quotav1alpha2 "kubesphere.io/api/quota/v1alpha2"
 	servicemeshv1alpha2 "kubesphere.io/api/servicemesh/v1alpha2"
+	storagev1alpha1 "kubesphere.io/api/storage/v1alpha1"
 	tenantv1alpha1 "kubesphere.io/api/tenant/v1alpha1"
 	tenantv1alpha2 "kubesphere.io/api/tenant/v1alpha2"
 	v1beta1 "kubesphere.io/api/types/v1beta1"
@@ -148,6 +149,12 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 	case servicemeshv1alpha2.SchemeGroupVersion.WithResource("strategies"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Servicemesh().V1alpha2().Strategies().Informer()}, nil

+		// Group=storage.kubesphere.io, Version=v1alpha1
+	case storagev1alpha1.SchemeGroupVersion.WithResource("provisionercapabilities"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().ProvisionerCapabilities().Informer()}, nil
+	case storagev1alpha1.SchemeGroupVersion.WithResource("storageclasscapabilities"):
+		return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha1().StorageClassCapabilities().Informer()}, nil
+
 		// Group=tenant.kubesphere.io, Version=v1alpha1
 	case tenantv1alpha1.SchemeGroupVersion.WithResource("workspaces"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Tenant().V1alpha1().Workspaces().Informer()}, nil
@@ -181,8 +188,6 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Types().V1beta1().FederatedNamespaces().Informer()}, nil
 	case v1beta1.SchemeGroupVersion.WithResource("federatedpersistentvolumeclaims"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Types().V1beta1().FederatedPersistentVolumeClaims().Informer()}, nil
-	case v1beta1.SchemeGroupVersion.WithResource("federatedresourcequotas"):
-		return &genericInformer{resource: resource.GroupResource(), informer: f.Types().V1beta1().FederatedResourceQuotas().Informer()}, nil
 	case v1beta1.SchemeGroupVersion.WithResource("federatedsecrets"):
 		return &genericInformer{resource: resource.GroupResource(), informer: f.Types().V1beta1().FederatedSecrets().Informer()}, nil
 	case v1beta1.SchemeGroupVersion.WithResource("federatedservices"):
--- a/pkg/client/informers/externalversions/storage/v1alpha1/interface.go
+++ b/pkg/client/informers/externalversions/storage/v1alpha1/interface.go
@@ -24,6 +24,10 @@ import (

 // Interface provides access to all the informers in this group version.
 type Interface interface {
+	// ProvisionerCapabilities returns a ProvisionerCapabilityInformer.
+	ProvisionerCapabilities() ProvisionerCapabilityInformer
+	// StorageClassCapabilities returns a StorageClassCapabilityInformer.
+	StorageClassCapabilities() StorageClassCapabilityInformer
 }

 type version struct {
@@ -36,3 +40,13 @@ type version struct {
 func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface {
 	return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions}
 }
+
+// ProvisionerCapabilities returns a ProvisionerCapabilityInformer.
+func (v *version) ProvisionerCapabilities() ProvisionerCapabilityInformer {
+	return &provisionerCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}
+
+// StorageClassCapabilities returns a StorageClassCapabilityInformer.
+func (v *version) StorageClassCapabilities() StorageClassCapabilityInformer {
+	return &storageClassCapabilityInformer{factory: v.factory, tweakListOptions: v.tweakListOptions}
+}
--- a/pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go
+++ b/pkg/client/informers/externalversions/storage/v1alpha1/provisionercapability.go
@@ -0,0 +1,89 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by informer-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"context"
+	time "time"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	watch "k8s.io/apimachinery/pkg/watch"
+	cache "k8s.io/client-go/tools/cache"
+	storagev1alpha1 "kubesphere.io/api/storage/v1alpha1"
+	versioned "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	internalinterfaces "kubesphere.io/kubesphere/pkg/client/informers/externalversions/internalinterfaces"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1"
+)
+
+// ProvisionerCapabilityInformer provides access to a shared informer and lister for
+// ProvisionerCapabilities.
+type ProvisionerCapabilityInformer interface {
+	Informer() cache.SharedIndexInformer
+	Lister() v1alpha1.ProvisionerCapabilityLister
+}
+
+type provisionerCapabilityInformer struct {
+	factory          internalinterfaces.SharedInformerFactory
+	tweakListOptions internalinterfaces.TweakListOptionsFunc
+}
+
+// NewProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, indexers, nil)
+}
+
+// NewFilteredProvisionerCapabilityInformer constructs a new informer for ProvisionerCapability type.
+// Always prefer using an informer factory to get a shared informer instead of getting an independent
+// one. This reduces memory footprint and number of connections to the server.
+func NewFilteredProvisionerCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+	return cache.NewSharedIndexInformer(
+		&cache.ListWatch{
+			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.StorageV1alpha1().ProvisionerCapabilities().List(context.TODO(), options)
+			},
+			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
+				if tweakListOptions != nil {
+					tweakListOptions(&options)
+				}
+				return client.StorageV1alpha1().ProvisionerCapabilities().Watch(context.TODO(), options)
+			},
+		},
+		&storagev1alpha1.ProvisionerCapability{},
+		resyncPeriod,
+		indexers,
+	)
+}
+
+func (f *provisionerCapabilityInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredProvisionerCapabilityInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+}
+
+func (f *provisionerCapabilityInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&storagev1alpha1.ProvisionerCapability{}, f.defaultInformer)
+}
+
+func (f *provisionerCapabilityInformer) Lister() v1alpha1.ProvisionerCapabilityLister {
+	return v1alpha1.NewProvisionerCapabilityLister(f.Informer().GetIndexer())
+}
--- a/pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go
+++ b/pkg/client/informers/externalversions/storage/v1alpha1/storageclasscapability.go
@@ -16,7 +16,7 @@ limitations under the License.

 // Code generated by informer-gen. DO NOT EDIT.

-package v1beta1
+package v1alpha1

 import (
 	"context"
@@ -26,65 +26,64 @@ import (
 	runtime "k8s.io/apimachinery/pkg/runtime"
 	watch "k8s.io/apimachinery/pkg/watch"
 	cache "k8s.io/client-go/tools/cache"
-	typesv1beta1 "kubesphere.io/api/types/v1beta1"
+	storagev1alpha1 "kubesphere.io/api/storage/v1alpha1"
 	versioned "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
 	internalinterfaces "kubesphere.io/kubesphere/pkg/client/informers/externalversions/internalinterfaces"
-	v1beta1 "kubesphere.io/kubesphere/pkg/client/listers/types/v1beta1"
+	v1alpha1 "kubesphere.io/kubesphere/pkg/client/listers/storage/v1alpha1"
 )

-// FederatedResourceQuotaInformer provides access to a shared informer and lister for
-// FederatedResourceQuotas.
-type FederatedResourceQuotaInformer interface {
+// StorageClassCapabilityInformer provides access to a shared informer and lister for
+// StorageClassCapabilities.
+type StorageClassCapabilityInformer interface {
 	Informer() cache.SharedIndexInformer
-	Lister() v1beta1.FederatedResourceQuotaLister
+	Lister() v1alpha1.StorageClassCapabilityLister
 }

-type federatedResourceQuotaInformer struct {
+type storageClassCapabilityInformer struct {
 	factory          internalinterfaces.SharedInformerFactory
 	tweakListOptions internalinterfaces.TweakListOptionsFunc
-	namespace        string
 }

-// NewFederatedResourceQuotaInformer constructs a new informer for FederatedResourceQuota type.
+// NewStorageClassCapabilityInformer constructs a new informer for StorageClassCapability type.
 // Always prefer using an informer factory to get a shared informer instead of getting an independent
 // one. This reduces memory footprint and number of connections to the server.
-func NewFederatedResourceQuotaInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
-	return NewFilteredFederatedResourceQuotaInformer(client, namespace, resyncPeriod, indexers, nil)
+func NewStorageClassCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer {
+	return NewFilteredStorageClassCapabilityInformer(client, resyncPeriod, indexers, nil)
 }

-// NewFilteredFederatedResourceQuotaInformer constructs a new informer for FederatedResourceQuota type.
+// NewFilteredStorageClassCapabilityInformer constructs a new informer for StorageClassCapability type.
 // Always prefer using an informer factory to get a shared informer instead of getting an independent
 // one. This reduces memory footprint and number of connections to the server.
-func NewFilteredFederatedResourceQuotaInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
+func NewFilteredStorageClassCapabilityInformer(client versioned.Interface, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer {
 	return cache.NewSharedIndexInformer(
 		&cache.ListWatch{
 			ListFunc: func(options v1.ListOptions) (runtime.Object, error) {
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.TypesV1beta1().FederatedResourceQuotas(namespace).List(context.TODO(), options)
+				return client.StorageV1alpha1().StorageClassCapabilities().List(context.TODO(), options)
 			},
 			WatchFunc: func(options v1.ListOptions) (watch.Interface, error) {
 				if tweakListOptions != nil {
 					tweakListOptions(&options)
 				}
-				return client.TypesV1beta1().FederatedResourceQuotas(namespace).Watch(context.TODO(), options)
+				return client.StorageV1alpha1().StorageClassCapabilities().Watch(context.TODO(), options)
 			},
 		},
-		&typesv1beta1.FederatedResourceQuota{},
+		&storagev1alpha1.StorageClassCapability{},
 		resyncPeriod,
 		indexers,
 	)
 }

-func (f *federatedResourceQuotaInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
-	return NewFilteredFederatedResourceQuotaInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
+func (f *storageClassCapabilityInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer {
+	return NewFilteredStorageClassCapabilityInformer(client, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions)
 }

-func (f *federatedResourceQuotaInformer) Informer() cache.SharedIndexInformer {
-	return f.factory.InformerFor(&typesv1beta1.FederatedResourceQuota{}, f.defaultInformer)
+func (f *storageClassCapabilityInformer) Informer() cache.SharedIndexInformer {
+	return f.factory.InformerFor(&storagev1alpha1.StorageClassCapability{}, f.defaultInformer)
 }

-func (f *federatedResourceQuotaInformer) Lister() v1beta1.FederatedResourceQuotaLister {
-	return v1beta1.NewFederatedResourceQuotaLister(f.Informer().GetIndexer())
+func (f *storageClassCapabilityInformer) Lister() v1alpha1.StorageClassCapabilityLister {
+	return v1alpha1.NewStorageClassCapabilityLister(f.Informer().GetIndexer())
 }
--- a/pkg/client/informers/externalversions/types/v1beta1/interface.go
+++ b/pkg/client/informers/externalversions/types/v1beta1/interface.go
@@ -48,8 +48,6 @@ type Interface interface {
 	FederatedNamespaces() FederatedNamespaceInformer
 	// FederatedPersistentVolumeClaims returns a FederatedPersistentVolumeClaimInformer.
 	FederatedPersistentVolumeClaims() FederatedPersistentVolumeClaimInformer
-	// FederatedResourceQuotas returns a FederatedResourceQuotaInformer.
-	FederatedResourceQuotas() FederatedResourceQuotaInformer
 	// FederatedSecrets returns a FederatedSecretInformer.
 	FederatedSecrets() FederatedSecretInformer
 	// FederatedServices returns a FederatedServiceInformer.
@@ -129,11 +127,6 @@ func (v *version) FederatedPersistentVolumeClaims() FederatedPersistentVolumeCla
 	return &federatedPersistentVolumeClaimInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
 }

-// FederatedResourceQuotas returns a FederatedResourceQuotaInformer.
-func (v *version) FederatedResourceQuotas() FederatedResourceQuotaInformer {
-	return &federatedResourceQuotaInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
-}
-
 // FederatedSecrets returns a FederatedSecretInformer.
 func (v *version) FederatedSecrets() FederatedSecretInformer {
 	return &federatedSecretInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions}
--- a/pkg/client/listers/application/v1alpha1/helmapplication.go
+++ b/pkg/client/listers/application/v1alpha1/helmapplication.go
@@ -26,10 +26,13 @@ import (
 )

 // HelmApplicationLister helps list HelmApplications.
+// All objects returned here must be treated as read-only.
 type HelmApplicationLister interface {
 	// List lists all HelmApplications in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1alpha1.HelmApplication, err error)
 	// Get retrieves the HelmApplication from the index for a given name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1alpha1.HelmApplication, error)
 	HelmApplicationListerExpansion
 }
--- a/pkg/client/listers/application/v1alpha1/helmapplicationversion.go
+++ b/pkg/client/listers/application/v1alpha1/helmapplicationversion.go
@@ -26,10 +26,13 @@ import (
 )

 // HelmApplicationVersionLister helps list HelmApplicationVersions.
+// All objects returned here must be treated as read-only.
 type HelmApplicationVersionLister interface {
 	// List lists all HelmApplicationVersions in the indexer.
+	// Objects returned here must be treated as read-only.
 	List(selector labels.Selector) (ret []*v1alpha1.HelmApplicationVersion, err error)
 	// Get retrieves the HelmApplicationVersion from the index for a given name.
+	// Objects returned here must be treated as read-only.
 	Get(name string) (*v1alpha1.HelmApplicationVersion, error)
 	HelmApplicationVersionListerExpansion
 }
--- a/Show More
+++ b/Show More