Merge pull request #2195 from zheng1/refactor_op

Refactor with OpenPitrix
2020-06-15 11:06:40 +08:00
parent ce55e44ee9 3778abf767
commit 61d827db54
122 changed files with 10393 additions and 17233 deletions
--- a/pkg/constants/constants.go
+++ b/pkg/constants/constants.go
@@ -34,6 +34,7 @@ const (

 	WorkspaceLabelKey              = "kubesphere.io/workspace"
 	NamespaceLabelKey              = "kubesphere.io/namespace"
+	RuntimeLabelKey                = "openpitrix.io/namespace"
 	DisplayNameAnnotationKey       = "kubesphere.io/alias-name"
 	DescriptionAnnotationKey       = "kubesphere.io/description"
 	CreatorAnnotationKey           = "kubesphere.io/creator"
--- a/pkg/controller/namespace/namespace_controller.go
+++ b/pkg/controller/namespace/namespace_controller.go
@@ -18,8 +18,6 @@ package namespace

 import (
 	"context"
-	"fmt"
-	"github.com/golang/protobuf/ptypes/wrappers"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -29,9 +27,7 @@ import (
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1"
 	"kubesphere.io/kubesphere/pkg/constants"
-	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
-	"openpitrix.io/openpitrix/pkg/pb"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -48,16 +44,15 @@ import (

 // Add creates a new Namespace Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller
 // and Start it when the Manager is Started.
-func Add(mgr manager.Manager, openpitrixClient openpitrix.Client) error {
-	return add(mgr, newReconciler(mgr, openpitrixClient))
+func Add(mgr manager.Manager) error {
+	return add(mgr, newReconciler(mgr))
 }

 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, openpitrixClient openpitrix.Client) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager) reconcile.Reconciler {
 	return &ReconcileNamespace{
-		Client:           mgr.GetClient(),
-		scheme:           mgr.GetScheme(),
-		openpitrixClient: openpitrixClient,
+		Client: mgr.GetClient(),
+		scheme: mgr.GetScheme(),
 	}
 }

@@ -83,8 +78,7 @@ var _ reconcile.Reconciler = &ReconcileNamespace{}
 // ReconcileNamespace reconciles a Namespace object
 type ReconcileNamespace struct {
 	client.Client
-	openpitrixClient openpitrix.Client
-	scheme           *runtime.Scheme
+	scheme *runtime.Scheme
 }

 // Reconcile reads that state of the cluster for a Namespace object and makes changes based on the state read
@@ -130,11 +124,6 @@ func (r *ReconcileNamespace) Reconcile(request reconcile.Request) (reconcile.Res
 				return reconcile.Result{}, err
 			}

-			// delete runtime
-			if err = r.deleteRuntime(instance); err != nil {
-				return reconcile.Result{}, err
-			}
-
 			// remove our finalizer from the list and update it.
 			instance.ObjectMeta.Finalizers = sliceutil.RemoveString(instance.ObjectMeta.Finalizers, func(item string) bool {
 				return item == finalizer
@@ -153,13 +142,6 @@ func (r *ReconcileNamespace) Reconcile(request reconcile.Request) (reconcile.Res
 		return reconcile.Result{}, err
 	}

-	// skip if openpitrix is not enabled
-	if r.openpitrixClient != nil {
-		if err := r.checkAndCreateRuntime(instance); err != nil {
-			return reconcile.Result{}, err
-		}
-	}
-
 	return reconcile.Result{}, nil
 }

@@ -175,88 +157,6 @@ func (r *ReconcileNamespace) isControlledByWorkspace(namespace *corev1.Namespace
 	return true, nil
 }

-// Create openpitrix runtime
-func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace) error {
-
-	if runtimeId := namespace.Annotations[constants.OpenPitrixRuntimeAnnotationKey]; runtimeId != "" {
-		return nil
-	}
-
-	adminKubeConfigName := fmt.Sprintf("kubeconfig-%s", constants.AdminUserName)
-
-	runtimeCredentials, err := r.openpitrixClient.DescribeRuntimeCredentials(openpitrix.SystemContext(), &pb.DescribeRuntimeCredentialsRequest{SearchWord: &wrappers.StringValue{Value: adminKubeConfigName}, Limit: 1})
-
-	if err != nil {
-		klog.Error(fmt.Sprintf("create runtime, namespace: %s, error: %s", namespace.Name, err))
-		return err
-	}
-
-	var kubesphereRuntimeCredentialId string
-
-	// runtime credential exist
-	if len(runtimeCredentials.GetRuntimeCredentialSet()) > 0 {
-		kubesphereRuntimeCredentialId = runtimeCredentials.GetRuntimeCredentialSet()[0].GetRuntimeCredentialId().GetValue()
-	} else {
-		adminKubeConfig := corev1.ConfigMap{}
-		err := r.Get(context.TODO(), types.NamespacedName{Namespace: constants.KubeSphereControlNamespace, Name: adminKubeConfigName}, &adminKubeConfig)
-
-		if err != nil {
-			klog.Error(fmt.Sprintf("create runtime, namespace: %s, error: %s", namespace.Name, err))
-			return err
-		}
-
-		resp, err := r.openpitrixClient.CreateRuntimeCredential(openpitrix.SystemContext(), &pb.CreateRuntimeCredentialRequest{
-			Name:                     &wrappers.StringValue{Value: adminKubeConfigName},
-			Provider:                 &wrappers.StringValue{Value: "kubernetes"},
-			Description:              &wrappers.StringValue{Value: "kubeconfig"},
-			RuntimeUrl:               &wrappers.StringValue{Value: "kubesphere"},
-			RuntimeCredentialContent: &wrappers.StringValue{Value: adminKubeConfig.Data["config"]},
-		})
-
-		if err != nil {
-			klog.Error(fmt.Sprintf("create runtime, namespace: %s, error: %s", namespace.Name, err))
-			return err
-		}
-
-		kubesphereRuntimeCredentialId = resp.GetRuntimeCredentialId().GetValue()
-	}
-
-	// TODO runtime id is invalid when recreate runtime
-	runtimeId, err := r.openpitrixClient.CreateRuntime(openpitrix.SystemContext(), &pb.CreateRuntimeRequest{
-		Name:                &wrappers.StringValue{Value: namespace.Name},
-		RuntimeCredentialId: &wrappers.StringValue{Value: kubesphereRuntimeCredentialId},
-		Provider:            &wrappers.StringValue{Value: openpitrix.KubernetesProvider},
-		Zone:                &wrappers.StringValue{Value: namespace.Name},
-	})
-
-	if err != nil {
-		klog.Error(fmt.Sprintf("create runtime, namespace: %s, error: %s", namespace.Name, err))
-		return err
-	}
-
-	klog.V(4).Infof("runtime created successfully, namespace: %s, runtime id: %s", namespace.Name, runtimeId)
-
-	return nil
-}
-
-// Delete openpitrix runtime
-func (r *ReconcileNamespace) deleteRuntime(namespace *corev1.Namespace) error {
-
-	if runtimeId := namespace.Annotations[constants.OpenPitrixRuntimeAnnotationKey]; runtimeId != "" {
-		_, err := r.openpitrixClient.DeleteRuntimes(openpitrix.SystemContext(), &pb.DeleteRuntimesRequest{RuntimeId: []string{runtimeId}, Force: &wrappers.BoolValue{Value: true}})
-
-		if err == nil || openpitrix.IsNotFound(err) || openpitrix.IsDeleted(err) {
-			return nil
-		} else {
-			klog.Errorf("delete openpitrix runtime: %s, error: %s", runtimeId, err)
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Create openpitrix runtime
 func (r *ReconcileNamespace) checkAndBindWorkspace(namespace *corev1.Namespace) error {

 	workspaceName := namespace.Labels[constants.WorkspaceLabelKey]
--- a/pkg/kapis/openpitrix/v1/handler.go
+++ b/pkg/kapis/openpitrix/v1/handler.go
@@ -1,3 +1,16 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package v1

 import (
@@ -10,7 +23,6 @@ import (
 	"kubesphere.io/kubesphere/pkg/api"
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/informers"
-	"kubesphere.io/kubesphere/pkg/models"
 	"kubesphere.io/kubesphere/pkg/models/openpitrix"
 	"kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/server/params"
@@ -32,78 +44,44 @@ func newOpenpitrixHandler(factory informers.InformerFactory, opClient op.Client)
 	}
 }

-func (h *openpitrixHandler) ListApplications(request *restful.Request, response *restful.Response) {
-	limit, offset := params.ParsePaging(request)
-	namespace := request.PathParameter("namespace")
-	orderBy := params.GetStringValueWithDefault(request, params.OrderByParam, openpitrix.CreateTime)
-	reverse := params.GetBoolValueWithDefault(request, params.ReverseParam, false)
-	conditions, err := params.ParseConditions(request)
+func (h *openpitrixHandler) ListApplications(req *restful.Request, resp *restful.Response) {
+	limit, offset := params.ParsePaging(req)
+	clusterName := req.PathParameter("cluster")
+	namespace := req.PathParameter("namespace")
+	orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, openpitrix.CreateTime)
+	reverse := params.GetBoolValueWithDefault(req, params.ReverseParam, false)
+	conditions, err := params.ParseConditions(req)

 	if err != nil {
 		klog.V(4).Infoln(err)
-		api.HandleBadRequest(response, nil, err)
+		api.HandleBadRequest(resp, nil, err)
 		return
 	}
-
-	// filter namespaced applications by runtime_id
-	if namespace != "" {
-		ns, err := h.informers.Core().V1().Namespaces().Lister().Get(namespace)
-
-		if err != nil {
-			klog.Errorln(err)
-			api.HandleInternalError(response, nil, err)
-			return
-		}
-
-		runtimeId := ns.Annotations[constants.OpenPitrixRuntimeAnnotationKey]
-
-		if runtimeId == "" {
-			// runtime id not exist,return empty response
-			response.WriteAsJson(models.PageableResponse{Items: []interface{}{}, TotalCount: 0})
-			return
-		} else {
-			// filter by runtime id
-			conditions.Match[openpitrix.RuntimeId] = runtimeId
-		}
-	}
+	conditions.Match[openpitrix.Zone] = namespace
+	// in openpitrix, runtime id is the cluster name
+	conditions.Match[openpitrix.RuntimeId] = clusterName

 	result, err := h.openpitrix.ListApplications(conditions, limit, offset, orderBy, reverse)

-	if err != nil {
-		klog.Errorln(err)
-		api.HandleInternalError(response, nil, err)
-		return
-	}
-
-	response.WriteAsJson(result)
-}
-
-func (h *openpitrixHandler) DescribeApplication(req *restful.Request, resp *restful.Response) {
-	clusterId := req.PathParameter("application")
-	namespace := req.PathParameter("namespace")
-
-	app, err := h.openpitrix.DescribeApplication(namespace, clusterId)
-
-	if err != nil {
-		klog.Errorln(err)
-		handleOpenpitrixError(resp, err)
-		return
-	}
-
-	ns, err := h.informers.Core().V1().Namespaces().Lister().Get(namespace)
-
 	if err != nil {
 		klog.Errorln(err)
 		api.HandleInternalError(resp, nil, err)
 		return
 	}

-	runtimeId := ns.Annotations[constants.OpenPitrixRuntimeAnnotationKey]
+	resp.WriteAsJson(result)
+}

-	if runtimeId != app.Cluster.RuntimeId {
-		err = fmt.Errorf("rumtime not match %s,%s", app.Cluster.RuntimeId, runtimeId)
-		klog.V(4).Infoln(err)
-		api.HandleForbidden(resp, nil, err)
+func (h *openpitrixHandler) DescribeApplication(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster")
+	namespace := req.PathParameter("namespace")
+	applicationId := req.PathParameter("application")
+
+	app, err := h.openpitrix.DescribeApplication(namespace, applicationId, clusterName)
+
+	if err != nil {
+		klog.Errorln(err)
+		handleOpenpitrixError(resp, err)
 		return
 	}

@@ -112,6 +90,7 @@ func (h *openpitrixHandler) DescribeApplication(req *restful.Request, resp *rest
 }

 func (h *openpitrixHandler) CreateApplication(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster")
 	namespace := req.PathParameter("namespace")
 	var createClusterRequest openpitrix.CreateClusterRequest
 	err := req.ReadEntity(&createClusterRequest)
@@ -123,7 +102,7 @@ func (h *openpitrixHandler) CreateApplication(req *restful.Request, resp *restfu

 	createClusterRequest.Username = req.HeaderParameter(constants.UserNameHeader)

-	err = h.openpitrix.CreateApplication(namespace, createClusterRequest)
+	err = h.openpitrix.CreateApplication(clusterName, namespace, createClusterRequest)

 	if err != nil {
 		klog.Errorln(err)
@@ -136,7 +115,8 @@ func (h *openpitrixHandler) CreateApplication(req *restful.Request, resp *restfu

 func (h *openpitrixHandler) ModifyApplication(req *restful.Request, resp *restful.Response) {
 	var modifyClusterAttributesRequest openpitrix.ModifyClusterAttributesRequest
-	clusterId := req.PathParameter("application")
+	clusterName := req.PathParameter("cluster")
+	applicationId := req.PathParameter("application")
 	namespace := req.PathParameter("namespace")
 	err := req.ReadEntity(&modifyClusterAttributesRequest)
 	if err != nil {
@@ -145,7 +125,7 @@ func (h *openpitrixHandler) ModifyApplication(req *restful.Request, resp *restfu
 		return
 	}

-	app, err := h.openpitrix.DescribeApplication(namespace, clusterId)
+	app, err := h.openpitrix.DescribeApplication(namespace, applicationId, clusterName)

 	if err != nil {
 		klog.Errorln(err)
@@ -153,18 +133,8 @@ func (h *openpitrixHandler) ModifyApplication(req *restful.Request, resp *restfu
 		return
 	}

-	ns, err := h.informers.Core().V1().Namespaces().Lister().Get(namespace)
-
-	if err != nil {
-		klog.Errorln(err)
-		api.HandleInternalError(resp, nil, err)
-		return
-	}
-
-	runtimeId := ns.Annotations[constants.OpenPitrixRuntimeAnnotationKey]
-
-	if runtimeId != app.Cluster.RuntimeId {
-		err = fmt.Errorf("rumtime not match %s,%s", app.Cluster.RuntimeId, runtimeId)
+	if clusterName != app.Cluster.RuntimeId {
+		err = fmt.Errorf("runtime and cluster not match %s,%s", app.Cluster.RuntimeId, clusterName)
 		klog.V(4).Infoln(err)
 		api.HandleForbidden(resp, nil, err)
 		return
@@ -182,9 +152,10 @@ func (h *openpitrixHandler) ModifyApplication(req *restful.Request, resp *restfu
 }

 func (h *openpitrixHandler) DeleteApplication(req *restful.Request, resp *restful.Response) {
-	clusterId := req.PathParameter("application")
+	clusterName := req.PathParameter("cluster")
+	applicationId := req.PathParameter("application")
 	namespace := req.PathParameter("namespace")
-	app, err := h.openpitrix.DescribeApplication(namespace, clusterId)
+	app, err := h.openpitrix.DescribeApplication(namespace, applicationId, clusterName)

 	if err != nil {
 		klog.Errorln(err)
@@ -192,24 +163,14 @@ func (h *openpitrixHandler) DeleteApplication(req *restful.Request, resp *restfu
 		return
 	}

-	ns, err := h.informers.Core().V1().Namespaces().Lister().Get(namespace)
-
-	if err != nil {
-		klog.Errorln(err)
-		api.HandleInternalError(resp, nil, err)
-		return
-	}
-
-	runtimeId := ns.Annotations[constants.OpenPitrixRuntimeAnnotationKey]
-
-	if runtimeId != app.Cluster.RuntimeId {
-		err = fmt.Errorf("rumtime not match %s,%s", app.Cluster.RuntimeId, runtimeId)
+	if clusterName != app.Cluster.RuntimeId {
+		err = fmt.Errorf("runtime and cluster not match %s,%s", app.Cluster.RuntimeId, clusterName)
 		klog.V(4).Infoln(err)
 		api.HandleForbidden(resp, nil, err)
 		return
 	}

-	err = h.openpitrix.DeleteApplication(clusterId)
+	err = h.openpitrix.DeleteApplication(applicationId)

 	if err != nil {
 		klog.Errorln(err)
@@ -220,6 +181,46 @@ func (h *openpitrixHandler) DeleteApplication(req *restful.Request, resp *restfu
 	resp.WriteEntity(errors.None)
 }

+func (h *openpitrixHandler) UpgradeApplication(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster")
+	namespace := req.PathParameter("namespace")
+	applicationId := req.PathParameter("application")
+	var upgradeClusterRequest openpitrix.UpgradeClusterRequest
+	err := req.ReadEntity(&upgradeClusterRequest)
+	if err != nil {
+		klog.V(4).Infoln(err)
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	upgradeClusterRequest.Username = req.HeaderParameter(constants.UserNameHeader)
+
+	app, err := h.openpitrix.DescribeApplication(namespace, applicationId, clusterName)
+
+	if err != nil {
+		klog.Errorln(err)
+		handleOpenpitrixError(resp, err)
+		return
+	}
+
+	if clusterName != app.Cluster.RuntimeId {
+		err = fmt.Errorf("runtime and cluster not match %s,%s", app.Cluster.RuntimeId, clusterName)
+		klog.V(4).Infoln(err)
+		api.HandleForbidden(resp, nil, err)
+		return
+	}
+
+	err = h.openpitrix.UpgradeApplication(upgradeClusterRequest)
+
+	if err != nil {
+		klog.Errorln(err)
+		api.HandleInternalError(resp, nil, err)
+		return
+	}
+
+	resp.WriteEntity(errors.None)
+}
+
 func (h *openpitrixHandler) GetAppVersionPackage(req *restful.Request, resp *restful.Response) {
 	appId := req.PathParameter("app")
 	versionId := req.PathParameter("version")
@@ -833,6 +834,7 @@ func (h *openpitrixHandler) DescribeRepo(req *restful.Request, resp *restful.Res

 	resp.WriteEntity(result)
 }
+
 func (h *openpitrixHandler) ListRepos(req *restful.Request, resp *restful.Response) {
 	limit, offset := params.ParsePaging(req)
 	orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, openpitrix.CreateTime)
@@ -856,6 +858,29 @@ func (h *openpitrixHandler) ListRepos(req *restful.Request, resp *restful.Respon
 	resp.WriteEntity(result)
 }

+func (h *openpitrixHandler) ListEvents(req *restful.Request, resp *restful.Response) {
+	limit, offset := params.ParsePaging(req)
+	orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, openpitrix.CreateTime)
+	reverse := params.GetBoolValueWithDefault(req, params.ReverseParam, false)
+	conditions, err := params.ParseConditions(req)
+
+	if err != nil {
+		klog.V(4).Infoln(err)
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	result, err := h.openpitrix.ListEvents(conditions, orderBy, reverse, limit, offset)
+
+	if err != nil {
+		klog.Errorln(err)
+		handleOpenpitrixError(resp, err)
+		return
+	}
+
+	resp.WriteEntity(result)
+}
+
 func (h *openpitrixHandler) ListRepoEvents(req *restful.Request, resp *restful.Response) {
 	repoId := req.PathParameter("repo")
 	limit, offset := params.ParsePaging(req)
--- a/pkg/kapis/openpitrix/v1/register.go
+++ b/pkg/kapis/openpitrix/v1/register.go
@@ -1,18 +1,16 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
+Copyright 2020 The KubeSphere Authors.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+
 package v1

 import (
@@ -54,13 +52,27 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, op
 			Required(false).
 			DataFormat("key=value,key~value").
 			DefaultValue("")).
-		Param(webservice.PathParameter("namespace", "the name of the project")).
 		Param(webservice.QueryParameter(params.PagingParam, "paging query, e.g. limit=100,page=1").
 			Required(false).
 			DataFormat("limit=%d,page=%d").
 			DefaultValue("limit=10,page=1")))

-	webservice.Route(webservice.GET("/namespaces/{namespace}/applications").
+	webservice.Route(webservice.GET("/clusters/{cluster}/applications").
+		To(handler.ListApplications).
+		Returns(http.StatusOK, api.StatusOK, models.PageableResponse{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
+		Doc("List all applications in special cluster").
+		Param(webservice.QueryParameter(params.ConditionsParam, "query conditions, connect multiple conditions with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").
+			Required(false).
+			DataFormat("key=value,key~value").
+			DefaultValue("")).
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.QueryParameter(params.PagingParam, "paging query, e.g. limit=100,page=1").
+			Required(false).
+			DataFormat("limit=%d,page=%d").
+			DefaultValue("limit=10,page=1")))
+
+	webservice.Route(webservice.GET("/clusters/{cluster}/namespaces/{namespace}/applications").
 		To(handler.ListApplications).
 		Returns(http.StatusOK, api.StatusOK, models.PageableResponse{}).
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
@@ -69,45 +81,61 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, op
 			Required(false).
 			DataFormat("key=value,key~value").
 			DefaultValue("")).
-		Param(webservice.PathParameter("namespace", "the name of the project")).
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)).
 		Param(webservice.QueryParameter(params.PagingParam, "paging query, e.g. limit=100,page=1").
 			Required(false).
 			DataFormat("limit=%d,page=%d").
 			DefaultValue("limit=10,page=1")))

-	webservice.Route(webservice.GET("/namespaces/{namespace}/applications/{application}").
+	webservice.Route(webservice.GET("/clusters/{cluster}/namespaces/{namespace}/applications/{application}").
 		To(handler.DescribeApplication).
 		Returns(http.StatusOK, api.StatusOK, openpitrix2.Application{}).
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
 		Doc("Describe the specified application of the namespace").
-		Param(webservice.PathParameter("namespace", "the name of the project")).
-		Param(webservice.PathParameter("application", "application ID")))
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)).
+		Param(webservice.PathParameter("application", "the id of the application").Required(true)))

-	webservice.Route(webservice.POST("/namespaces/{namespace}/applications").
+	webservice.Route(webservice.POST("/clusters/{cluster}/namespaces/{namespace}/applications").
 		To(handler.CreateApplication).
 		Doc("Deploy a new application").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
 		Reads(openpitrix2.CreateClusterRequest{}).
 		Returns(http.StatusOK, api.StatusOK, errors.Error{}).
-		Param(webservice.PathParameter("namespace", "the name of the project")))
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)))

-	webservice.Route(webservice.PATCH("/namespaces/{namespace}/applications/{application}").
+	webservice.Route(webservice.PATCH("/clusters/{cluster}/namespaces/{namespace}/applications/{application}").
 		Consumes(mimePatch...).
 		To(handler.ModifyApplication).
 		Doc("Modify application").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
 		Reads(openpitrix2.ModifyClusterAttributesRequest{}).
 		Returns(http.StatusOK, api.StatusOK, errors.Error{}).
-		Param(webservice.PathParameter("namespace", "the name of the project")).
-		Param(webservice.PathParameter("application", "the id of the application cluster")))
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)).
+		Param(webservice.PathParameter("application", "the id of the application").Required(true)))

-	webservice.Route(webservice.DELETE("/namespaces/{namespace}/applications/{application}").
+	webservice.Route(webservice.DELETE("/clusters/{cluster}/namespaces/{namespace}/applications/{application}").
 		To(handler.DeleteApplication).
 		Doc("Delete the specified application").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
 		Returns(http.StatusOK, api.StatusOK, errors.Error{}).
-		Param(webservice.PathParameter("namespace", "the name of the project")).
-		Param(webservice.PathParameter("application", "the id of the application cluster")))
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)).
+		Param(webservice.PathParameter("application", "the id of the application").Required(true)))
+
+	webservice.Route(webservice.POST("/clusters/{cluster}/namespaces/{namespace}/applications/{application}").
+		Consumes(mimePatch...).
+		To(handler.UpgradeApplication).
+		Doc("Upgrade application").
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
+		Reads(openpitrix2.UpgradeClusterRequest{}).
+		Returns(http.StatusOK, api.StatusOK, errors.Error{}).
+		Param(webservice.PathParameter("cluster", "the name of the cluster.").Required(true)).
+		Param(webservice.PathParameter("namespace", "the name of the project").Required(true)).
+		Param(webservice.PathParameter("application", "the id of the application").Required(true)))

 	webservice.Route(webservice.POST("/apps/{app}/versions").
 		To(handler.CreateAppVersion).
@@ -333,6 +361,13 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, op
 		Reads(openpitrix2.RepoActionRequest{}).
 		Returns(http.StatusOK, api.StatusOK, errors.Error{}).
 		Param(webservice.PathParameter("repo", "repo id")))
+	webservice.Route(webservice.GET("/events").
+		To(handler.ListEvents).
+		Doc("Get events").
+		Param(webservice.QueryParameter(params.ConditionsParam, "query conditions,connect multiple conditions with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").
+			Required(false).
+			DataFormat("key=%s,key~%s")).
+		Returns(http.StatusOK, api.StatusOK, models.PageableResponse{}))
 	webservice.Route(webservice.GET("/repos/{repo}/events").
 		To(handler.ListRepoEvents).
 		Doc("Get repository events").
--- a/pkg/models/openpitrix/applications.go
+++ b/pkg/models/openpitrix/applications.go
@@ -1,22 +1,20 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+
 package openpitrix

 import (
-	"fmt"
+	"encoding/json"
 	"github.com/golang/protobuf/ptypes/wrappers"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -27,20 +25,21 @@ import (
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/client-go/informers"
 	"k8s.io/klog"
-	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/models"
 	"kubesphere.io/kubesphere/pkg/server/params"
 	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"openpitrix.io/openpitrix/pkg/pb"
+	"openpitrix.io/openpitrix/pkg/util/pbutil"
 	"strings"
 )

 type ApplicationInterface interface {
 	ListApplications(conditions *params.Conditions, limit, offset int, orderBy string, reverse bool) (*models.PageableResponse, error)
-	DescribeApplication(namespace, clusterId string) (*Application, error)
-	CreateApplication(namespace string, request CreateClusterRequest) error
+	DescribeApplication(namespace, applicationId, clusterName string) (*Application, error)
+	CreateApplication(clusterName, namespace string, request CreateClusterRequest) error
 	ModifyApplication(request ModifyClusterAttributesRequest) error
 	DeleteApplication(id string) error
+	UpgradeApplication(request UpgradeClusterRequest) error
 }

 type applicationOperator struct {
@@ -68,6 +67,14 @@ type workLoads struct {
 	Daemonsets   []appsv1.DaemonSet   `json:"daemonsets,omitempty" description:"daemonset list"`
 }

+type resourceInfo struct {
+	Deployments  []appsv1.Deployment  `json:"deployments,omitempty" description:"deployment list"`
+	Statefulsets []appsv1.StatefulSet `json:"statefulsets,omitempty" description:"statefulset list"`
+	Daemonsets   []appsv1.DaemonSet   `json:"daemonsets,omitempty" description:"daemonset list"`
+	Services     []v1.Service         `json:"services,omitempty" description:"application services"`
+	Ingresses    []v1beta1.Ingress    `json:"ingresses,omitempty" description:"application ingresses"`
+}
+
 func (c *applicationOperator) ListApplications(conditions *params.Conditions, limit, offset int, orderBy string, reverse bool) (*models.PageableResponse, error) {
 	describeClustersRequest := &pb.DescribeClustersRequest{
 		Limit:  uint32(limit),
@@ -87,6 +94,9 @@ func (c *applicationOperator) ListApplications(conditions *params.Conditions, li
 	if status := conditions.Match[Status]; status != "" {
 		describeClustersRequest.Status = strings.Split(status, "|")
 	}
+	if zone := conditions.Match[Zone]; zone != "" {
+		describeClustersRequest.Zone = []string{zone}
+	}
 	if orderBy != "" {
 		describeClustersRequest.SortKey = &wrappers.StringValue{Value: orderBy}
 	}
@@ -133,9 +143,15 @@ func (c *applicationOperator) describeApplication(cluster *pb.Cluster) (*Applica
 	return &app, nil
 }

-func (c *applicationOperator) DescribeApplication(namespace string, clusterId string) (*Application, error) {
-
-	clusters, err := c.opClient.DescribeClusters(openpitrix.SystemContext(), &pb.DescribeClustersRequest{ClusterId: []string{clusterId}, Limit: 1})
+func (c *applicationOperator) DescribeApplication(namespace string, applicationId string, clusterName string) (*Application, error) {
+	describeClusterRequest := &pb.DescribeClustersRequest{
+		ClusterId:  []string{applicationId},
+		RuntimeId:  []string{clusterName},
+		Zone:       []string{namespace},
+		WithDetail: pbutil.ToProtoBool(true),
+		Limit:      1,
+	}
+	clusters, err := c.opClient.DescribeClusters(openpitrix.SystemContext(), describeClusterRequest)

 	if err != nil {
 		klog.Errorln(err)
@@ -156,16 +172,26 @@ func (c *applicationOperator) DescribeApplication(namespace string, clusterId st
 		return nil, err
 	}

-	workloads, err := c.getWorkLoads(namespace, cluster.ClusterRoleSet)
-
+	resource := new(resourceInfo)
+	workloads := cluster.AdditionalInfo.GetValue()
+	if workloads == "" {
+		err := status.New(codes.NotFound, "cannot get workload").Err()
+		klog.Errorln(err)
+		return nil, err
+	}
+	err = json.Unmarshal([]byte(workloads), resource)
 	if err != nil {
 		klog.Errorln(err)
 		return nil, err
 	}
-	app.WorkLoads = workloads
-	workloadLabels := c.getLabels(namespace, app.WorkLoads)
-	app.Services = c.getSvcs(namespace, workloadLabels)
-	app.Ingresses = c.getIng(namespace, app.Services)
+
+	app.WorkLoads = &workLoads{
+		Deployments:  resource.Deployments,
+		Statefulsets: resource.Statefulsets,
+		Daemonsets:   resource.Daemonsets,
+	}
+	app.Services = resource.Services
+	app.Ingresses = resource.Ingresses
 	return app, nil
 }

@@ -332,24 +358,13 @@ func (c *applicationOperator) getIng(namespace string, services []v1.Service) []
 	return ings
 }

-func (c *applicationOperator) CreateApplication(namespace string, request CreateClusterRequest) error {
-	ns, err := c.informers.Core().V1().Namespaces().Lister().Get(namespace)
-	if err != nil {
-		klog.Error(err)
-		return err
-	}
-
-	if runtimeId := ns.Annotations[constants.OpenPitrixRuntimeAnnotationKey]; runtimeId != "" {
-		request.RuntimeId = runtimeId
-	} else {
-		return fmt.Errorf("runtime not init: namespace %s", namespace)
-	}
-
-	_, err = c.opClient.CreateCluster(openpitrix.ContextWithUsername(request.Username), &pb.CreateClusterRequest{
+func (c *applicationOperator) CreateApplication(clusterName, namespace string, request CreateClusterRequest) error {
+	_, err := c.opClient.CreateCluster(openpitrix.ContextWithUsername(request.Username), &pb.CreateClusterRequest{
 		AppId:     &wrappers.StringValue{Value: request.AppId},
 		VersionId: &wrappers.StringValue{Value: request.VersionId},
-		RuntimeId: &wrappers.StringValue{Value: request.RuntimeId},
+		RuntimeId: &wrappers.StringValue{Value: clusterName},
 		Conf:      &wrappers.StringValue{Value: request.Conf},
+		Zone:      &wrappers.StringValue{Value: namespace},
 	})

 	if err != nil {
@@ -380,8 +395,22 @@ func (c *applicationOperator) ModifyApplication(request ModifyClusterAttributesR
 	return nil
 }

-func (c *applicationOperator) DeleteApplication(clusterId string) error {
-	_, err := c.opClient.DeleteClusters(openpitrix.SystemContext(), &pb.DeleteClustersRequest{ClusterId: []string{clusterId}, Force: &wrappers.BoolValue{Value: true}})
+func (c *applicationOperator) DeleteApplication(applicationId string) error {
+	_, err := c.opClient.DeleteClusters(openpitrix.SystemContext(), &pb.DeleteClustersRequest{ClusterId: []string{applicationId}, Force: &wrappers.BoolValue{Value: true}})
+
+	if err != nil {
+		klog.Errorln(err)
+		return err
+	}
+
+	return nil
+}
+
+func (c *applicationOperator) UpgradeApplication(request UpgradeClusterRequest) error {
+	_, err := c.opClient.UpgradeCluster(openpitrix.ContextWithUsername(request.Username), &pb.UpgradeClusterRequest{
+		ClusterId: &wrappers.StringValue{Value: request.ClusterId},
+		VersionId: &wrappers.StringValue{Value: request.VersionId},
+	})

 	if err != nil {
 		klog.Errorln(err)
--- a/pkg/models/openpitrix/applications_test.go
+++ b/pkg/models/openpitrix/applications_test.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -97,11 +94,12 @@ func TestApplicationOperator_CreateApplication(t *testing.T) {
 			VersionId: &wrappers.StringValue{Value: test.createClusterRequest.VersionId},
 			RuntimeId: &wrappers.StringValue{Value: test.createClusterRequest.RuntimeId},
 			Conf:      &wrappers.StringValue{Value: test.createClusterRequest.Conf},
+			Zone:      &wrappers.StringValue{Value: test.targetNamespace},
 		}).Return(&pb.CreateClusterResponse{}, nil).AnyTimes()

 		t.Run(test.description, func(t *testing.T) {

-			err := applicationOperator.CreateApplication(test.targetNamespace, test.createClusterRequest)
+			err := applicationOperator.CreateApplication(test.createClusterRequest.RuntimeId, test.targetNamespace, test.createClusterRequest)

 			if err != nil && err.Error() != test.expected.Error() {
 				t.Error(err)
--- a/pkg/models/openpitrix/apps.go
+++ b/pkg/models/openpitrix/apps.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- a/pkg/models/openpitrix/attachments.go
+++ b/pkg/models/openpitrix/attachments.go
@@ -1,12 +1,9 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
+Copyright 2020 The KubeSphere Authors.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- a/pkg/models/openpitrix/categories.go
+++ b/pkg/models/openpitrix/categories.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- a/pkg/models/openpitrix/interface.go
+++ b/pkg/models/openpitrix/interface.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- a/pkg/models/openpitrix/repos.go
+++ b/pkg/models/openpitrix/repos.go
@@ -1,12 +1,9 @@
 /*
 Copyright 2020 The KubeSphere Authors.
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -37,6 +34,7 @@ type RepoInterface interface {
 	ListRepos(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error)
 	ValidateRepo(request *ValidateRepoRequest) (*ValidateRepoResponse, error)
 	DoRepoAction(repoId string, request *RepoActionRequest) error
+	ListEvents(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error)
 	ListRepoEvents(repoId string, conditions *params.Conditions, limit, offset int) (*models.PageableResponse, error)
 }

@@ -267,3 +265,33 @@ func (c *repoOperator) ListRepoEvents(repoId string, conditions *params.Conditio

 	return &models.PageableResponse{Items: items, TotalCount: int(resp.TotalCount)}, nil
 }
+
+func (c *repoOperator) ListEvents(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) {
+	describeRepoEventsRequest := &pb.DescribeRepoEventsRequest{}
+	if repoId := conditions.Match["repo_id"]; repoId != "" {
+		describeRepoEventsRequest.RepoId = strings.Split(repoId, "|")
+	}
+	if eventId := conditions.Match["repo_event_id"]; eventId != "" {
+		describeRepoEventsRequest.RepoEventId = strings.Split(eventId, "|")
+	}
+	if status := conditions.Match["status"]; status != "" {
+		describeRepoEventsRequest.Status = strings.Split(status, "|")
+	}
+	describeRepoEventsRequest.Limit = uint32(limit)
+	describeRepoEventsRequest.Offset = uint32(offset)
+
+	resp, err := c.opClient.DescribeRepoEvents(openpitrix.SystemContext(), describeRepoEventsRequest)
+
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	items := make([]interface{}, 0)
+
+	for _, item := range resp.RepoEventSet {
+		items = append(items, convertRepoEvent(item))
+	}
+
+	return &models.PageableResponse{Items: items, TotalCount: int(resp.TotalCount)}, nil
+}
--- a/pkg/models/openpitrix/types.go
+++ b/pkg/models/openpitrix/types.go
@@ -1,3 +1,16 @@
+/*
+Copyright 2020 The KubeSphere Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package openpitrix

 import (
@@ -737,6 +750,25 @@ type CreateClusterRequest struct {
 	Username string `json:"-"`
 }

+type UpgradeClusterRequest struct {
+	// cluster id
+	ClusterId string `json:"cluster_id"`
+
+	// advanced param
+	AdvancedParam []string `json:"advanced_param"`
+
+	// required, conf a json string, include cpu, memory info of cluster
+	Conf string `json:"conf,omitempty"`
+
+	// required, id of runtime
+	RuntimeId string `json:"runtime_id,omitempty"`
+
+	// required, id of app version
+	VersionId string `json:"version_id,omitempty"`
+
+	Username string `json:"-"`
+}
+
 type Cluster struct {

 	// additional info
@@ -840,6 +872,7 @@ const (
 	CreateTime      = "create_time"
 	StatusTime      = "status_time"
 	RuntimeId       = "runtime_id"
+	Zone            = "zone"
 	VersionId       = "version_id"
 	RepoId          = "repo_id"
 	CategoryId      = "category_id"
--- a/pkg/models/openpitrix/utils.go
+++ b/pkg/models/openpitrix/utils.go
@@ -1,12 +1,9 @@
 /*
-Copyright 2019 The KubeSphere Authors.
-
+Copyright 2020 The KubeSphere Authors.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
    http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- a/pkg/simple/client/openpitrix/mock.go
+++ b/pkg/simple/client/openpitrix/mock.go
--- a/pkg/simple/client/openpitrix/openpitrix.go
+++ b/pkg/simple/client/openpitrix/openpitrix.go
@@ -1,6 +1,6 @@
 /*

- Copyright 2019 The KubeSphere Authors.
+ Copyright 2020 The KubeSphere Authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ package openpitrix
 import (
 	"context"
 	"fmt"
+	"github.com/golang/protobuf/ptypes/wrappers"
 	"k8s.io/klog"
 	"openpitrix.io/openpitrix/pkg/manager"
 	"openpitrix.io/openpitrix/pkg/pb"
@@ -48,6 +49,12 @@ type Client interface {
 	pb.CategoryManagerClient
 	pb.AttachmentManagerClient
 	pb.RepoIndexerClient
+	// upsert the openpitrix runtime when cluster is updated or created
+	UpsertRuntime(cluster string, kubeConfig string) error
+	// clean up the openpitrix runtime when cluster is deleted
+	CleanupRuntime(cluster string) error
+	// migrate the openpitrix runtime when upgrade ks2.x to ks3.x
+	MigrateRuntime(runtimeId string, cluster string) error
 }

 type client struct {
@@ -60,6 +67,47 @@ type client struct {
 	pb.RepoIndexerClient
 }

+func (c *client) UpsertRuntime(cluster string, kubeConfig string) error {
+	ctx := SystemContext()
+	req := &pb.CreateRuntimeCredentialRequest{
+		Name:                     &wrappers.StringValue{Value: fmt.Sprintf("kubeconfig-%s", cluster)},
+		Provider:                 &wrappers.StringValue{Value: "kubernetes"},
+		Description:              &wrappers.StringValue{Value: "kubeconfig"},
+		RuntimeUrl:               &wrappers.StringValue{Value: "kubesphere"},
+		RuntimeCredentialContent: &wrappers.StringValue{Value: kubeConfig},
+		RuntimeCredentialId:      &wrappers.StringValue{Value: cluster},
+	}
+	_, err := c.CreateRuntimeCredential(ctx, req)
+	if err != nil {
+		return err
+	}
+	_, err = c.CreateRuntime(ctx, &pb.CreateRuntimeRequest{
+		Name:                &wrappers.StringValue{Value: cluster},
+		RuntimeCredentialId: &wrappers.StringValue{Value: cluster},
+		Provider:            &wrappers.StringValue{Value: KubernetesProvider},
+		Zone:                &wrappers.StringValue{Value: cluster},
+	})
+
+	return err
+}
+
+func (c *client) CleanupRuntime(cluster string) error {
+	ctx := SystemContext()
+	_, err := c.DeleteClusterInRuntime(ctx, &pb.DeleteClusterInRuntimeRequest{
+		RuntimeId: []string{cluster},
+	})
+	return err
+}
+
+func (c *client) MigrateRuntime(runtimeId string, cluster string) error {
+	ctx := SystemContext()
+	_, err := c.MigrateClusterInRuntime(ctx, &pb.MigrateClusterInRuntimeRequest{
+		FromRuntimeId: runtimeId,
+		ToRuntimeId:   cluster,
+	})
+	return err
+}
+
 func parseToHostPort(endpoint string) (string, int, error) {
 	args := strings.Split(endpoint, ":")
 	if len(args) != 2 {