feat: Serving CRD in ks apiserver

Signed-off-by: Roland.Ma <rolandma@kubesphere.io>
2022-01-13 06:27:16 +00:00
parent 365924e76b
commit 0ec32f29fb
7 changed files with 948 additions and 14 deletions
--- a/pkg/apiserver/apiserver.go
+++ b/pkg/apiserver/apiserver.go
@@ -25,17 +25,8 @@ import (
 	"strconv"
 	"time"

-	"kubesphere.io/kubesphere/pkg/utils/iputil"
-
-	"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
-
-	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
-
-	"kubesphere.io/api/notification/v2beta1"
-
-	openpitrixv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v2alpha1"
-
 	"github.com/emicklei/go-restful"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -57,6 +48,8 @@ import (
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/anonymous"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken"
 	"kubesphere.io/kubesphere/pkg/apiserver/authentication/request/bearertoken"
+	"kubesphere.io/kubesphere/pkg/apiserver/authentication/token"
+	"kubesphere.io/kubesphere/pkg/apiserver/authorization"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/path"
@@ -71,6 +64,7 @@ import (
 	alertingv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v2alpha1"
 	clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
 	configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
+	crd "kubesphere.io/kubesphere/pkg/kapis/crd"
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
 	gatewayv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha1"
@@ -84,6 +78,7 @@ import (
 	notificationkapisv2beta2 "kubesphere.io/kubesphere/pkg/kapis/notification/v2beta2"
 	"kubesphere.io/kubesphere/pkg/kapis/oauth"
 	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
+	openpitrixv2alpha1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v2alpha1"
 	operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2"
 	resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2"
 	resourcev1alpha3 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha3"
@@ -107,6 +102,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 	"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
+	"kubesphere.io/kubesphere/pkg/utils/iputil"
 	"kubesphere.io/kubesphere/pkg/utils/metrics"
 )

@@ -169,6 +165,7 @@ func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
 	})

 	s.installKubeSphereAPIs()
+	s.installCRDAPIs()
 	s.installMetricsAPI()
 	s.container.Filter(monitorRequest)

@@ -260,6 +257,14 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(gatewayv1alpha1.AddToContainer(s.container, s.Config.GatewayOptions, s.RuntimeCache, s.RuntimeClient, s.InformerFactory, s.KubernetesClient.Kubernetes(), s.LoggingClient))
 }

+// installCRDAPIs Install CRDs to the KAPIs with List and Get options
+func (s *APIServer) installCRDAPIs() {
+	crds := &extv1.CustomResourceDefinitionList{}
+	//TODO Maybe we need a better label name
+	urlruntime.Must(s.RuntimeClient.List(context.TODO(), crds, runtimeclient.MatchingLabels{"kubesphere.io/resource-served": "true"}))
+	urlruntime.Must(crd.AddToContainer(s.container, s.RuntimeClient, s.RuntimeCache, crds))
+}
+
 func (s *APIServer) Run(ctx context.Context) (err error) {

 	err = s.waitForResourceSync(ctx)
@@ -298,8 +303,8 @@ func (s *APIServer) buildHandlerChain(stopCh <-chan struct{}) {
 			tenantv1alpha2.Resource(clusterv1alpha1.ResourcesPluralCluster),
 			clusterv1alpha1.Resource(clusterv1alpha1.ResourcesPluralCluster),
 			resourcev1alpha3.Resource(clusterv1alpha1.ResourcesPluralCluster),
-			notificationv2beta1.Resource(v2beta1.ResourcesPluralConfig),
-			notificationv2beta1.Resource(v2beta1.ResourcesPluralReceiver),
+			notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralConfig),
+			notificationv2beta1.Resource(notificationv2beta1.ResourcesPluralReceiver),
 		},
 	}

@@ -436,8 +441,8 @@ func (s *APIServer) waitForResourceSync(ctx context.Context) error {
 		{Group: "iam.kubesphere.io", Version: "v1alpha2", Resource: "loginrecords"},
 		{Group: "cluster.kubesphere.io", Version: "v1alpha1", Resource: "clusters"},
 		{Group: "network.kubesphere.io", Version: "v1alpha1", Resource: "ippools"},
-		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: v2beta1.ResourcesPluralConfig},
-		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: v2beta1.ResourcesPluralReceiver},
+		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: notificationv2beta1.ResourcesPluralConfig},
+		{Group: "notification.kubesphere.io", Version: "v2beta1", Resource: notificationv2beta1.ResourcesPluralReceiver},
 	}

 	devopsGVRs := []schema.GroupVersionResource{
--- a/pkg/kapis/crd/crd.go
+++ b/pkg/kapis/crd/crd.go
@@ -0,0 +1,159 @@
+/*
+Copyright 2022 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crd
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+	ksruntime "kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/models/crds"
+)
+
+const (
+	ok = "OK"
+)
+
+//AddToContainer register GET and LIST API for CRD to the web service
+func AddToContainer(c *restful.Container, cli client.Client, cache cache.Cache, crdList *extv1.CustomResourceDefinitionList) error {
+
+	for _, crd := range crdList.Items {
+		gvk := schema.GroupVersionKind{Group: crd.Spec.Group, Version: currentVersion(&crd), Kind: crd.Spec.Names.Kind}
+		resource := crd.Spec.Names.Plural
+
+		var h crds.Handler
+		if cli.Scheme().Recognizes(gvk) {
+			h = crds.NewTyped(cache, gvk, cli.Scheme())
+		} else {
+			h = crds.NewUnstructured(cache, gvk)
+		}
+
+		if containsRouter(c.RegisteredWebServices(), gvk.GroupVersion().String()) {
+			//Skip existing Root service for now
+			//TODO Register to existing WebService
+			continue
+		}
+
+		//Create new WebService
+		webservice := ksruntime.NewWebService(gvk.GroupVersion())
+
+		listURL := fmt.Sprintf("/%s", resource)
+		webservice.Route(webservice.GET(listURL).
+			To(func(request *restful.Request, response *restful.Response) {
+				h.ListResources(request, response)
+			}).
+			Doc(fmt.Sprintf("Cluster level resource %s query", resource)).
+			Param(webservice.QueryParameter(query.ParameterName, "name used to do filtering").Required(false)).
+			Param(webservice.QueryParameter(query.ParameterPage, "page").Required(false).DataFormat("page=%d").DefaultValue("page=1")).
+			Param(webservice.QueryParameter(query.ParameterLimit, "limit").Required(false)).
+			Param(webservice.QueryParameter(query.ParameterAscending, "sort parameters, e.g. reverse=true").Required(false).DefaultValue("ascending=false")).
+			Param(webservice.QueryParameter(query.ParameterOrderBy, "sort parameters, e.g. orderBy=createTime")).
+			Returns(http.StatusOK, ok, api.ListResult{}))
+
+		if crd.Spec.Scope == extv1.ClusterScoped {
+			getURL := fmt.Sprintf("/%s/{name}", resource)
+			webservice.Route(webservice.GET(getURL).
+				To(func(request *restful.Request, response *restful.Response) {
+					h.GetResources(request, response)
+				}).
+				Doc(fmt.Sprintf("Cluster level resource %s", resource)).
+				Param(webservice.PathParameter("name", "the name of the clustered resources")).
+				Returns(http.StatusOK, api.StatusOK, nil))
+		}
+
+		if crd.Spec.Scope == extv1.NamespaceScoped {
+			listNsURL := fmt.Sprintf("/namespaces/{namespace}/%s/", resource)
+			webservice.Route(webservice.GET(listNsURL).
+				To(func(request *restful.Request, response *restful.Response) {
+					h.ListResources(request, response)
+				}).
+				Doc(fmt.Sprintf("Namespace level resource %s query", resource)).
+				Param(webservice.PathParameter("namespace", "the name of the project")).
+				Param(webservice.QueryParameter(query.ParameterName, "name used to do filtering").Required(false)).
+				Param(webservice.QueryParameter(query.ParameterPage, "page").Required(false).DataFormat("page=%d").DefaultValue("page=1")).
+				Param(webservice.QueryParameter(query.ParameterLimit, "limit").Required(false)).
+				Param(webservice.QueryParameter(query.ParameterAscending, "sort parameters, e.g. reverse=true").Required(false).DefaultValue("ascending=false")).
+				Param(webservice.QueryParameter(query.ParameterOrderBy, "sort parameters, e.g. orderBy=createTime")).
+				Returns(http.StatusOK, ok, api.ListResult{}))
+
+			getNsURL := fmt.Sprintf("/namespaces/{namespace}/%s/{name}", resource)
+			webservice.Route(webservice.GET(getNsURL).
+				To(func(request *restful.Request, response *restful.Response) {
+					h.GetResources(request, response)
+				}).
+				Doc(fmt.Sprintf("Namespace level resource %s", resource)).
+				Param(webservice.PathParameter("namespace", "the name of the project")).
+				Param(webservice.PathParameter("name", "the name of resource")).
+				Returns(http.StatusOK, ok, nil))
+		}
+
+		if crd.Spec.Scope == extv1.ClusterScoped && crd.Annotations != nil && crd.Annotations["kubesphere.io/resource-scope"] == "workspaced" {
+			listWsURL := fmt.Sprintf("/workspaces/{workspace}/%s/", resource)
+			webservice.Route(webservice.GET(listWsURL).
+				To(func(request *restful.Request, response *restful.Response) {
+					h.ListResources(request, response)
+				}).
+				Doc(fmt.Sprintf("Workspace level resource %s query", resource)).
+				Param(webservice.PathParameter("workspace", "the name of the workspace")).
+				Param(webservice.QueryParameter(query.ParameterName, "name used to do filtering").Required(false)).
+				Param(webservice.QueryParameter(query.ParameterPage, "page").Required(false).DataFormat("page=%d").DefaultValue("page=1")).
+				Param(webservice.QueryParameter(query.ParameterLimit, "limit").Required(false)).
+				Param(webservice.QueryParameter(query.ParameterAscending, "sort parameters, e.g. reverse=true").Required(false).DefaultValue("ascending=false")).
+				Param(webservice.QueryParameter(query.ParameterOrderBy, "sort parameters, e.g. orderBy=createTime")).
+				Returns(http.StatusOK, ok, api.ListResult{}))
+
+			getWsURL := fmt.Sprintf("/workspaces/{workspace}/%s/{name}", resource)
+			webservice.Route(webservice.GET(getWsURL).
+				To(func(request *restful.Request, response *restful.Response) {
+					h.GetResources(request, response)
+				}).
+				Doc(fmt.Sprintf("Workspace level resource %s", resource)).
+				Param(webservice.PathParameter("workspace", "the name of the workspace")).
+				Param(webservice.PathParameter("name", "the name of resource")).
+				Returns(http.StatusOK, ok, nil))
+		}
+		c.Add(webservice)
+	}
+
+	return nil
+}
+
+func containsRouter(services []*restful.WebService, root string) bool {
+	for _, svc := range services {
+		if svc.RootPath() == "/kapis/"+root {
+			return true
+		}
+	}
+	return false
+}
+
+func currentVersion(crd *extv1.CustomResourceDefinition) string {
+	for _, v := range crd.Spec.Versions {
+		if v.Served && v.Storage {
+			return v.Name
+		}
+	}
+	return ""
+}
--- a/pkg/models/crds/interface.go
+++ b/pkg/models/crds/interface.go
@@ -0,0 +1,227 @@
+/*
+Copyright 2022 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crds
+
+import (
+	"sort"
+	"strings"
+
+	"github.com/emicklei/go-restful"
+
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+)
+
+var (
+	Filters      = map[schema.GroupVersionKind]FilterFunc{}
+	Comparers    = map[schema.GroupVersionKind]CompareFunc{}
+	Transformers = map[schema.GroupVersionKind][]TransformFunc{}
+)
+
+type Reader interface {
+	// Get retrieves a single object by its namespace and name
+	Get(key types.NamespacedName) (client.Object, error)
+
+	// List retrieves a collection of objects matches given query
+	List(namespace string, query *query.Query) (*api.ListResult, error)
+}
+
+type Handler interface {
+	GetResources(request *restful.Request, response *restful.Response)
+	ListResources(request *restful.Request, response *restful.Response)
+}
+
+type Client interface {
+	Handler
+	Reader
+}
+
+// CompareFunc return true is left great than right
+type CompareFunc func(metav1.Object, metav1.Object, query.Field) bool
+
+type FilterFunc func(metav1.Object, query.Filter) bool
+
+type TransformFunc func(metav1.Object) runtime.Object
+
+func DefaultList(objList runtime.Object, q *query.Query, compareFunc CompareFunc, filterFunc FilterFunc, transformFuncs ...TransformFunc) *api.ListResult {
+	// selected matched ones
+	var filtered []runtime.Object
+
+	meta.EachListItem(objList, func(obj runtime.Object) error {
+		selected := true
+		o, err := meta.Accessor(obj)
+		if err != nil {
+			return err
+		}
+		for field, value := range q.Filters {
+
+			if !filterFunc(o, query.Filter{Field: field, Value: value}) {
+				selected = false
+				break
+			}
+		}
+
+		if selected {
+			for _, transform := range transformFuncs {
+				obj = transform(o)
+			}
+			filtered = append(filtered, obj)
+		}
+		return nil
+	})
+
+	// sort by sortBy field
+	sort.Slice(filtered, func(i, j int) bool {
+		l, err := meta.Accessor(filtered[i])
+		if err != nil {
+			return false
+		}
+		r, err := meta.Accessor(filtered[j])
+		if err != nil {
+			return false
+		}
+		if !q.Ascending {
+			return compareFunc(l, r, q.SortBy)
+		}
+		return !compareFunc(l, r, q.SortBy)
+	})
+
+	total := len(filtered)
+
+	if q.Pagination == nil {
+		q.Pagination = query.NoPagination
+	}
+
+	start, end := q.Pagination.GetValidPagination(total)
+
+	return &api.ListResult{
+		TotalItems: len(filtered),
+		Items:      objectsToInterfaces(filtered[start:end]),
+	}
+}
+
+// DefaultObjectMetaCompare return true is left great than right
+func DefaultObjectMetaCompare(left, right metav1.Object, sortBy query.Field) bool {
+	switch sortBy {
+	// ?sortBy=name
+	case query.FieldName:
+		return strings.Compare(left.GetName(), right.GetName()) > 0
+	//	?sortBy=creationTimestamp
+	default:
+		fallthrough
+	case query.FieldCreateTime:
+		fallthrough
+	case query.FieldCreationTimeStamp:
+		// compare by name if creation timestamp is equal
+		ltime := left.GetCreationTimestamp()
+		rtime := right.GetCreationTimestamp()
+		if ltime.Equal(&rtime) {
+			return strings.Compare(left.GetName(), right.GetName()) > 0
+		}
+		return left.GetCreationTimestamp().After(right.GetCreationTimestamp().Time)
+	}
+}
+
+//  Default metadata filter
+func DefaultObjectMetaFilter(item metav1.Object, filter query.Filter) bool {
+	switch filter.Field {
+	case query.FieldNames:
+		for _, name := range strings.Split(string(filter.Value), ",") {
+			if item.GetName() == name {
+				return true
+			}
+		}
+		return false
+	// /namespaces?page=1&limit=10&name=default
+	case query.FieldName:
+		return strings.Contains(item.GetName(), string(filter.Value))
+		// /namespaces?page=1&limit=10&uid=a8a8d6cf-f6a5-4fea-9c1b-e57610115706
+	case query.FieldUID:
+		return strings.Compare(string(item.GetUID()), string(filter.Value)) == 0
+		// /deployments?page=1&limit=10&namespace=kubesphere-system
+	case query.FieldNamespace:
+		return strings.Compare(item.GetNamespace(), string(filter.Value)) == 0
+		// /namespaces?page=1&limit=10&ownerReference=a8a8d6cf-f6a5-4fea-9c1b-e57610115706
+	case query.FieldOwnerReference:
+		for _, ownerReference := range item.GetOwnerReferences() {
+			if strings.Compare(string(ownerReference.UID), string(filter.Value)) == 0 {
+				return true
+			}
+		}
+		return false
+		// /namespaces?page=1&limit=10&ownerKind=Workspace
+	case query.FieldOwnerKind:
+		for _, ownerReference := range item.GetOwnerReferences() {
+			if strings.Compare(ownerReference.Kind, string(filter.Value)) == 0 {
+				return true
+			}
+		}
+		return false
+		// /namespaces?page=1&limit=10&annotation=openpitrix_runtime
+	case query.FieldAnnotation:
+		return labelMatch(item.GetAnnotations(), string(filter.Value))
+		// /namespaces?page=1&limit=10&label=kubesphere.io/workspace:system-workspace
+	case query.FieldLabel:
+		return labelMatch(item.GetLabels(), string(filter.Value))
+	default:
+		return false
+	}
+}
+
+func labelMatch(labels map[string]string, filter string) bool {
+	fields := strings.SplitN(filter, "=", 2)
+	var key, value string
+	var opposite bool
+	if len(fields) == 2 {
+		key = fields[0]
+		if strings.HasSuffix(key, "!") {
+			key = strings.TrimSuffix(key, "!")
+			opposite = true
+		}
+		value = fields[1]
+	} else {
+		key = fields[0]
+		value = "*"
+	}
+	for k, v := range labels {
+		if opposite {
+			if (k == key) && v != value {
+				return true
+			}
+		} else {
+			if (k == key) && (value == "*" || v == value) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func objectsToInterfaces(objs []runtime.Object) []interface{} {
+	res := make([]interface{}, 0)
+	for _, obj := range objs {
+		res = append(res, obj)
+	}
+	return res
+}
--- a/pkg/models/crds/interface_test.go
+++ b/pkg/models/crds/interface_test.go
@@ -0,0 +1,85 @@
+/*
+
+ Copyright 2022 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package crds
+
+import "testing"
+
+func TestLabelMatch(t *testing.T) {
+	tests := []struct {
+		labels       map[string]string
+		filter       string
+		expectResult bool
+	}{
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace",
+			expectResult: true,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/creator": "system",
+			},
+			filter:       "kubesphere.io/workspace",
+			expectResult: false,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace=",
+			expectResult: false,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace!=",
+			expectResult: true,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace!=kubesphere-system",
+			expectResult: false,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace=kubesphere-system",
+			expectResult: true,
+		},
+		{
+			labels: map[string]string{
+				"kubesphere.io/workspace": "kubesphere-system",
+			},
+			filter:       "kubesphere.io/workspace=system",
+			expectResult: false,
+		},
+	}
+	for i, test := range tests {
+		result := labelMatch(test.labels, test.filter)
+		if result != test.expectResult {
+			t.Errorf("case %d, got %#v, expected %#v", i, result, test.expectResult)
+		}
+	}
+}
--- a/pkg/models/crds/typed_handler.go
+++ b/pkg/models/crds/typed_handler.go
@@ -0,0 +1,131 @@
+/*
+Copyright 2022 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crds
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/emicklei/go-restful"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type TypedCRDHandler struct {
+	c   client.Reader
+	s   *runtime.Scheme
+	gvk schema.GroupVersionKind
+}
+
+func NewTyped(cache client.Reader, gvk schema.GroupVersionKind, s *runtime.Scheme) Client {
+	return &TypedCRDHandler{c: cache, gvk: gvk, s: s}
+}
+
+func (h *TypedCRDHandler) GetResources(request *restful.Request, response *restful.Response) {
+	namespace := request.PathParameter("namespace")
+	name := request.PathParameter("name")
+
+	obj, err := h.Get(types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	})
+
+	if err != nil {
+		api.HandleError(response, nil, err)
+		return
+	}
+	response.WriteEntity(obj)
+}
+
+// handleListResources retrieves resources
+func (h *TypedCRDHandler) ListResources(request *restful.Request, response *restful.Response) {
+	q := query.ParseQueryParameter(request)
+	namespace := request.PathParameter("namespace")
+	workspace := request.PathParameter("workspace")
+	if workspace != "" {
+		// filter by workspace
+		q.Filters[query.FieldLabel] = query.Value(fmt.Sprintf("%s=%s", "kubesphere.io/workspace", workspace))
+	}
+	list, err := h.List(namespace, q)
+	if err != nil {
+		api.HandleError(response, nil, err)
+		return
+	}
+	response.WriteEntity(list)
+}
+
+func (h *TypedCRDHandler) Get(key types.NamespacedName) (client.Object, error) {
+	obj, err := h.s.New(h.gvk)
+	if err != nil {
+		return nil, err
+	}
+	clobj := obj.(client.Object)
+	if err := h.c.Get(context.TODO(), key, clobj); err != nil {
+		return nil, err
+	}
+	return clobj, err
+}
+
+func (h *TypedCRDHandler) List(namespace string, q *query.Query) (*api.ListResult, error) {
+
+	listGvk := schema.GroupVersionKind{
+		Group:   h.gvk.Group,
+		Version: h.gvk.Version,
+		Kind:    h.gvk.Kind + "List",
+	}
+	obj, err := h.s.New(listGvk)
+	if err != nil {
+		return nil, err
+	}
+	objlist := obj.(client.ObjectList)
+	if err := h.c.List(context.TODO(), objlist, &client.ListOptions{LabelSelector: q.Selector()}, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+
+	result := DefaultList(objlist, q, h.compare, h.filter, h.transforms()...)
+
+	return result, err
+}
+
+func (d *TypedCRDHandler) compare(left, right metav1.Object, field query.Field) bool {
+	if fn, ok := Comparers[d.gvk]; ok {
+		fn(left, right, field)
+	}
+	return DefaultObjectMetaCompare(left, right, field)
+}
+
+func (d *TypedCRDHandler) filter(object metav1.Object, filter query.Filter) bool {
+	if fn, ok := Filters[d.gvk]; ok {
+		fn(object, filter)
+	}
+	return DefaultObjectMetaFilter(object, filter)
+}
+
+func (d *TypedCRDHandler) transforms() []TransformFunc {
+	if fn, ok := Transformers[d.gvk]; ok {
+		return fn
+	}
+	trans := []TransformFunc{}
+	return trans
+}
--- a/pkg/models/crds/typed_handler_test.go
+++ b/pkg/models/crds/typed_handler_test.go
@@ -0,0 +1,216 @@
+/*
+Copyright 2022 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crds
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+)
+
+type fakeClient struct {
+	Client client.Client
+}
+
+// Get retrieves an obj for the given object key from the Kubernetes Cluster.
+// obj must be a struct pointer so that obj can be updated with the response
+// returned by the Server.
+func (f *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
+	return f.Client.Get(ctx, key, obj)
+}
+
+// List retrieves list of objects for a given namespace and list options. On a
+// successful call, Items field in the list will be populated with the
+// result returned from the server.
+func (f *fakeClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
+	return f.Client.List(ctx, list, opts...)
+}
+
+// GetInformer fetches or constructs an informer for the given object that corresponds to a single
+// API kind and resource.
+func (f *fakeClient) GetInformer(ctx context.Context, obj client.Object) (cache.Informer, error) {
+	return nil, nil
+}
+
+// GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead
+// of the underlying object.
+func (f *fakeClient) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (cache.Informer, error) {
+	return nil, nil
+}
+
+// Start runs all the informers known to this cache until the context is closed.
+// It blocks.
+func (f *fakeClient) Start(ctx context.Context) error {
+	return nil
+}
+
+// WaitForCacheSync waits for all the caches to sync.  Returns false if it could not sync a cache.
+func (f *fakeClient) WaitForCacheSync(ctx context.Context) bool {
+	return false
+}
+
+func (f *fakeClient) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
+	return nil
+}
+
+var (
+	cm1 = &corev1.ConfigMap{
+		TypeMeta: v1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "cm1",
+			Namespace: "default",
+		},
+	}
+
+	cm2 = &corev1.ConfigMap{
+		TypeMeta: v1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "cm2",
+			Namespace: "default",
+		},
+	}
+
+	cm3 = &corev1.ConfigMap{
+		TypeMeta: v1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "cm3",
+			Namespace: "default",
+		},
+	}
+)
+
+func TestHandler_Get(t *testing.T) {
+
+	var Scheme = runtime.NewScheme()
+	// v1alpha1.AddToScheme(Scheme)
+	corev1.AddToScheme(Scheme)
+
+	c := fake.NewClientBuilder().WithScheme(Scheme).WithRuntimeObjects(cm1).Build()
+
+	h := NewTyped(&fakeClient{c}, cm1.GroupVersionKind(), Scheme)
+
+	type args struct {
+		gvk schema.GroupVersionKind
+		key types.NamespacedName
+	}
+	tests := []struct {
+		name    string
+		handler Client
+		args    args
+		want    client.Object
+		wantErr bool
+	}{
+		{
+			name:    "test",
+			handler: h,
+			args: args{
+				key: types.NamespacedName{
+					Namespace: "default",
+					Name:      "cm1",
+				},
+			},
+			want: cm1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			h := tt.handler
+			got, err := h.Get(tt.args.key)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Handler.Get() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Handler.Get() \nDiff:\n %s", diff.ObjectGoPrintSideBySide(tt.want, got))
+			}
+		})
+	}
+}
+
+func TestResourceGetter_List(t *testing.T) {
+
+	var Scheme = runtime.NewScheme()
+	// v1alpha1.AddToScheme(Scheme)
+	corev1.AddToScheme(Scheme)
+
+	c := fake.NewClientBuilder().WithScheme(Scheme).WithRuntimeObjects(cm1, cm2, cm3).Build()
+
+	h := NewTyped(&fakeClient{c}, cm1.GroupVersionKind(), Scheme)
+
+	tests := []struct {
+		name      string
+		Resource  string
+		Namespace string
+		Query     *query.Query
+		wantErr   bool
+		want      *api.ListResult
+	}{
+		{
+			name:      "list configmaps",
+			Resource:  "configmaps",
+			Namespace: "default",
+			Query: &query.Query{
+				Pagination: &query.Pagination{
+					Limit:  10,
+					Offset: 0,
+				},
+				SortBy:    query.FieldName,
+				Ascending: false,
+				Filters:   map[query.Field]query.Value{},
+			},
+			want: &api.ListResult{
+				Items:      []interface{}{cm3, cm2, cm1},
+				TotalItems: 3,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := h.List(tt.Namespace, tt.Query)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Handler.Get() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Handler.Get() \nDiff:\n %s", diff.ObjectGoPrintSideBySide(tt.want, got))
+			}
+		})
+	}
+}
--- a/pkg/models/crds/unstructured_handler.go
+++ b/pkg/models/crds/unstructured_handler.go
@@ -0,0 +1,111 @@
+/*
+Copyright 2022 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package crds
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/emicklei/go-restful"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/query"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+type UnstructuredCRDHandler struct {
+	c       client.Reader
+	gvk     schema.GroupVersionKind
+	gvkList schema.GroupVersionKind
+}
+
+func NewUnstructured(cache client.Reader, gvk schema.GroupVersionKind) Client {
+	return &UnstructuredCRDHandler{c: cache, gvk: gvk, gvkList: schema.GroupVersionKind{Version: gvk.Version, Group: gvk.Group, Kind: gvk.Kind + "List"}}
+}
+
+func (h *UnstructuredCRDHandler) GetResources(request *restful.Request, response *restful.Response) {
+	namespace := request.PathParameter("namespace")
+	name := request.PathParameter("name")
+
+	obj, err := h.Get(types.NamespacedName{
+		Namespace: namespace,
+		Name:      name,
+	})
+
+	if err != nil {
+		api.HandleError(response, nil, err)
+		return
+	}
+	response.WriteEntity(obj)
+}
+
+// handleListResources retrieves resources
+func (h *UnstructuredCRDHandler) ListResources(request *restful.Request, response *restful.Response) {
+	q := query.ParseQueryParameter(request)
+	namespace := request.PathParameter("namespace")
+	workspace := request.PathParameter("workspace")
+	if workspace != "" {
+		// filter by workspace
+		q.Filters[query.FieldLabel] = query.Value(fmt.Sprintf("%s=%s", "kubesphere.io/workspace", workspace))
+	}
+	list, err := h.List(namespace, q)
+	if err != nil {
+		api.HandleError(response, nil, err)
+		return
+	}
+	response.WriteEntity(list)
+}
+
+func (h *UnstructuredCRDHandler) Get(key types.NamespacedName) (client.Object, error) {
+
+	obj := &unstructured.Unstructured{}
+	obj.SetGroupVersionKind(h.gvk)
+
+	if err := h.c.Get(context.TODO(), key, obj); err != nil {
+		return nil, err
+	}
+	return obj, nil
+}
+
+func (h *UnstructuredCRDHandler) List(namespace string, q *query.Query) (*api.ListResult, error) {
+
+	listObj := &unstructured.UnstructuredList{}
+	listObj.SetGroupVersionKind(h.gvkList)
+
+	if err := h.c.List(context.TODO(), listObj, &client.ListOptions{LabelSelector: q.Selector()}, client.InNamespace(namespace)); err != nil {
+		return nil, err
+	}
+
+	result := DefaultList(listObj, q, h.compare, h.filter)
+
+	return result, nil
+}
+
+func (d *UnstructuredCRDHandler) compare(left, right metav1.Object, field query.Field) bool {
+
+	return DefaultObjectMetaCompare(left, right, field)
+}
+
+func (d *UnstructuredCRDHandler) filter(object metav1.Object, filter query.Filter) bool {
+	//TODO Maybe we can use a json path Filter here
+	return DefaultObjectMetaFilter(object, filter)
+}