add gateway api

Signed-off-by: Roland.Ma <rolandma@yunify.com>

cmd/ks-apiserver/app/options/options.go

@@ -21,14 +21,15 @@ import (
 	"flag"
 	"fmt"
 
+	"k8s.io/client-go/kubernetes/scheme"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog"
 	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	"kubesphere.io/kubesphere/pkg/apis"
 	"kubesphere.io/kubesphere/pkg/apiserver"
 	apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
-	"kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
 	"kubesphere.io/kubesphere/pkg/informers"
 	genericoptions "kubesphere.io/kubesphere/pkg/server/options"
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
@@ -229,7 +230,12 @@ func (s *ServerRunOptions) NewAPIServer(stopCh <-chan struct{}) (*apiserver.APIS
 
 	apiServer.RuntimeCache, err = runtimecache.New(apiServer.KubernetesClient.Config(), runtimecache.Options{Scheme: sch})
 	if err != nil {
-		klog.Fatalf("unable to create runtime cache: %v", err)
+		klog.Fatalf("unable to create controller runtime cache: %v", err)
+	}
+
+	apiServer.RuntimeClient, err = runtimeclient.New(apiServer.KubernetesClient.Config(), runtimeclient.Options{Scheme: sch})
+	if err != nil {
+		klog.Fatalf("unable to create controller runtime client: %v", err)
 	}
 
 	apiServer.Server = server

config/crds/gateway.kubesphere.io_gateways.yaml

@@ -77,7 +77,6 @@ spec:
             type: object
           status:
             type: object
-            x-kubernetes-embedded-resource: true
             x-kubernetes-preserve-unknown-fields: true
         type: object
     served: true

pkg/apiserver/apiserver.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	"k8s.io/klog"
 	runtimecache "sigs.k8s.io/controller-runtime/pkg/cache"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
 
 	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
 	iamv1alpha2 "kubesphere.io/api/iam/v1alpha2"
@@ -68,6 +69,7 @@ import (
 	configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
+	gatewayv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/gateway/v1alpha1"
 	iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
 	kubeedgev1alpha1 "kubesphere.io/kubesphere/pkg/kapis/kubeedge/v1alpha1"
 	meteringv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/metering/v1alpha1"
@@ -162,6 +164,8 @@ type APIServer struct {
 
 	// controller-runtime cache
 	RuntimeCache runtimecache.Cache
+	// controller-runtime client
+	RuntimeClient runtimeclient.Client
 }
 
 func (s *APIServer) PrepareRun(stopCh <-chan struct{}) error {
@@ -267,6 +271,7 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(kubeedgev1alpha1.AddToContainer(s.container, s.Config.KubeEdgeOptions.Endpoint))
 	urlruntime.Must(notificationkapisv2beta1.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
 		s.KubernetesClient.KubeSphere()))
+	urlruntime.Must(gatewayv1alpha1.AddToContainer(s.container, s.Config.GatewayOptions, s.RuntimeClient))
 }
 
 func (s *APIServer) Run(ctx context.Context) (err error) {

pkg/constants/constants.go

@@ -93,6 +93,8 @@ const (
 	ClusterResourcesTag   = "Cluster Resources"
 	ComponentStatusTag    = "Component Status"
 
+	GatewayTag = "Gateway"
+
 	NetworkTopologyTag = "Network Topology"
 
 	KubeSphereMetricsTag = "KubeSphere Metrics"

pkg/kapis/gateway/v1alpha1/handler.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2021 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"github.com/emicklei/go-restful"
+	"kubesphere.io/api/gateway/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	operator "kubesphere.io/kubesphere/pkg/models/gateway"
+	servererr "kubesphere.io/kubesphere/pkg/server/errors"
+	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
+)
+
+type handler struct {
+	options *gateway.Options
+	gw      operator.GatewayOperator
+}
+
+//newHandler create an instance of the handler
+func newHandler(options *gateway.Options, client client.Client) *handler {
+	// Do not register Gateway scheme globally. Which will cause conflict in ks-controller-manager.
+	v1alpha1.AddToScheme(client.Scheme())
+	return &handler{
+		options: options,
+		gw:      operator.NewGatewayOperator(client, options),
+	}
+}
+
+func (h *handler) Create(request *restful.Request, response *restful.Response) {
+	ns := request.PathParameter("namespace")
+	var gateway v1alpha1.Gateway
+
+	err := request.ReadEntity(&gateway)
+	if err != nil {
+		api.HandleBadRequest(response, request, err)
+		return
+	}
+
+	created, err := h.gw.CreateGateway(ns, &gateway)
+	if err != nil {
+		api.HandleError(response, request, err)
+		return
+	}
+
+	response.WriteEntity(created)
+}
+
+func (h *handler) Update(request *restful.Request, response *restful.Response) {
+	ns := request.PathParameter("namespace")
+	var gateway v1alpha1.Gateway
+	err := request.ReadEntity(&gateway)
+	if err != nil {
+		api.HandleBadRequest(response, request, err)
+		return
+	}
+
+	updated, err := h.gw.UpdateGateway(ns, &gateway)
+	if err != nil {
+		api.HandleError(response, request, err)
+		return
+	}
+
+	response.WriteEntity(updated)
+}
+
+func (h *handler) Get(request *restful.Request, response *restful.Response) {
+	ns := request.PathParameter("namespace")
+	gateway, err := h.gw.GetGateways(ns)
+	if err != nil {
+		api.HandleError(response, request, err)
+		return
+	}
+	response.WriteEntity(gateway)
+}
+
+func (h *handler) Delete(request *restful.Request, response *restful.Response) {
+	ns := request.PathParameter("namespace")
+
+	err := h.gw.DeleteGateway(ns)
+	if err != nil {
+		api.HandleError(response, request, err)
+		return
+	}
+
+	response.WriteEntity(servererr.None)
+}
+
+func (h *handler) Upgrade(request *restful.Request, response *restful.Response) {
+	ns := request.PathParameter("namespace")
+
+	g, err := h.gw.UpgradeGateway(ns)
+	if err != nil {
+		api.HandleError(response, request, err)
+		return
+	}
+
+	response.WriteEntity(g)
+}
+
+func (h *handler) List(request *restful.Request, response *restful.Response) {
+	//TODO
+}

pkg/kapis/gateway/v1alpha1/register.go

@@ -0,0 +1,90 @@
+/*
+Copyright 2021 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"net/http"
+
+	"github.com/emicklei/go-restful"
+	restfulspec "github.com/emicklei/go-restful-openapi"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"kubesphere.io/api/gateway/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/server/errors"
+	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
+)
+
+// there are no versions specified cause we want to proxy all versions of requests to backend service
+var GroupVersion = schema.GroupVersion{Group: "gateway.kubesphere.io", Version: "v1alpha1"}
+
+func AddToContainer(container *restful.Container, options *gateway.Options, client client.Client) error {
+	ws := runtime.NewWebService(GroupVersion)
+
+	handler := newHandler(options, client)
+
+	// gateways
+	ws.Route(ws.POST("/namespaces/{namespace}/gateways").
+		To(handler.Create).
+		Doc("Create a gateway for a specified namespace.").
+		Param(ws.PathParameter("namespace", "the watching namespace of the gateway")).
+		Param(ws.BodyParameter("gateway", "Gateway specification")).
+		Returns(http.StatusOK, api.StatusOK, v1alpha1.Gateway{}).
+		Reads(v1alpha1.Gateway{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+	ws.Route(ws.DELETE("/namespaces/{namespace}/gateways/").
+		To(handler.Delete).
+		Doc("Delete the specified gateway in namespace.").
+		Param(ws.PathParameter("namespace", "the watching namespace of the gateway")).
+		Returns(http.StatusOK, api.StatusOK, errors.None).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+	ws.Route(ws.PUT("/namespaces/{namespace}/gateways/").
+		To(handler.Update).
+		Doc("Update gateway for a specified namespace.").
+		Reads(v1alpha1.Gateway{}).
+		Param(ws.PathParameter("namespace", "the watching namespace of the gateway")).
+		Param(ws.BodyParameter("gateway", "Gateway specification")).
+		Returns(http.StatusOK, api.StatusOK, v1alpha1.Gateway{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/gateways/").
+		To(handler.Get).
+		Doc("Retrieve gateways details.").
+		Param(ws.PathParameter("namespace", "the watching namespace of the gateway")).
+		Returns(http.StatusOK, api.StatusOK, v1alpha1.Gateway{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+
+	ws.Route(ws.POST("/namespaces/{namespace}/gateways/{gateway}/upgrade").
+		To(handler.Upgrade).
+		Doc("Upgrade the legacy Project Gateway to the CRD based Gateway.").
+		Param(ws.PathParameter("namespace", "the watching namespace of the gateway")).
+		Returns(http.StatusOK, api.StatusOK, v1alpha1.Gateway{}).
+		Reads(v1alpha1.Gateway{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+
+	ws.Route(ws.GET("/gateways/").
+		To(handler.List).
+		Doc("List Gateway details.").
+		Returns(http.StatusOK, api.StatusOK, v1alpha1.Gateway{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.GatewayTag}))
+
+	container.Add(ws)
+	return nil
+}

pkg/kapis/resources/v1alpha2/register.go

@@ -191,6 +191,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
 		Returns(http.StatusOK, api.StatusOK, appsv1.StatefulSet{}))
 
 	webservice.Route(webservice.GET("/namespaces/{namespace}/router").
+		Deprecate().
 		To(handler.handleGetRouter).
 		Doc("List router of a specified project").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
@@ -198,6 +199,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
 		Param(webservice.PathParameter("namespace", "the name of the project")))
 
 	webservice.Route(webservice.DELETE("/namespaces/{namespace}/router").
+		Deprecate().
 		To(handler.handleDeleteRouter).
 		Doc("List router of a specified project").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
@@ -205,6 +207,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
 		Param(webservice.PathParameter("namespace", "the name of the project")))
 
 	webservice.Route(webservice.POST("/namespaces/{namespace}/router").
+		Deprecate().
 		To(handler.handleCreateRouter).
 		Doc("Create a router for a specified project").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).
@@ -212,6 +215,7 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, factor
 		Param(webservice.PathParameter("namespace", "the name of the project")))
 
 	webservice.Route(webservice.PUT("/namespaces/{namespace}/router").
+		Deprecate().
 		To(handler.handleUpdateRouter).
 		Doc("Update a router for a specified project").
 		Metadata(restfulspec.KeyOpenAPITags, []string{constants.NamespaceResourcesTag}).

pkg/models/gateway/gateway.go

@@ -0,0 +1,248 @@
+/*
+Copyright 2021 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gateway
+
+import (
+	"context"
+	"fmt"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	"kubesphere.io/api/gateway/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
+)
+
+const (
+	gatewayPrefix     = "kubesphere-router-"
+	workingNamespace  = "kubesphere-controls-system"
+	globalGatewayname = gatewayPrefix + "kubesphere-system"
+	helmPatch         = `{"metadata":{"annotations":{"meta.helm.sh/release-name":"%s-ingress","meta.helm.sh/release-namespace":"%s"},"labels":{"helm.sh/chart":"ingress-nginx-3.35.0","app.kubernetes.io/managed-by":"Helm","app":null,"component":null,"tier":null}}}`
+)
+
+type GatewayOperator interface {
+	GetGateways(namespace string) ([]*v1alpha1.Gateway, error)
+	CreateGateway(namespace string, obj *v1alpha1.Gateway) (*v1alpha1.Gateway, error)
+	DeleteGateway(namespace string) error
+	UpdateGateway(namespace string, obj *v1alpha1.Gateway) (*v1alpha1.Gateway, error)
+	UpgradeGateway(namespace string) (*v1alpha1.Gateway, error)
+}
+
+type gatewayOperator struct {
+	client  client.Client
+	options *gateway.Options
+}
+
+func NewGatewayOperator(client client.Client, options *gateway.Options) GatewayOperator {
+	return &gatewayOperator{
+		client:  client,
+		options: options,
+	}
+}
+
+func (c *gatewayOperator) getWorkingNamespace(namespace string) string {
+	ns := c.options.Namespace
+	// Set the working namespace to watching namespace when the Gatway's Namsapce Option is empty
+	if ns == "" {
+		ns = namespace
+	}
+	return ns
+}
+
+// overide user's setting when create/update a project gateway.
+func (c *gatewayOperator) overideDefaultValue(gateway *v1alpha1.Gateway, namespace string) *v1alpha1.Gateway {
+	// overide default name
+	gateway.Name = fmt.Sprint(gatewayPrefix, namespace)
+	if gateway.Name != globalGatewayname {
+		gateway.Spec.Conroller.Scope = v1alpha1.Scope{Enabled: true, Namespace: namespace}
+	}
+	gateway.Namespace = c.getWorkingNamespace(namespace)
+	return gateway
+}
+
+// getGlobalGateway returns the global gateway
+func (c *gatewayOperator) getGlobalGateway() *v1alpha1.Gateway {
+	globalkey := types.NamespacedName{
+		Namespace: workingNamespace,
+		Name:      globalGatewayname,
+	}
+
+	global := &v1alpha1.Gateway{}
+	if err := c.client.Get(context.TODO(), globalkey, global); err != nil {
+		return nil
+	}
+	return global
+}
+
+// getLegacyGateway returns gateway created by the router api.
+// Should always prompt user to upgrade the gateway.
+func (c *gatewayOperator) getLegacyGateway(namespace string) *v1alpha1.Gateway {
+	var legacy v1alpha1.Gateway
+
+	s := &corev1.ServiceList{}
+
+	// filter legacy service by labels
+	_ = c.client.List(context.TODO(), s, &client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(
+			labels.Set{
+				"app":       "kubesphere",
+				"component": "ks-router",
+				"tier":      "backend",
+				"project":   namespace,
+			}),
+	})
+
+	// create a fake Gateway object when legacy service exists
+	if len(s.Items) > 0 {
+		svc := s.Items[0]
+		legacy = v1alpha1.Gateway{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "",
+				APIVersion: "",
+			},
+			ObjectMeta: v1.ObjectMeta{
+				Name:      svc.Name,
+				Namespace: svc.Namespace,
+			},
+			Spec: v1alpha1.GatewaySpec{
+				Conroller: v1alpha1.ControllerSpec{
+					Scope: v1alpha1.Scope{
+						Enabled:   true,
+						Namespace: namespace,
+					},
+				},
+				Service: v1alpha1.ServiceSpec{
+					Annotations: svc.Annotations,
+					Type:        svc.Spec.Type,
+				},
+			},
+		}
+		return &legacy
+	}
+	return nil
+}
+
+// GetGateways returns all Gateways from the project. There are at most 2 gatways exists in a project,
+// a Glabal Gateway and a Project Gateway or a Legacy Project Gateway.
+func (c *gatewayOperator) GetGateways(namespace string) ([]*v1alpha1.Gateway, error) {
+
+	var gateways []*v1alpha1.Gateway
+
+	if g := c.getGlobalGateway(); g != nil {
+		gateways = append(gateways, g)
+	}
+	if g := c.getLegacyGateway(namespace); g != nil {
+		gateways = append(gateways, g)
+	}
+
+	key := types.NamespacedName{
+		Namespace: c.getWorkingNamespace(namespace),
+		Name:      fmt.Sprint(gatewayPrefix, namespace),
+	}
+	obj := &v1alpha1.Gateway{}
+	err := c.client.Get(context.TODO(), key, obj)
+	if errors.IsNotFound(err) {
+		return gateways, nil
+	} else if err != nil {
+		return nil, err
+	}
+	gateways = append(gateways, obj)
+	return gateways, err
+}
+
+// Create a Gateway in a namespace
+func (c *gatewayOperator) CreateGateway(namespace string, obj *v1alpha1.Gateway) (*v1alpha1.Gateway, error) {
+
+	if g := c.getGlobalGateway(); g != nil {
+		return nil, fmt.Errorf("can't create project gateway if global gateway enabled")
+	}
+
+	if g := c.getLegacyGateway(namespace); g != nil {
+		return nil, fmt.Errorf("can't create project gateway if legacy gateway exists, please upgrade the gateway firstly")
+	}
+
+	c.overideDefaultValue(obj, namespace)
+	err := c.client.Create(context.TODO(), obj)
+	return obj, err
+}
+
+// DeleteGateway is used to delete Gateway related resources in the namespace
+func (c *gatewayOperator) DeleteGateway(namespace string) error {
+	obj := &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Namespace: c.getWorkingNamespace(namespace),
+			Name:      fmt.Sprint(gatewayPrefix, namespace),
+		},
+	}
+	return c.client.Delete(context.TODO(), obj)
+}
+
+// Update Gateway
+func (c *gatewayOperator) UpdateGateway(namespace string, obj *v1alpha1.Gateway) (*v1alpha1.Gateway, error) {
+	if c.options.Namespace == "" && obj.Namespace != namespace || c.options.Namespace != "" && c.options.Namespace != obj.Namespace {
+		return nil, fmt.Errorf("namepsace doesn't match with origin namesapce")
+	}
+	c.overideDefaultValue(obj, namespace)
+	err := c.client.Update(context.TODO(), obj)
+	return obj, err
+}
+
+// UpgradeGateway upgrade the legacy Project Gateway to a Gateway CRD.
+// No rolling upgrade guaranteed, Service would be interrupted when deleting old deployment.
+func (c *gatewayOperator) UpgradeGateway(namespace string) (*v1alpha1.Gateway, error) {
+	l := c.getLegacyGateway(namespace)
+	if l == nil {
+		return nil, fmt.Errorf("invalid operation, no legacy gateway was found")
+	}
+	if l.Namespace != c.options.Namespace {
+		return nil, fmt.Errorf("invalid operation, can't upgrade legacy gateway when working namespace changed")
+	}
+
+	// Delete old deployment, because it's not compatile with the deployment in the helm chart.
+	d := &appsv1.Deployment{
+		ObjectMeta: v1.ObjectMeta{
+			Namespace: l.Namespace,
+			Name:      l.Name,
+		},
+	}
+	err := c.client.Delete(context.TODO(), d)
+	if err != nil && !errors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// Patch the legacy Serivce with helm annotations, So that it can be mannaged by the helm release.
+	patch := []byte(fmt.Sprintf(helmPatch, l.Name, l.Namespace))
+	err = c.client.Patch(context.Background(), &corev1.Service{
+		ObjectMeta: v1.ObjectMeta{
+			Namespace: l.Namespace,
+			Name:      l.Name,
+		},
+	}, client.RawPatch(types.StrategicMergePatchType, patch))
+
+	if err != nil {
+		return nil, err
+	}
+
+	c.overideDefaultValue(l, namespace)
+	err = c.client.Create(context.TODO(), l)
+	return l, err
+}

pkg/models/gateway/gateway_test.go

@@ -0,0 +1,582 @@
+/*
+Copyright 2021 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package gateway
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/diff"
+	"kubesphere.io/api/gateway/v1alpha1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"kubesphere.io/kubesphere/pkg/simple/client/gateway"
+)
+
+func Test_gatewayOperator_GetGateways(t *testing.T) {
+
+	type fields struct {
+		client  client.Client
+		options *gateway.Options
+	}
+	type args struct {
+		namespace string
+	}
+
+	var Scheme = runtime.NewScheme()
+	v1alpha1.AddToScheme(Scheme)
+	client := fake.NewFakeClientWithScheme(Scheme)
+
+	client.Create(context.TODO(), &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "kubesphere-router-project3",
+			Namespace: "project3",
+		},
+	})
+
+	client.Create(context.TODO(), &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "kubesphere-router-project4",
+			Namespace: "kubesphere-controls-system",
+		},
+	})
+
+	client2 := fake.NewFakeClientWithScheme(Scheme)
+	create_GlobalGateway(client2)
+
+	corev1.AddToScheme(Scheme)
+	client3 := fake.NewFakeClientWithScheme(Scheme)
+	create_LegacyGateway(client3, "project6")
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    []*v1alpha1.Gateway
+		wantErr bool
+	}{
+		{
+			name: "return empty gateway list from watching namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "projct1",
+			},
+		},
+		{
+			name: "return empty gateway list from working namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "kubesphere-controls-system",
+				},
+			},
+			args: args{
+				namespace: "projct1",
+			},
+		},
+		{
+			name: "get gateway from watching namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "project3",
+			},
+			want: wantedResult("kubesphere-router-project3", "project3"),
+		},
+		{
+			name: "get gateway from working namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "kubesphere-controls-system",
+				},
+			},
+			args: args{
+				namespace: "project4",
+			},
+			want: wantedResult("kubesphere-router-project4", "kubesphere-controls-system"),
+		},
+		{
+			name: "get global gateway",
+			fields: fields{
+				client: client2,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "project5",
+			},
+			want: wantedResult("kubesphere-router-kubesphere-system", "kubesphere-controls-system"),
+		},
+		{
+			name: "get Legacy gateway",
+			fields: fields{
+				client: client3,
+				options: &gateway.Options{
+					Namespace: "kubesphere-controls-system",
+				},
+			},
+			args: args{
+				namespace: "project6",
+			},
+			want: []*v1alpha1.Gateway{
+				{
+					ObjectMeta: v1.ObjectMeta{
+						Name:      fmt.Sprint(gatewayPrefix, "project6"),
+						Namespace: "kubesphere-controls-system",
+					},
+					Spec: v1alpha1.GatewaySpec{
+						Conroller: v1alpha1.ControllerSpec{
+							Scope: v1alpha1.Scope{
+								Enabled:   true,
+								Namespace: "project6",
+							},
+						},
+						Service: v1alpha1.ServiceSpec{
+							Annotations: map[string]string{
+								"fake": "true",
+							},
+							Type: corev1.ServiceTypeNodePort,
+						},
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &gatewayOperator{
+				client:  tt.fields.client,
+				options: tt.fields.options,
+			}
+			got, err := c.GetGateways(tt.args.namespace)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("gatewayOperator.GetGateways() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("gatewayOperator.GetGateways() has wrong object\nDiff:\n %s", diff.ObjectGoPrintSideBySide(tt.want, got))
+			}
+		})
+	}
+}
+
+func wantedResult(name, namspace string) []*v1alpha1.Gateway {
+	return []*v1alpha1.Gateway{
+		{
+			TypeMeta: v1.TypeMeta{
+				Kind:       "Gateway",
+				APIVersion: "gateway.kubesphere.io/v1alpha1",
+			},
+			ObjectMeta: v1.ObjectMeta{
+				Name:            name,
+				Namespace:       namspace,
+				ResourceVersion: "1",
+			},
+		},
+	}
+}
+
+func create_GlobalGateway(c client.Client) *v1alpha1.Gateway {
+	g := &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "kubesphere-router-kubesphere-system",
+			Namespace: "kubesphere-controls-system",
+		},
+	}
+	_ = c.Create(context.TODO(), g)
+	return g
+}
+
+func create_LegacyGateway(c client.Client, namespace string) {
+	s := &corev1.Service{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      fmt.Sprint(gatewayPrefix, namespace),
+			Namespace: workingNamespace,
+			Annotations: map[string]string{
+				"fake": "true",
+			},
+			Labels: map[string]string{
+				"app":       "kubesphere",
+				"component": "ks-router",
+				"tier":      "backend",
+				"project":   namespace,
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeNodePort,
+		},
+	}
+	c.Create(context.TODO(), s)
+}
+
+func Test_gatewayOperator_CreateGateway(t *testing.T) {
+	type fields struct {
+		client  client.Client
+		options *gateway.Options
+	}
+	type args struct {
+		namespace string
+		obj       *v1alpha1.Gateway
+	}
+
+	var Scheme = runtime.NewScheme()
+	v1alpha1.AddToScheme(Scheme)
+	client := fake.NewFakeClientWithScheme(Scheme)
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    func(GatewayOperator, string) *v1alpha1.Gateway
+		wantErr bool
+	}{
+		{
+			name: "creates gateway in watching namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "projct1",
+				obj: &v1alpha1.Gateway{
+					TypeMeta: v1.TypeMeta{
+						Kind:       "Gateway",
+						APIVersion: "gateway.kubesphere.io/v1alpha1",
+					},
+					Spec: v1alpha1.GatewaySpec{
+						Conroller: v1alpha1.ControllerSpec{
+							Scope: v1alpha1.Scope{
+								Enabled:   true,
+								Namespace: "projct1",
+							},
+						},
+					},
+				},
+			},
+			want: func(o GatewayOperator, s string) *v1alpha1.Gateway {
+				g, _ := o.GetGateways(s)
+				return g[0]
+			},
+		},
+		{
+			name: "creates gateway in working namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "kubesphere-controls-system",
+				},
+			},
+			args: args{
+				namespace: "projct2",
+				obj: &v1alpha1.Gateway{
+					TypeMeta: v1.TypeMeta{
+						Kind:       "Gateway",
+						APIVersion: "gateway.kubesphere.io/v1alpha1",
+					},
+					Spec: v1alpha1.GatewaySpec{
+						Conroller: v1alpha1.ControllerSpec{
+							Scope: v1alpha1.Scope{
+								Enabled:   true,
+								Namespace: "projct2",
+							},
+						},
+					},
+				},
+			},
+			want: func(o GatewayOperator, s string) *v1alpha1.Gateway {
+				g, _ := o.GetGateways(s)
+				return g[0]
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &gatewayOperator{
+				client:  tt.fields.client,
+				options: tt.fields.options,
+			}
+			got, err := c.CreateGateway(tt.args.namespace, tt.args.obj)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("gatewayOperator.CreateGateway() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			w := tt.want(c, tt.args.namespace)
+			if !reflect.DeepEqual(got, w) {
+				t.Errorf("gatewayOperator.CreateGateway() has wrong object\nDiff:\n %s", diff.ObjectGoPrintSideBySide(w, got))
+			}
+		})
+	}
+}
+
+func Test_gatewayOperator_DeleteGateway(t *testing.T) {
+	type fields struct {
+		client  client.Client
+		options *gateway.Options
+	}
+	type args struct {
+		namespace string
+	}
+
+	var Scheme = runtime.NewScheme()
+	v1alpha1.AddToScheme(Scheme)
+	client := fake.NewFakeClientWithScheme(Scheme)
+
+	client.Create(context.TODO(), &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "kubesphere-router-project1",
+			Namespace: "project1",
+		},
+	})
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "delete gateway",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "project1",
+			},
+		},
+		{
+			name: "delete none exist gateway",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "project2",
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &gatewayOperator{
+				client:  tt.fields.client,
+				options: tt.fields.options,
+			}
+			if err := c.DeleteGateway(tt.args.namespace); (err != nil) != tt.wantErr {
+				t.Errorf("gatewayOperator.DeleteGateway() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_gatewayOperator_UpdateGateway(t *testing.T) {
+	type fields struct {
+		client  client.Client
+		options *gateway.Options
+	}
+	type args struct {
+		namespace string
+		obj       *v1alpha1.Gateway
+	}
+
+	var Scheme = runtime.NewScheme()
+	v1alpha1.AddToScheme(Scheme)
+	client := fake.NewFakeClientWithScheme(Scheme)
+
+	client.Create(context.TODO(), &v1alpha1.Gateway{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "kubesphere-router-project3",
+			Namespace: "project3",
+		},
+	})
+
+	obj := &v1alpha1.Gateway{
+		TypeMeta: v1.TypeMeta{
+			Kind:       "Gateway",
+			APIVersion: "gateway.kubesphere.io/v1alpha1",
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:            "kubesphere-router-project3",
+			Namespace:       "project3",
+			ResourceVersion: "1",
+		},
+		Spec: v1alpha1.GatewaySpec{
+			Conroller: v1alpha1.ControllerSpec{
+				Scope: v1alpha1.Scope{
+					Enabled:   true,
+					Namespace: "project3",
+				},
+			},
+		},
+	}
+
+	want := obj.DeepCopy()
+	want.ResourceVersion = "2"
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    *v1alpha1.Gateway
+		wantErr bool
+	}{
+		{
+			name: "update gateway from watching namespace",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "project3",
+				obj:       obj,
+			},
+			want: want,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &gatewayOperator{
+				client:  tt.fields.client,
+				options: tt.fields.options,
+			}
+			got, err := c.UpdateGateway(tt.args.namespace, tt.args.obj)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("gatewayOperator.UpdateGateway() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("gatewayOperator.UpdateGateway() has wrong object\nDiff:\n %s", diff.ObjectGoPrintSideBySide(tt.want, got))
+			}
+		})
+	}
+}
+
+func Test_gatewayOperator_UpgradeGateway(t *testing.T) {
+	type fields struct {
+		client  client.Client
+		options *gateway.Options
+	}
+	type args struct {
+		namespace string
+	}
+
+	var Scheme = runtime.NewScheme()
+	v1alpha1.AddToScheme(Scheme)
+	client := fake.NewFakeClientWithScheme(Scheme)
+
+	corev1.AddToScheme(Scheme)
+	appsv1.AddToScheme(Scheme)
+	client2 := fake.NewFakeClientWithScheme(Scheme)
+	create_LegacyGateway(client2, "project2")
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    *v1alpha1.Gateway
+		wantErr bool
+	}{
+		{
+			name: "no legacy gateway exists",
+			fields: fields{
+				client: client,
+				options: &gateway.Options{
+					Namespace: "",
+				},
+			},
+			args: args{
+				namespace: "projct1",
+			},
+			wantErr: true,
+		},
+		{
+			name: "upgrade legacy gateway",
+			fields: fields{
+				client: client2,
+				options: &gateway.Options{
+					Namespace: "kubesphere-controls-system",
+				},
+			},
+			args: args{
+				namespace: "project2",
+			},
+			want: &v1alpha1.Gateway{
+				ObjectMeta: v1.ObjectMeta{
+					Name:            "kubesphere-router-project2",
+					Namespace:       "kubesphere-controls-system",
+					ResourceVersion: "1",
+				},
+				Spec: v1alpha1.GatewaySpec{
+					Conroller: v1alpha1.ControllerSpec{
+						Scope: v1alpha1.Scope{
+							Enabled:   true,
+							Namespace: "project2",
+						},
+					},
+					Service: v1alpha1.ServiceSpec{
+						Annotations: map[string]string{
+							"fake": "true",
+						},
+						Type: corev1.ServiceTypeNodePort,
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			c := &gatewayOperator{
+				client:  tt.fields.client,
+				options: tt.fields.options,
+			}
+			got, err := c.UpgradeGateway(tt.args.namespace)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("gatewayOperator.UpgradeGateway() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("gatewayOperator.UpgradeGateway() has wrong object\nDiff:\n %s", diff.ObjectGoPrintSideBySide(tt.want, got))
+			}
+		})
+	}
+}

staging/src/kubesphere.io/api/gateway/group.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2021 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package gateway contains gateway API versions
+package gateway

staging/src/kubesphere.io/api/gateway/v1alpha1/gateway_types.go

@@ -0,0 +1,86 @@
+/*
+Copyright 2021 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+)
+
+// GatewaySpec defines the desired state of Gateway
+type GatewaySpec struct {
+	Conroller  ControllerSpec `json:"controller,omitempty"`
+	Service    ServiceSpec    `json:"service,omitempty"`
+	Deployment DeploymentSpec `json:"deployment,omitempty"`
+}
+
+type ControllerSpec struct {
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty" protobuf:"varint,1,opt,name=replicas"`
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// +optional
+	Scope Scope `json:"scope,omitempty"`
+}
+
+type ServiceSpec struct {
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+	// +optional
+	Type corev1.ServiceType `json:"type,omitempty"`
+}
+
+type DeploymentSpec struct {
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty" protobuf:"varint,1,opt,name=replicas"`
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+type Scope struct {
+	Enabled   bool   `json:"enabled,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+//+kubebuilder:subresource:status
+//+genclient
+
+// Gateway is the Schema for the gateways API
+type Gateway struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec GatewaySpec `json:"spec,omitempty"`
+
+	// +kubebuilder:pruning:PreserveUnknownFields
+	Status runtime.RawExtension `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// GatewayList contains a list of Gateway
+type GatewayList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Gateway `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Gateway{}, &GatewayList{})
+}

staging/src/kubesphere.io/api/gateway/v1alpha1/register.go

@@ -0,0 +1,41 @@
+/*
+Copyright 2021 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1alpha1 contains API Schema definitions for the gateway.kubesphere.io v1alpha1 API group
+//+kubebuilder:object:generate=true
+//+groupName=gateway.kubesphere.io
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	SchemeGroupVersion = schema.GroupVersion{Group: "gateway.kubesphere.io", Version: "v1alpha1"}
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// Resource is required by pkg/client/listers/...
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}

staging/src/kubesphere.io/api/gateway/v1alpha1/zz_generated.deepcopy.go

@@ -0,0 +1,194 @@
+// +build !ignore_autogenerated
+
+/*
+Copyright 2020 The KubeSphere Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ControllerSpec) DeepCopyInto(out *ControllerSpec) {
+	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	out.Scope = in.Scope
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerSpec.
+func (in *ControllerSpec) DeepCopy() *ControllerSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ControllerSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) {
+	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentSpec.
+func (in *DeploymentSpec) DeepCopy() *DeploymentSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(DeploymentSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Gateway) DeepCopyInto(out *Gateway) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Gateway.
+func (in *Gateway) DeepCopy() *Gateway {
+	if in == nil {
+		return nil
+	}
+	out := new(Gateway)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *Gateway) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GatewayList) DeepCopyInto(out *GatewayList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]Gateway, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewayList.
+func (in *GatewayList) DeepCopy() *GatewayList {
+	if in == nil {
+		return nil
+	}
+	out := new(GatewayList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *GatewayList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *GatewaySpec) DeepCopyInto(out *GatewaySpec) {
+	*out = *in
+	in.Conroller.DeepCopyInto(&out.Conroller)
+	in.Service.DeepCopyInto(&out.Service)
+	in.Deployment.DeepCopyInto(&out.Deployment)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GatewaySpec.
+func (in *GatewaySpec) DeepCopy() *GatewaySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(GatewaySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Scope) DeepCopyInto(out *Scope) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Scope.
+func (in *Scope) DeepCopy() *Scope {
+	if in == nil {
+		return nil
+	}
+	out := new(Scope)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ServiceSpec) DeepCopyInto(out *ServiceSpec) {
+	*out = *in
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceSpec.
+func (in *ServiceSpec) DeepCopy() *ServiceSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ServiceSpec)
+	in.DeepCopyInto(out)
+	return out
+}