admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add cluster proxy gerneration code (#2042)

Browse Source
This commit is contained in:
zryfish
2020-04-30 22:33:23 +08:00
committed by GitHub
parent cdd116242a
commit 933207d247
32 changed files with 4812 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
pkg/apiserver/apiserver.go
View File

@@ -30,6 +30,7 @@ import (
"kubesphere.io/kubesphere/pkg/apiserver/filters"
"kubesphere.io/kubesphere/pkg/apiserver/request"
"kubesphere.io/kubesphere/pkg/informers"
clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
@@ -139,6 +140,9 @@ func (s *APIServer) PrepareRun() error {
return nil
}
// Install all kubesphere api groups
// Installation happens before all informers start to cache objects, so
// any attempt to list objects using listers will get empty results.
func (s *APIServer) installKubeSphereAPIs() {
urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory))
@@ -150,12 +154,26 @@ func (s *APIServer) installKubeSphereAPIs() {
urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory))
urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory))
urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config()))
urlruntime.Must(iamapi.AddToContainer(s.container, im.NewOperator(s.KubernetesClient.KubeSphere(), s.InformerFactory),
urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
s.InformerFactory.KubernetesSharedInformerFactory(),
s.InformerFactory.KubeSphereSharedInformerFactory(),
s.Config.MultiClusterOptions.ProxyPublishService,
s.Config.MultiClusterOptions.ProxyPublishAddress,
s.Config.MultiClusterOptions.AgentImage))
urlruntime.Must(iamapi.AddToContainer(s.container,
im.NewOperator(s.KubernetesClient.KubeSphere(), s.InformerFactory),
am.NewAMOperator(s.InformerFactory),
s.Config.AuthenticationOptions))
urlruntime.Must(oauth.AddToContainer(s.container, token.NewJwtTokenIssuer(token.DefaultIssuerName, s.Config.AuthenticationOptions, s.CacheClient), s.Config.AuthenticationOptions))
urlruntime.Must(oauth.AddToContainer(s.container,
token.NewJwtTokenIssuer(token.DefaultIssuerName, s.Config.AuthenticationOptions, s.CacheClient),
s.Config.AuthenticationOptions))
urlruntime.Must(servicemeshv1alpha2.AddToContainer(s.container))
urlruntime.Must(devopsv1alpha2.AddToContainer(s.container, s.InformerFactory.KubeSphereSharedInformerFactory(), s.DevopsClient, s.SonarClient, s.KubernetesClient.KubeSphere(), s.S3Client))
urlruntime.Must(devopsv1alpha2.AddToContainer(s.container,
s.InformerFactory.KubeSphereSharedInformerFactory(),
s.DevopsClient,
s.SonarClient,
s.KubernetesClient.KubeSphere(),
s.S3Client))
}
func (s *APIServer) Run(stopCh <-chan struct{}) (err error) {

173
pkg/kapis/cluster/v1alpha1/handler.go Normal file
View File

@@ -0,0 +1,173 @@
package v1alpha1
import (
"bytes"
"fmt"
"github.com/emicklei/go-restful"
"io"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
v1 "k8s.io/client-go/listers/core/v1"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1"
clusterlister "kubesphere.io/kubesphere/pkg/client/listers/cluster/v1alpha1"
"strings"
"k8s.io/cli-runtime/pkg/printers"
)
const (
defaultAgentImage = "kubesphere/tower:v1.0"
)
type handler struct {
serviceLister v1.ServiceLister
clusterLister clusterlister.ClusterLister
proxyService string
proxyAddress string
agentImage string
yamlPrinter *printers.YAMLPrinter
}
func NewHandler(serviceLister v1.ServiceLister, clusterLister clusterlister.ClusterLister, proxyService, proxyAddress, agentImage string) *handler {
if len(agentImage) == 0 {
agentImage = defaultAgentImage
}
return &handler{
serviceLister: serviceLister,
clusterLister: clusterLister,
proxyService: proxyService,
proxyAddress: proxyAddress,
agentImage: agentImage,
yamlPrinter: &printers.YAMLPrinter{},
}
}
func (h *handler) GenerateAgentDeployment(request *restful.Request, response *restful.Response) {
clusterName := request.PathParameter("cluster")
cluster, err := h.clusterLister.Get(clusterName)
if err != nil {
if errors.IsNotFound(err) {
api.HandleNotFound(response, request, err)
return
} else {
api.HandleInternalError(response, request, err)
return
}
}
// use service ingress address
if len(h.proxyAddress) == 0 {
err = h.populateProxyAddress()
if err != nil {
api.HandleNotFound(response, request, err)
return
}
}
var buf bytes.Buffer
err = h.generateDefaultDeployment(cluster, &buf)
if err != nil {
api.HandleInternalError(response, request, err)
return
}
response.Write(buf.Bytes())
}
//
func (h *handler) populateProxyAddress() error {
if len(h.proxyService) == 0 {
return fmt.Errorf("neither proxy address nor proxy service provided")
}
namespace := "kubesphere-system"
parts := strings.Split(h.proxyService, ".")
if len(parts) > 1 && len(parts[1]) != 0 {
namespace = parts[1]
}
service, err := h.serviceLister.Services(namespace).Get(parts[0])
if err != nil {
return err
}
if len(service.Spec.Ports) == 0 {
return fmt.Errorf("there are no ports in proxy service spec")
}
port := service.Spec.Ports[0].Port
var serviceAddress string
for _, ingress := range service.Status.LoadBalancer.Ingress {
if len(ingress.Hostname) != 0 {
serviceAddress = fmt.Sprintf("http://%s:%d", ingress.Hostname, port)
}
if len(ingress.IP) != 0 {
serviceAddress = fmt.Sprintf("http://%s:%d", ingress.IP, port)
}
}
if len(serviceAddress) == 0 {
return fmt.Errorf("service ingress is empty")
}
h.proxyAddress = serviceAddress
return nil
}
func (h *handler) generateDefaultDeployment(cluster *v1alpha1.Cluster, w io.Writer) error {
agent := appsv1.Deployment{
TypeMeta: metav1.TypeMeta{
Kind: "Deployment",
APIVersion: "apps/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "cluster-agent",
},
Spec: appsv1.DeploymentSpec{
Selector: &metav1.LabelSelector{},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "agent",
Command: []string{
"/agent",
fmt.Sprintf("--name=%s", cluster.Name),
fmt.Sprintf("--token=%s", cluster.Spec.Connection.Token),
fmt.Sprintf("--proxy-server=%s", h.proxyAddress),
fmt.Sprintf("--kubesphere-service=ks-apiserver.kubesphere-system.svc:80"),
fmt.Sprintf("--kubernetes-service=kubernetes.default.svc:443"),
},
Image: h.agentImage,
Resources: corev1.ResourceRequirements{
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
corev1.ResourceMemory: resource.MustParse("200M"),
},
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("100M"),
},
},
},
},
ServiceAccountName: "kubesphere",
},
},
},
}
return h.yamlPrinter.PrintObj(&agent, w)
}

141
pkg/kapis/cluster/v1alpha1/handler_test.go Normal file
View File

@@ -0,0 +1,141 @@
package v1alpha1
import (
"bytes"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/cli-runtime/pkg/printers"
fake2 "k8s.io/client-go/kubernetes/fake"
"kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1"
"kubesphere.io/kubesphere/pkg/client/clientset/versioned/fake"
"kubesphere.io/kubesphere/pkg/informers"
"testing"
)
const (
proxyAddress = "http://139.198.121.121:8080"
agentImage = "kubesphere/tower:v1.0"
proxyService = "tower.kubesphere-system.svc"
)
var cluster = &v1alpha1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "gondor",
},
Spec: v1alpha1.ClusterSpec{
Connection: v1alpha1.Connection{
Type: v1alpha1.ConnectionTypeProxy,
Token: "randomtoken",
},
},
}
var service = &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "tower",
Namespace: "kubesphere-system",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Port: 8080,
Protocol: corev1.ProtocolTCP,
},
},
},
Status: corev1.ServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{
Ingress: []corev1.LoadBalancerIngress{
{
IP: "139.198.121.121",
Hostname: "foo.bar",
},
},
},
},
}
var expected = `apiVersion: apps/v1
kind: Deployment
metadata:
creationTimestamp: null
name: cluster-agent
spec:
selector: {}
strategy: {}
template:
metadata:
creationTimestamp: null
spec:
containers:
- command:
- /agent
- --name=gondor
- --token=randomtoken
- --proxy-server=http://139.198.121.121:8080
- --kubesphere-service=ks-apiserver.kubesphere-system.svc:80
- --kubernetes-service=kubernetes.default.svc:443
image: kubesphere/tower:v1.0
name: agent
resources:
limits:
cpu: "1"
memory: 200M
requests:
cpu: 100m
memory: 100M
serviceAccountName: kubesphere
status: {}
`
func TestGeranteAgentDeployment(t *testing.T) {
k8sclient := fake2.NewSimpleClientset(service)
ksclient := fake.NewSimpleClientset(cluster)
informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil)
informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service)
informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster)
h := NewHandler(informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Lister(),
informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Lister(),
proxyService,
"",
agentImage)
var buf bytes.Buffer
err := h.populateProxyAddress()
if err != nil {
t.Error(err)
}
err = h.generateDefaultDeployment(cluster, &buf)
if diff := cmp.Diff(buf.String(), expected); len(diff) != 0 {
t.Error(diff)
}
}
func TestInnerGenerateAgentDeployment(t *testing.T) {
h := &handler{
proxyAddress: proxyAddress,
agentImage: agentImage,
yamlPrinter: &printers.YAMLPrinter{},
}
var buf bytes.Buffer
err := h.generateDefaultDeployment(cluster, &buf)
if err != nil {
t.Error(err)
}
t.Log(buf.String())
if diff := cmp.Diff(buf.String(), expected); len(diff) != 0 {
t.Error(diff)
}
}

39
pkg/kapis/cluster/v1alpha1/register.go Normal file
View File

@@ -0,0 +1,39 @@
package v1alpha1
import (
"github.com/emicklei/go-restful"
"k8s.io/apimachinery/pkg/runtime/schema"
k8sinformers "k8s.io/client-go/informers"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
"kubesphere.io/kubesphere/pkg/client/informers/externalversions"
"net/http"
)
const (
GroupName = "cluster.kubesphere.io"
)
var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha1"}
func AddToContainer(container *restful.Container,
k8sInformers k8sinformers.SharedInformerFactory,
ksInformers externalversions.SharedInformerFactory,
proxyService string,
proxyAddress string,
agentImage string) error {
webservice := runtime.NewWebService(GroupVersion)
h := NewHandler(k8sInformers.Core().V1().Services().Lister(), ksInformers.Cluster().V1alpha1().Clusters().Lister(), proxyService, proxyAddress, agentImage)
// returns deployment yaml for cluster agent
webservice.Route(webservice.GET("/clusters/{cluster}/agent/deployment").
Doc("Return deployment yaml for cluster agent.").
Param(webservice.PathParameter("cluster", "Name of the cluster.").Required(true)).
To(h.GenerateAgentDeployment).
Returns(http.StatusOK, api.StatusOK, nil))
container.Add(webservice)
return nil
}

4
pkg/kapis/config/v1alpha2/register.go
View File

@@ -21,7 +21,7 @@ package v1alpha2
import (
"github.com/emicklei/go-restful"
"k8s.io/apimachinery/pkg/runtime/schema"
apiserverconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
kubesphereconfig "kubesphere.io/kubesphere/pkg/apiserver/config"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
)
@@ -31,7 +31,7 @@ const (
var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
func AddToContainer(c *restful.Container, config *apiserverconfig.Config) error {
func AddToContainer(c *restful.Container, config *kubesphereconfig.Config) error {
webservice := runtime.NewWebService(GroupVersion)
webservice.Route(webservice.GET("/configs/oauth").

32
pkg/simple/client/multicluster/options.go
View File

@@ -6,13 +6,30 @@ type Options struct {
// Enable
Enable bool `json:"enable"`
EnableFederation bool `json:"enableFederation,omitempty"`
// ProxyPublishService is the service name of multicluster component tower.
// If this field provided, apiserver going to use the ingress.ip of this service.
// This field will be used when generating agent deployment yaml for joining clusters.
ProxyPublishService string `json:"proxyPublishService,omitempty"`
// ProxyPublishAddress is the public address of tower for all cluster agents.
// This field takes precedence over field ProxyPublishService.
// If both field ProxyPublishService and ProxyPublishAddress are empty, apiserver will
// return 404 Not Found for all cluster agent yaml requests.
ProxyPublishAddress string `json:"proxyPublishAddress,omitempty"`
// AgentImage is the image used when generating deployment for all cluster agents.
AgentImage string `json:"agentImage,omitempty"`
}
// NewOptions() returns a default nil options
func NewOptions() *Options {
return &Options{
Enable: false,
EnableFederation: false,
Enable: false,
EnableFederation: false,
ProxyPublishAddress: "",
ProxyPublishService: "",
AgentImage: "kubesphere/tower:v1.0",
}
}
@@ -23,4 +40,15 @@ func (o *Options) Validate() []error {
func (o *Options) AddFlags(fs *pflag.FlagSet, s *Options) {
fs.BoolVar(&o.Enable, "multiple-clusters", s.Enable, ""+
"This field instructs KubeSphere to enter multiple-cluster mode or not.")
fs.StringVar(&o.ProxyPublishService, "proxy-publish-service", s.ProxyPublishService, ""+
"Service name of tower. APIServer will use its ingress address as proxy publish address."+
"For example, tower.kubesphere-system.svc.")
fs.StringVar(&o.ProxyPublishAddress, "proxy-publish-address", s.ProxyPublishAddress, ""+
"Public address of tower, APIServer will use this field as proxy publish address. This field "+
"takes precedence over field proxy-publish-service. For example, 139.198.121.121:8080.")
fs.StringVar(&o.ProxyPublishAddress, "agent-image", s.AgentImage, ""+
"This field is used when generating deployment yaml for agent.")
}