diff --git a/cmd/controller-manager/app/controllers.go b/cmd/controller-manager/app/controllers.go index a93a841d8..89615c725 100644 --- a/cmd/controller-manager/app/controllers.go +++ b/cmd/controller-manager/app/controllers.go @@ -51,7 +51,8 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{ istioInformer.Networking().V1alpha3().DestinationRules(), servicemeshinformer.Servicemesh().V1alpha2().Strategies(), kubeClient, - istioclient) + istioclient, + servicemeshclient) drController := destinationrule.NewDestinationRuleController(informerFactory.Apps().V1().Deployments(), istioInformer.Networking().V1alpha3().DestinationRules(), diff --git a/cmd/ks-apiserver/app/server.go b/cmd/ks-apiserver/app/server.go index b4d1ba45e..84e9319f2 100644 --- a/cmd/ks-apiserver/app/server.go +++ b/cmd/ks-apiserver/app/server.go @@ -114,7 +114,7 @@ func initializeKialiConfig(s *options.ServerRunOptions) { func initializeESClientConfig() { // List all outputs - outputs,err := logging.GetFluentbitOutputFromConfigMap() + outputs, err := logging.GetFluentbitOutputFromConfigMap() if err != nil { glog.Errorln(err) return diff --git a/config/crds/servicemesh_v1alpha2_strategy.yaml b/config/crds/servicemesh_v1alpha2_strategy.yaml index b4a73efba..16bfac105 100644 --- a/config/crds/servicemesh_v1alpha2_strategy.yaml +++ b/config/crds/servicemesh_v1alpha2_strategy.yaml @@ -49,10 +49,6 @@ spec: description: Governor version, the version takes control of all incoming traffic label version value type: string - paused: - description: Indicates that the strategy is paused and will not be processed - by the strategy controller - type: boolean principal: description: Principal version, the one as reference version label version value @@ -60,6 +56,10 @@ spec: selector: description: Label selector for virtual services. type: object + strategyPolicy: + description: strategy policy, how the strategy will be applied by the + strategy controller + type: string template: description: Template describes the virtual service that will be created. properties: diff --git a/config/rbac/rbac_role.yaml b/config/rbac/rbac_role.yaml index 0662cc129..35019132b 100644 --- a/config/rbac/rbac_role.yaml +++ b/config/rbac/rbac_role.yaml @@ -4,50 +4,6 @@ metadata: creationTimestamp: null name: manager-role rules: -- apiGroups: - - networking.istio.io - resources: - - virtualservices - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - networking.istio.io - resources: - - virtualservices/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - servicemesh.kubesphere.io - resources: - - strategies - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - servicemesh.kubesphere.io - resources: - - strategies/status - verbs: - - get - - update - - patch - apiGroups: - admissionregistration.k8s.io resources: diff --git a/pkg/apis/logging/v1alpha2/register.go b/pkg/apis/logging/v1alpha2/register.go index 0e1672b35..2247e7f76 100644 --- a/pkg/apis/logging/v1alpha2/register.go +++ b/pkg/apis/logging/v1alpha2/register.go @@ -214,4 +214,4 @@ func addWebService(c *restful.Container) error { c.Add(ws) return nil -} \ No newline at end of file +} diff --git a/pkg/apis/servicemesh/v1alpha2/strategy_types.go b/pkg/apis/servicemesh/v1alpha2/strategy_types.go index b4197a332..54ceb4d4a 100644 --- a/pkg/apis/servicemesh/v1alpha2/strategy_types.go +++ b/pkg/apis/servicemesh/v1alpha2/strategy_types.go @@ -28,7 +28,6 @@ import ( type StrategyType string const ( - // Canary strategy type CanaryType StrategyType = "Canary" @@ -39,9 +38,21 @@ const ( Mirror StrategyType = "Mirror" ) +type StrategyPolicy string + +const ( + // apply strategy only until workload is ready + PolicyWaitForWorkloadReady StrategyPolicy = "WaitForWorkloadReady" + + // apply strategy immediately no matter workload status is + PolicyImmediately StrategyPolicy = "Immediately" + + // pause strategy + PolicyPause StrategyPolicy = "Paused" +) + // StrategySpec defines the desired state of Strategy type StrategySpec struct { - // Strategy type Type StrategyType `json:"type,omitempty"` @@ -62,9 +73,9 @@ type StrategySpec struct { // Template describes the virtual service that will be created. Template VirtualServiceTemplateSpec `json:"template,omitempty"` - // Indicates that the strategy is paused and will not be processed + // strategy policy, how the strategy will be applied // by the strategy controller - Paused bool `json:"paused,omitempty"` + StrategyPolicy StrategyPolicy `json:"strategyPolicy,omitempty"` } // VirtualServiceTemplateSpec diff --git a/pkg/controller/destinationrule/destinationrule_controller.go b/pkg/controller/destinationrule/destinationrule_controller.go index 90adce819..235f546b5 100644 --- a/pkg/controller/destinationrule/destinationrule_controller.go +++ b/pkg/controller/destinationrule/destinationrule_controller.go @@ -220,10 +220,13 @@ func (v *DestinationRuleController) syncService(key string) error { return nil } - if len(service.Labels) < len(util.ApplicationLabels) || !util.IsApplicationComponent(service.Labels) || + if len(service.Labels) < len(util.ApplicationLabels) || + !util.IsApplicationComponent(service.Labels) || + !util.IsServicemeshEnabled(service.Annotations) || len(service.Spec.Ports) == 0 { // services don't have enough labels to create a virtualservice // or they don't have necessary labels + // or they don't have servicemesh enabled // or they don't have any ports defined return nil } @@ -240,7 +243,10 @@ func (v *DestinationRuleController) syncService(key string) error { for _, deployment := range deployments { // not a valid deployment we required - if !util.IsApplicationComponent(deployment.Labels) || !util.IsApplicationComponent(deployment.Spec.Selector.MatchLabels) { + if !util.IsApplicationComponent(deployment.Labels) || + !util.IsApplicationComponent(deployment.Spec.Selector.MatchLabels) || + deployment.Status.ReadyReplicas == 0 || + !util.IsServicemeshEnabled(deployment.Annotations) { continue } @@ -406,7 +412,9 @@ func (v *DestinationRuleController) getDeploymentServiceMemberShip(deployment *a for i := range allServices { service := allServices[i] - if service.Spec.Selector == nil || !util.IsApplicationComponent(service.Labels) { + if service.Spec.Selector == nil || + !util.IsApplicationComponent(service.Labels) || + !util.IsServicemeshEnabled(service.Annotations) { // services with nil selectors match nothing, not everything. continue } diff --git a/pkg/controller/virtualservice/util/util.go b/pkg/controller/virtualservice/util/util.go index 007337c04..f20d1fba6 100644 --- a/pkg/controller/virtualservice/util/util.go +++ b/pkg/controller/virtualservice/util/util.go @@ -8,17 +8,16 @@ import ( ) const ( - AppLabel = "app" - VersionLabel = "version" - ApplicationNameLabel = "app.kubernetes.io/name" - ApplicationVersionLabel = "app.kubernetes.io/version" - ServiceMeshEnabledLabel = "servicemesh.kubesphere.io/enabled" + AppLabel = "app" + VersionLabel = "version" + ApplicationNameLabel = "app.kubernetes.io/name" + ApplicationVersionLabel = "app.kubernetes.io/version" + ServiceMeshEnabledAnnotation = "servicemesh.kubesphere.io/enabled" ) // resource with these following labels considered as part of servicemesh var ApplicationLabels = [...]string{ ApplicationNameLabel, - ServiceMeshEnabledLabel, AppLabel, } @@ -40,6 +39,15 @@ func GetComponentName(meta *metav1.ObjectMeta) string { return "" } +func IsServicemeshEnabled(annotations map[string]string) bool { + if enabled, ok := annotations[ServiceMeshEnabledAnnotation]; ok { + if enabled == "true" { + return true + } + } + return false +} + func GetComponentVersion(meta *metav1.ObjectMeta) string { if len(meta.Labels[VersionLabel]) > 0 { return meta.Labels[VersionLabel] diff --git a/pkg/controller/virtualservice/virtualservice_controller.go b/pkg/controller/virtualservice/virtualservice_controller.go index 8fb75c435..453d3afc1 100644 --- a/pkg/controller/virtualservice/virtualservice_controller.go +++ b/pkg/controller/virtualservice/virtualservice_controller.go @@ -17,9 +17,8 @@ import ( "k8s.io/kubernetes/pkg/util/metrics" "kubesphere.io/kubesphere/pkg/controller/virtualservice/util" "reflect" - "strings" - logf "sigs.k8s.io/controller-runtime/pkg/runtime/log" + "strings" istioclient "github.com/knative/pkg/client/clientset/versioned" istioinformers "github.com/knative/pkg/client/informers/externalversions/istio/v1alpha3" @@ -31,6 +30,7 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/apis/servicemesh/v1alpha2" + servicemeshclient "kubesphere.io/kubesphere/pkg/client/clientset/versioned" servicemeshinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions/servicemesh/v1alpha2" servicemeshlisters "kubesphere.io/kubesphere/pkg/client/listers/servicemesh/v1alpha2" @@ -52,6 +52,7 @@ type VirtualServiceController struct { client clientset.Interface virtualServiceClient istioclient.Interface + servicemeshClient servicemeshclient.Interface eventBroadcaster record.EventBroadcaster eventRecorder record.EventRecorder @@ -78,7 +79,8 @@ func NewVirtualServiceController(serviceInformer coreinformers.ServiceInformer, destinationRuleInformer istioinformers.DestinationRuleInformer, strategyInformer servicemeshinformers.StrategyInformer, client clientset.Interface, - virtualServiceClient istioclient.Interface) *VirtualServiceController { + virtualServiceClient istioclient.Interface, + servicemeshClient servicemeshclient.Interface) *VirtualServiceController { broadcaster := record.NewBroadcaster() broadcaster.StartLogging(func(format string, args ...interface{}) { @@ -94,6 +96,7 @@ func NewVirtualServiceController(serviceInformer coreinformers.ServiceInformer, v := &VirtualServiceController{ client: client, virtualServiceClient: virtualServiceClient, + servicemeshClient: servicemeshClient, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "virtualservice"), workerLoopPeriod: time.Second, } @@ -234,7 +237,9 @@ func (v *VirtualServiceController) syncService(key string) error { return err } - if len(service.Labels) < len(util.ApplicationLabels) || !util.IsApplicationComponent(service.Labels) || + if len(service.Labels) < len(util.ApplicationLabels) || + !util.IsApplicationComponent(service.Labels) || + !util.IsServicemeshEnabled(service.Annotations) || len(service.Spec.Ports) == 0 { // services don't have enough labels to create a virtualservice // or they don't have necessary labels @@ -294,40 +299,69 @@ func (v *VirtualServiceController) syncService(key string) error { } vs := currentVirtualService.DeepCopy() + // create a whole new virtualservice + + // TODO(jeff): use FQDN to replace service name + vs.Spec.Hosts = []string{name} + + // check if service has TCP protocol ports + for _, port := range service.Spec.Ports { + var route v1alpha3.DestinationWeight + if port.Protocol == v1.ProtocolTCP { + route = v1alpha3.DestinationWeight{ + Destination: v1alpha3.Destination{ + Host: name, + Subset: subsets[0].Name, + Port: v1alpha3.PortSelector{ + Number: uint32(port.Port), + }, + }, + Weight: 100, + } + + // a http port, add to HTTPRoute + if len(port.Name) > 0 && (port.Name == "http" || strings.HasPrefix(port.Name, "http-")) { + vs.Spec.Http = []v1alpha3.HTTPRoute{{Route: []v1alpha3.DestinationWeight{route}}} + break + } + + // everything else treated as TCPRoute + vs.Spec.Tcp = []v1alpha3.TCPRoute{{Route: []v1alpha3.DestinationWeight{route}}} + } + } + if len(strategies) > 0 { // apply strategy spec to virtualservice - vs.Spec = v.generateVirtualServiceSpec(strategies[0], service).Spec - } else { - // create a whole new virtualservice - // TODO(jeff): use FQDN to replace service name - vs.Spec.Hosts = []string{name} + switch strategies[0].Spec.StrategyPolicy { + case servicemeshv1alpha2.PolicyPause: + break + case servicemeshv1alpha2.PolicyWaitForWorkloadReady: + set := v.getSubsets(strategies[0]) - // check if service has TCP protocol ports - for _, port := range service.Spec.Ports { - var route v1alpha3.DestinationWeight - if port.Protocol == v1.ProtocolTCP { - route = v1alpha3.DestinationWeight{ - Destination: v1alpha3.Destination{ - Host: name, - Subset: subsets[0].Name, - Port: v1alpha3.PortSelector{ - Number: uint32(port.Port), - }, - }, - Weight: 100, - } - - // a http port, add to HTTPRoute - if len(port.Name) > 0 && (port.Name == "http" || strings.HasPrefix(port.Name, "http-")) { - vs.Spec.Http = []v1alpha3.HTTPRoute{{Route: []v1alpha3.DestinationWeight{route}}} - break - } - - // everything else treated as TCPRoute - vs.Spec.Tcp = []v1alpha3.TCPRoute{{Route: []v1alpha3.DestinationWeight{route}}} + setNames := sets.String{} + for i := range subsets { + setNames.Insert(subsets[i].Name) } + + nonExist := false + for k := range set { + if !setNames.Has(k) { + nonExist = true + } + } + // strategy has subset that are not ready + if nonExist { + break + } else { + vs.Spec = v.generateVirtualServiceSpec(strategies[0], service).Spec + } + case servicemeshv1alpha2.PolicyImmediately: + vs.Spec = v.generateVirtualServiceSpec(strategies[0], service).Spec + default: + vs.Spec = v.generateVirtualServiceSpec(strategies[0], service).Spec } + } createVirtualService := len(currentVirtualService.ResourceVersion) == 0 @@ -359,7 +393,6 @@ func (v *VirtualServiceController) syncService(key string) error { } if err != nil { - if createVirtualService { v.eventRecorder.Event(newVirtualService, v1.EventTypeWarning, "FailedToCreateVirtualService", fmt.Sprintf("Failed to create virtualservice for service %v/%v: %v", namespace, name, err)) } else { @@ -463,6 +496,34 @@ func (v *VirtualServiceController) handleErr(err error, key interface{}) { utilruntime.HandleError(err) } +func (v *VirtualServiceController) getSubsets(strategy *servicemeshv1alpha2.Strategy) sets.String { + set := sets.String{} + + for _, httpRoute := range strategy.Spec.Template.Spec.Http { + for _, dw := range httpRoute.Route { + set.Insert(dw.Destination.Subset) + } + + if httpRoute.Mirror != nil { + set.Insert(httpRoute.Mirror.Subset) + } + } + + for _, tcpRoute := range strategy.Spec.Template.Spec.Tcp { + for _, dw := range tcpRoute.Route { + set.Insert(dw.Destination.Subset) + } + } + + for _, tlsRoute := range strategy.Spec.Template.Spec.Tls { + for _, dw := range tlsRoute.Route { + set.Insert(dw.Destination.Subset) + } + } + + return set +} + func (v *VirtualServiceController) generateVirtualServiceSpec(strategy *servicemeshv1alpha2.Strategy, service *v1.Service) *v1alpha3.VirtualService { // Define VirtualService to be created @@ -472,7 +533,6 @@ func (v *VirtualServiceController) generateVirtualServiceSpec(strategy *servicem // one version rules them all if len(strategy.Spec.GovernorVersion) > 0 { - governorDestinationWeight := v1alpha3.DestinationWeight{ Destination: v1alpha3.Destination{ Host: service.Name, diff --git a/pkg/models/log/constants.go b/pkg/models/log/constants.go index a184b6f64..cfb8efcce 100644 --- a/pkg/models/log/constants.go +++ b/pkg/models/log/constants.go @@ -27,4 +27,4 @@ const ( QueryLevelWorkload QueryLevelPod QueryLevelContainer -) \ No newline at end of file +) diff --git a/pkg/models/log/logcrd.go b/pkg/models/log/logcrd.go index 3abce8e7d..fa3d686c7 100644 --- a/pkg/models/log/logcrd.go +++ b/pkg/models/log/logcrd.go @@ -248,7 +248,7 @@ func FluentbitOutputInsert(output fb.OutputPlugin) *FluentbitOutputsResult { // 1. Update ConfigMap var outputs []fb.OutputPlugin outputs, err := GetFluentbitOutputFromConfigMap() - if err != nil { + if err != nil { // If the ConfigMap doesn't exist, a new one will be created later glog.Errorln(err) } diff --git a/pkg/models/resources/s2irun.go b/pkg/models/resources/s2irun.go index 8f36084ad..f8f7aa7df 100644 --- a/pkg/models/resources/s2irun.go +++ b/pkg/models/resources/s2irun.go @@ -42,7 +42,7 @@ func (*s2iRunSearcher) match(match map[string]string, item *v1alpha1.S2iRun) boo return false } case status: - if string(item.Status.RunState) != v{ + if string(item.Status.RunState) != v { return false } default: