diff --git a/cmd/controller-manager/app/options/options.go b/cmd/controller-manager/app/options/options.go index 2d98ef8b4..698366695 100644 --- a/cmd/controller-manager/app/options/options.go +++ b/cmd/controller-manager/app/options/options.go @@ -18,6 +18,7 @@ package options import ( "flag" + "k8s.io/apimachinery/pkg/labels" "strings" "time" @@ -49,6 +50,15 @@ type KubeSphereControllerManagerOptions struct { LeaderElect bool LeaderElection *leaderelection.LeaderElectionConfig WebhookCertDir string + + // KubeSphere is using sigs.k8s.io/application as fundamental object to implement Application Management. + // There are other projects also built on sigs.k8s.io/application, when KubeSphere installed along side + // them, conflicts happen. So we leave an option to only reconcile applications matched with the given + // selector. Default will reconcile all applications. + // For example + // "kubesphere.io/creator=" means reconcile applications with this label key + // "!kubesphere.io/creator" means exclude applications with this key + ApplicationSelector string } func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions { @@ -67,8 +77,9 @@ func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions RenewDeadline: 15 * time.Second, RetryPeriod: 5 * time.Second, }, - LeaderElect: false, - WebhookCertDir: "", + LeaderElect: false, + WebhookCertDir: "", + ApplicationSelector: "", } return s @@ -99,6 +110,11 @@ func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets { "if not set, webhook server would look up the server key and certificate in"+ "{TempDir}/k8s-webhook-server/serving-certs") + gfs := fss.FlagSet("generic") + gfs.StringVar(&s.ApplicationSelector, "application-selector", s.ApplicationSelector, ""+ + "Only reconcile application(sigs.k8s.io/application) objects match given selector, this could avoid conflicts with "+ + "other projects built on top of sig-application. Default behavior is to reconcile all of application objects.") + kfs := fss.FlagSet("klog") local := flag.NewFlagSet("klog", flag.ExitOnError) klog.InitFlags(local) @@ -118,6 +134,14 @@ func (s *KubeSphereControllerManagerOptions) Validate() []error { errs = append(errs, s.OpenPitrixOptions.Validate()...) errs = append(errs, s.NetworkOptions.Validate()...) errs = append(errs, s.LdapOptions.Validate()...) + + if len(s.ApplicationSelector) != 0 { + _, err := labels.Parse(s.ApplicationSelector) + if err != nil { + errs = append(errs, err) + } + } + return errs } diff --git a/cmd/controller-manager/app/server.go b/cmd/controller-manager/app/server.go index d0fc7399b..e93c87065 100644 --- a/cmd/controller-manager/app/server.go +++ b/cmd/controller-manager/app/server.go @@ -18,6 +18,8 @@ package app import ( "fmt" + "k8s.io/apimachinery/pkg/labels" + "kubesphere.io/kubesphere/pkg/controller/application" "os" "github.com/spf13/cobra" @@ -28,7 +30,6 @@ import ( "kubesphere.io/kubesphere/cmd/controller-manager/app/options" "kubesphere.io/kubesphere/pkg/apis" controllerconfig "kubesphere.io/kubesphere/pkg/apiserver/config" - appcontroller "kubesphere.io/kubesphere/pkg/controller/application" "kubesphere.io/kubesphere/pkg/controller/namespace" "kubesphere.io/kubesphere/pkg/controller/network/webhooks" "kubesphere.io/kubesphere/pkg/controller/serviceaccount" @@ -46,7 +47,6 @@ import ( "kubesphere.io/kubesphere/pkg/simple/client/s3" "kubesphere.io/kubesphere/pkg/utils/metrics" "kubesphere.io/kubesphere/pkg/utils/term" - application "sigs.k8s.io/application/controllers" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/runtime/signals" @@ -219,16 +219,12 @@ func run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) klog.Fatal("Unable to create namespace controller") } - err = appcontroller.Add(mgr) - if err != nil { - klog.Fatal("Unable to create ks application controller") - } - + selector, _ := labels.Parse(s.ApplicationSelector) applicationReconciler := &application.ApplicationReconciler{ - Scheme: mgr.GetScheme(), - Client: mgr.GetClient(), - Mapper: mgr.GetRESTMapper(), - Log: klogr.New(), + Scheme: mgr.GetScheme(), + Client: mgr.GetClient(), + Mapper: mgr.GetRESTMapper(), + ApplicationSelector: selector, } if err = applicationReconciler.SetupWithManager(mgr); err != nil { klog.Fatal("Unable to create application controller") diff --git a/go.mod b/go.mod index 942e86631..5c3196d49 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/gorilla/websocket v1.4.1 github.com/json-iterator/go v1.1.10 github.com/kelseyhightower/envconfig v1.4.0 // indirect + github.com/kr/text v0.2.0 // indirect github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0 github.com/kubesphere/sonargo v0.0.2 github.com/mitchellh/mapstructure v1.2.2 @@ -69,12 +70,14 @@ require ( github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.6.1 github.com/xanzy/ssh-agent v0.2.1 // indirect - golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de + golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 golang.org/x/net v0.0.0-20200707034311-ab3426394381 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect google.golang.org/grpc v1.30.0 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/cas.v2 v2.2.0 + gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect gopkg.in/square/go-jose.v2 v2.4.0 gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect gopkg.in/src-d/go-git.v4 v4.11.0 @@ -189,6 +192,7 @@ replace ( github.com/brancz/kube-rbac-proxy => github.com/brancz/kube-rbac-proxy v0.5.0 github.com/bshuster-repo/logrus-logstash-hook => github.com/bshuster-repo/logrus-logstash-hook v0.4.1 github.com/bugsnag/bugsnag-go => github.com/bugsnag/bugsnag-go v1.5.0 + github.com/bugsnag/osext => github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b github.com/bugsnag/panicwrap => github.com/bugsnag/panicwrap v1.2.0 github.com/c-bata/go-prompt => github.com/c-bata/go-prompt v0.2.2 github.com/campoy/embedmd => github.com/campoy/embedmd v1.0.0 @@ -207,6 +211,7 @@ replace ( github.com/circonus-labs/circonus-gometrics => github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible github.com/circonus-labs/circonusllhist => github.com/circonus-labs/circonusllhist v0.1.3 github.com/clbanning/x2j => github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec + github.com/cloudflare/cfssl => github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 github.com/cockroachdb/apd => github.com/cockroachdb/apd v1.1.0 github.com/cockroachdb/cockroach-go => github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c github.com/cockroachdb/datadriven => github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa @@ -214,6 +219,9 @@ replace ( github.com/container-storage-interface/spec => github.com/container-storage-interface/spec v1.2.0 github.com/containerd/containerd => github.com/containerd/containerd v1.3.0 github.com/containerd/continuity => github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 + github.com/containerd/fifo => github.com/containerd/fifo v0.0.0-20210129194248-f8e8fdba47ef + github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2 + github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.1 github.com/containernetworking/cni => github.com/containernetworking/cni v0.8.0 github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.3 github.com/coreos/etcd => github.com/coreos/etcd v3.3.17+incompatible @@ -247,16 +255,21 @@ replace ( github.com/dhui/dktest => github.com/dhui/dktest v0.3.0 github.com/disintegration/imaging => github.com/disintegration/imaging v1.6.1 github.com/docker/cli => github.com/docker/cli v0.0.0-20190506213505-d88565df0c2d + github.com/docker/compose-on-kubernetes => github.com/docker/compose-on-kubernetes v0.4.24 github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible github.com/docker/docker => github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4 github.com/docker/docker-credential-helpers => github.com/docker/docker-credential-helpers v0.6.1 + github.com/docker/go => github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c github.com/docker/go-connections => github.com/docker/go-connections v0.4.0 + github.com/docker/go-events => github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c github.com/docker/go-metrics => github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 github.com/docker/go-units => github.com/docker/go-units v0.4.0 github.com/docker/libtrust => github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 github.com/docker/spdystream => github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c + github.com/docker/swarmkit => github.com/docker/swarmkit v1.12.0 github.com/docopt/docopt-go => github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 github.com/dustin/go-humanize => github.com/dustin/go-humanize v1.0.0 + github.com/dvsekhvalnov/jose2go => github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae github.com/eapache/go-resiliency => github.com/eapache/go-resiliency v1.1.0 github.com/eapache/go-xerial-snappy => github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 github.com/eapache/queue => github.com/eapache/queue v1.1.0 @@ -360,6 +373,7 @@ replace ( github.com/gomodule/redigo => github.com/gomodule/redigo v2.0.0+incompatible github.com/google/addlicense => github.com/google/addlicense v0.0.0-20200906110928-a0294312aa76 github.com/google/btree => github.com/google/btree v1.0.0 + github.com/google/certificate-transparency-go => github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 github.com/google/flatbuffers => github.com/google/flatbuffers v1.11.0 github.com/google/go-cmp => github.com/google/go-cmp v0.4.0 github.com/google/go-github => github.com/google/go-github v17.0.0+incompatible @@ -369,6 +383,7 @@ replace ( github.com/google/martian => github.com/google/martian v2.1.0+incompatible github.com/google/pprof => github.com/google/pprof v0.0.0-20200417002340-c6e0a841f49a github.com/google/renameio => github.com/google/renameio v0.1.0 + github.com/google/shlex => github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 github.com/google/uuid => github.com/google/uuid v1.1.1 github.com/googleapis/gax-go => github.com/googleapis/gax-go v2.0.2+incompatible github.com/googleapis/gax-go/v2 => github.com/googleapis/gax-go/v2 v2.0.5 @@ -437,6 +452,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v0.9.1 github.com/jsternberg/zap-logfmt => github.com/jsternberg/zap-logfmt v1.0.0 github.com/jtolds/gls => github.com/jtolds/gls v4.20.0+incompatible + github.com/juju/loggo => github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 github.com/julienschmidt/httprouter => github.com/julienschmidt/httprouter v1.3.0 github.com/jung-kurt/gofpdf => github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 github.com/jwilder/encoding => github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef @@ -488,6 +504,7 @@ replace ( github.com/mdlayher/wifi => github.com/mdlayher/wifi v0.0.0-20190303161829-b1436901ddee github.com/mgutz/ansi => github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b github.com/miekg/dns => github.com/miekg/dns v1.1.29 + github.com/miekg/pkcs11 => github.com/miekg/pkcs11 v1.0.2 github.com/minio/md5-simd => github.com/minio/md5-simd v1.1.0 github.com/minio/minio-go/v7 => github.com/minio/minio-go/v7 v7.0.2 github.com/minio/sha256-simd => github.com/minio/sha256-simd v0.1.1 @@ -517,6 +534,7 @@ replace ( github.com/nats-io/nkeys => github.com/nats-io/nkeys v0.1.3 github.com/nats-io/nuid => github.com/nats-io/nuid v1.0.1 github.com/ncw/swift => github.com/ncw/swift v1.0.50 + github.com/niemeyer/pretty => github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e github.com/nxadm/tail => github.com/nxadm/tail v1.4.4 github.com/oklog/oklog => github.com/oklog/oklog v0.3.2 github.com/oklog/run => github.com/oklog/run v1.1.0 @@ -529,6 +547,7 @@ replace ( github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.1 github.com/opencontainers/runc => github.com/opencontainers/runc v0.1.1 + github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.0.2 github.com/opentracing-contrib/go-grpc => github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing-contrib/go-observer => github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 github.com/opentracing-contrib/go-stdlib => github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9 @@ -614,7 +633,9 @@ replace ( github.com/streadway/handy => github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a github.com/stretchr/objx => github.com/stretchr/objx v0.2.0 github.com/stretchr/testify => github.com/stretchr/testify v1.4.0 + github.com/syndtr/gocapability => github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/thanos-io/thanos => github.com/thanos-io/thanos v0.13.1-0.20200910143741-e0b7f7b32e9c + github.com/theupdateframework/notary => github.com/theupdateframework/notary v0.7.0 github.com/tidwall/pretty => github.com/tidwall/pretty v1.0.0 github.com/tinylib/msgp => github.com/tinylib/msgp v1.1.0 github.com/tmc/grpc-websocket-proxy => github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 @@ -668,6 +689,7 @@ replace ( golang.org/x/oauth2 => golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys => golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e + golang.org/x/term => golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 golang.org/x/text => golang.org/x/text v0.3.0 golang.org/x/time => golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 golang.org/x/tools => golang.org/x/tools v0.0.0-20190710153321-831012c29e42 @@ -686,6 +708,7 @@ replace ( gopkg.in/alexcesaro/quotedprintable.v3 => gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc gopkg.in/asn1-ber.v1 => gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d gopkg.in/cas.v2 => gopkg.in/cas.v2 v2.2.0 + gopkg.in/cenkalti/backoff.v2 => gopkg.in/cenkalti/backoff.v2 v2.2.1 gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 gopkg.in/cheggaaa/pb.v1 => gopkg.in/cheggaaa/pb.v1 v1.0.25 gopkg.in/errgo.v2 => gopkg.in/errgo.v2 v2.1.0 @@ -701,6 +724,7 @@ replace ( gopkg.in/ini.v1 => gopkg.in/ini.v1 v1.57.0 gopkg.in/mail.v2 => gopkg.in/mail.v2 v2.3.1 gopkg.in/natefinch/lumberjack.v2 => gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/rethinkdb/rethinkdb-go.v6 => gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 gopkg.in/square/go-jose.v1 => gopkg.in/square/go-jose.v1 v1.1.2 gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.4.0 gopkg.in/src-d/go-billy.v4 => gopkg.in/src-d/go-billy.v4 v4.3.0 diff --git a/go.sum b/go.sum index 1cd294de9..fa0829362 100644 --- a/go.sum +++ b/go.sum @@ -130,11 +130,14 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/container-storage-interface/spec v1.2.0 h1:bD9KIVgaVKKkQ/UbVUY9kCaH/CJbhNxe0eeB4JeJV2s= github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/containerd/containerd v1.3.0 h1:xjvXQWABwS2uiv3TWgQt5Uth60Gu86LTGZXMJkjc7rY= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.17+incompatible h1:f/Z3EoDSx1yjaIjLQGo1diYUlQYSBrrAQ5vP8NjwXwo= github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -172,14 +175,17 @@ github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMa github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dhui/dktest v0.3.0/go.mod h1:cyzIUfGsBEbZ6BT7tnXqAShHSXCZhSNmFl70sZ7c1yc= github.com/disintegration/imaging v1.6.1/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ= +github.com/docker/cli v0.0.0-20190506213505-d88565df0c2d h1:qdD+BtyCE1XXpDyhvn0yZVcZOLILdj9Cw4pKu0kQbPQ= github.com/docker/cli v0.0.0-20190506213505-d88565df0c2d/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker-credential-helpers v0.6.1 h1:Dq4iIfcM7cNtddhLVWe9h4QDjsi4OER3Z8voPu/I52g= github.com/docker/docker-credential-helpers v0.6.1/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4 h1:+VAGRKyn9Ca+ckzV/PJsaRO7UXO9KQjFmSffcSDrWdE= github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA= github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -311,6 +317,7 @@ github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6/go.mod h1:K/9g3pPouf13 github.com/godbus/dbus v0.0.0-20190402143921-271e53dc4968/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gofrs/flock v0.7.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -386,6 +393,7 @@ github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR3 github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -554,6 +562,7 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -890,4 +899,5 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnM sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc h1:MksmcCZQWAQJCTA5T0jgI/0sJ51AVm4Z41MrmfczEoc= vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go index 2920eccd3..53810ab25 100644 --- a/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/iam/v1alpha2/zz_generated.deepcopy.go @@ -770,6 +770,11 @@ func (in *UserSpec) DeepCopy() *UserSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *UserStatus) DeepCopyInto(out *UserStatus) { *out = *in + if in.State != nil { + in, out := &in.State, &out.State + *out = new(UserState) + **out = **in + } if in.LastTransitionTime != nil { in, out := &in.LastTransitionTime, &out.LastTransitionTime *out = (*in).DeepCopy() diff --git a/pkg/apis/servicemesh/v1alpha2/strategy_types.go b/pkg/apis/servicemesh/v1alpha2/strategy_types.go index c86b459d9..bf5bd227e 100644 --- a/pkg/apis/servicemesh/v1alpha2/strategy_types.go +++ b/pkg/apis/servicemesh/v1alpha2/strategy_types.go @@ -31,17 +31,17 @@ const ( ResourcePluralStrategy = "strategies" ) -type StrategyType string +type strategyType string const ( // Canary strategy type - CanaryType StrategyType = "Canary" + CanaryType strategyType = "Canary" // BlueGreen strategy type - BlueGreenType StrategyType = "BlueGreen" + BlueGreenType strategyType = "BlueGreen" // Mirror strategy type - Mirror StrategyType = "Mirror" + Mirror strategyType = "Mirror" ) type StrategyPolicy string @@ -60,7 +60,7 @@ const ( // StrategySpec defines the desired state of Strategy type StrategySpec struct { // Strategy type - Type StrategyType `json:"type,omitempty"` + Type strategyType `json:"type,omitempty"` // Principal version, the one as reference version // label version value diff --git a/pkg/controller/application/application_controller.go b/pkg/controller/application/application_controller.go index ab8658132..9ef9bd8b7 100644 --- a/pkg/controller/application/application_controller.go +++ b/pkg/controller/application/application_controller.go @@ -18,45 +18,239 @@ package application import ( "context" + "fmt" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" v1beta12 "k8s.io/api/networking/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/tools/record" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/util/retry" "k8s.io/klog" servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/apis/servicemesh/v1alpha2" "kubesphere.io/kubesphere/pkg/controller/utils/servicemesh" - "sigs.k8s.io/application/api/v1beta1" + appv1beta1 "sigs.k8s.io/application/api/v1beta1" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" - "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" - "time" ) -// Add creates a new Application Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller -// and Start it when the Manager is Started. -func Add(mgr manager.Manager) error { - return add(mgr, newReconciler(mgr)) +// ApplicationReconciler reconciles a Application object +type ApplicationReconciler struct { + client.Client + Mapper meta.RESTMapper + Scheme *runtime.Scheme + ApplicationSelector labels.Selector // } -// newReconciler returns a new reconcile.Reconciler -func newReconciler(mgr manager.Manager) reconcile.Reconciler { - return &ReconcileApplication{Client: mgr.GetClient(), scheme: mgr.GetScheme(), - recorder: mgr.GetEventRecorderFor("application-controller")} +func (r *ApplicationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { + var app appv1beta1.Application + err := r.Get(context.Background(), req.NamespacedName, &app) + if err != nil { + if apierrors.IsNotFound(err) { + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + // If label selector were given, only reconcile matched applications + // match annotations and labels + if !r.ApplicationSelector.Empty() { + if !r.ApplicationSelector.Matches(labels.Set(app.Labels)) && + !r.ApplicationSelector.Matches(labels.Set(app.Annotations)) { + return ctrl.Result{}, err + } + } + + // Application is in the process of being deleted, so no need to do anything. + if app.DeletionTimestamp != nil { + return ctrl.Result{}, nil + } + + resources, errs := r.updateComponents(context.Background(), &app) + newApplicationStatus := r.getNewApplicationStatus(context.Background(), &app, resources, &errs) + + newApplicationStatus.ObservedGeneration = app.Generation + if equality.Semantic.DeepEqual(newApplicationStatus, &app.Status) { + return ctrl.Result{}, nil + } + + err = r.updateApplicationStatus(context.Background(), req.NamespacedName, newApplicationStatus) + return ctrl.Result{}, err } -// add adds a new Controller to mgr with r as the reconcile.Reconciler -func add(mgr manager.Manager, r reconcile.Reconciler) error { - // Create a new controller - c, err := controller.New("application-controller", mgr, controller.Options{Reconciler: r}) +func (r *ApplicationReconciler) updateComponents(ctx context.Context, app *appv1beta1.Application) ([]*unstructured.Unstructured, []error) { + var errs []error + resources := r.fetchComponentListResources(ctx, app.Spec.ComponentGroupKinds, app.Spec.Selector, app.Namespace, &errs) + + if app.Spec.AddOwnerRef { + ownerRef := metav1.NewControllerRef(app, appv1beta1.GroupVersion.WithKind("Application")) + *ownerRef.Controller = false + if err := r.setOwnerRefForResources(ctx, *ownerRef, resources); err != nil { + errs = append(errs, err) + } + } + return resources, errs +} + +func (r *ApplicationReconciler) getNewApplicationStatus(ctx context.Context, app *appv1beta1.Application, resources []*unstructured.Unstructured, errList *[]error) *appv1beta1.ApplicationStatus { + objectStatuses := r.objectStatuses(ctx, resources, errList) + errs := utilerrors.NewAggregate(*errList) + + aggReady, countReady := aggregateReady(objectStatuses) + + newApplicationStatus := app.Status.DeepCopy() + newApplicationStatus.ComponentList = appv1beta1.ComponentList{ + Objects: objectStatuses, + } + newApplicationStatus.ComponentsReady = fmt.Sprintf("%d/%d", countReady, len(objectStatuses)) + if errs != nil { + setReadyUnknownCondition(newApplicationStatus, "ComponentsReadyUnknown", "failed to aggregate all components' statuses, check the Error condition for details") + } else if aggReady { + setReadyCondition(newApplicationStatus, "ComponentsReady", "all components ready") + } else { + setNotReadyCondition(newApplicationStatus, "ComponentsNotReady", fmt.Sprintf("%d components not ready", len(objectStatuses)-countReady)) + } + + if errs != nil { + setErrorCondition(newApplicationStatus, "ErrorSeen", errs.Error()) + } else { + clearErrorCondition(newApplicationStatus) + } + + return newApplicationStatus +} + +func (r *ApplicationReconciler) fetchComponentListResources(ctx context.Context, groupKinds []metav1.GroupKind, selector *metav1.LabelSelector, namespace string, errs *[]error) []*unstructured.Unstructured { + var resources []*unstructured.Unstructured + + if selector == nil { + klog.V(2).Info("No selector is specified") + return resources + } + + for _, gk := range groupKinds { + mapping, err := r.Mapper.RESTMapping(schema.GroupKind{ + Group: appv1beta1.StripVersion(gk.Group), + Kind: gk.Kind, + }) + if err != nil { + klog.V(2).Info("NoMappingForGK", "gk", gk.String()) + continue + } + + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(mapping.GroupVersionKind) + if err = r.Client.List(ctx, list, client.InNamespace(namespace), client.MatchingLabels(selector.MatchLabels)); err != nil { + klog.Error(err, "unable to list resources for GVK", "gvk", mapping.GroupVersionKind) + *errs = append(*errs, err) + continue + } + + for _, u := range list.Items { + resource := u + resources = append(resources, &resource) + } + } + return resources +} + +func (r *ApplicationReconciler) setOwnerRefForResources(ctx context.Context, ownerRef metav1.OwnerReference, resources []*unstructured.Unstructured) error { + for _, resource := range resources { + ownerRefs := resource.GetOwnerReferences() + ownerRefFound := false + for i, refs := range ownerRefs { + if ownerRef.Kind == refs.Kind && + ownerRef.APIVersion == refs.APIVersion && + ownerRef.Name == refs.Name { + ownerRefFound = true + if ownerRef.UID != refs.UID { + ownerRefs[i] = ownerRef + } + } + } + + if !ownerRefFound { + ownerRefs = append(ownerRefs, ownerRef) + } + resource.SetOwnerReferences(ownerRefs) + err := r.Client.Update(ctx, resource) + if err != nil { + // We log this error, but we continue and try to set the ownerRefs on the other resources. + klog.Error(err, "ErrorSettingOwnerRef", "gvk", resource.GroupVersionKind().String(), + "namespace", resource.GetNamespace(), "name", resource.GetName()) + } + } + return nil +} + +func (r *ApplicationReconciler) objectStatuses(ctx context.Context, resources []*unstructured.Unstructured, errs *[]error) []appv1beta1.ObjectStatus { + var objectStatuses []appv1beta1.ObjectStatus + for _, resource := range resources { + os := appv1beta1.ObjectStatus{ + Group: resource.GroupVersionKind().Group, + Kind: resource.GetKind(), + Name: resource.GetName(), + Link: resource.GetSelfLink(), + } + s, err := status(resource) + if err != nil { + klog.Error(err, "unable to compute status for resource", "gvk", resource.GroupVersionKind().String(), + "namespace", resource.GetNamespace(), "name", resource.GetName()) + *errs = append(*errs, err) + } + os.Status = s + objectStatuses = append(objectStatuses, os) + } + return objectStatuses +} + +func aggregateReady(objectStatuses []appv1beta1.ObjectStatus) (bool, int) { + countReady := 0 + for _, os := range objectStatuses { + if os.Status == StatusReady { + countReady++ + } + } + if countReady == len(objectStatuses) { + return true, countReady + } + return false, countReady +} + +func (r *ApplicationReconciler) updateApplicationStatus(ctx context.Context, nn types.NamespacedName, status *appv1beta1.ApplicationStatus) error { + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + original := &appv1beta1.Application{} + if err := r.Get(ctx, nn, original); err != nil { + return err + } + original.Status = *status + if err := r.Client.Status().Update(ctx, original); err != nil { + return err + } + return nil + }); err != nil { + return fmt.Errorf("failed to update status of Application %s/%s: %v", nn.Namespace, nn.Name, err) + } + return nil +} + +func (r *ApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { + c, err := ctrl.NewControllerManagedBy(mgr). + Named("application-controller"). + For(&appv1beta1.Application{}).Build(r) + if err != nil { return err } @@ -98,46 +292,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler) error { return nil } -var _ reconcile.Reconciler = &ReconcileApplication{} - -// ReconcileApplication reconciles a Workspace object -type ReconcileApplication struct { - client.Client - scheme *runtime.Scheme - recorder record.EventRecorder -} - -// +kubebuilder:rbac:groups=app.k8s.io,resources=applications,verbs=get;list;watch;create;update;patch;delete -func (r *ReconcileApplication) Reconcile(request reconcile.Request) (reconcile.Result, error) { - // Fetch the Application instance - ctx := context.Background() - app := &v1beta1.Application{} - err := r.Get(ctx, request.NamespacedName, app) - if err != nil { - if errors.IsNotFound(err) { - klog.Errorf("application %s not found in namespace %s", request.Name, request.Namespace) - return reconcile.Result{}, nil - } - return reconcile.Result{}, err - } - - // add specified annotation for app when triggered by sub-resources, - // so the application in sigs.k8s.io can reconcile to update status - annotations := app.GetObjectMeta().GetAnnotations() - if annotations == nil { - annotations = make(map[string]string) - } - annotations["kubesphere.io/last-updated"] = time.Now().String() - app.SetAnnotations(annotations) - err = r.Update(ctx, app) - if err != nil { - if errors.IsNotFound(err) { - klog.V(4).Infof("application %s has been deleted during update in namespace %s", request.Name, request.Namespace) - return reconcile.Result{}, nil - } - } - return reconcile.Result{}, nil -} +var _ reconcile.Reconciler = &ApplicationReconciler{} func isApp(obs ...metav1.Object) bool { for _, o := range obs { diff --git a/pkg/controller/application/application_controller_test.go b/pkg/controller/application/application_controller_test.go index 723fc2e8c..e81d2755a 100644 --- a/pkg/controller/application/application_controller_test.go +++ b/pkg/controller/application/application_controller_test.go @@ -19,6 +19,10 @@ package application import ( "context" "fmt" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" v1 "k8s.io/api/apps/v1" @@ -27,71 +31,131 @@ import ( "k8s.io/apimachinery/pkg/types" "kubesphere.io/kubesphere/pkg/controller/utils/servicemesh" "sigs.k8s.io/application/api/v1beta1" - "time" +) + +const ( + applicationName = "bookinfo" + serviceName = "productpage" + timeout = time.Second * 30 + interval = time.Second * 2 ) var replicas = int32(2) -var _ = Describe("Application", func() { - - const timeout = time.Second * 30 - const interval = time.Second * 1 +var _ = Context("Inside of a new namespace", func() { ctx := context.TODO() + ns := SetupTest(ctx) - service := newService("productpage") - app := newAppliation(service) - deployments := []*v1.Deployment{newDeployments(service, "v1")} - - BeforeEach(func() { - - // Create application service and deployment - Expect(k8sClient.Create(ctx, app)).Should(Succeed()) - Expect(k8sClient.Create(ctx, service)).Should(Succeed()) - for i := range deployments { - deployment := deployments[i] - Expect(k8sClient.Create(ctx, deployment)).Should(Succeed()) + Describe("Application", func() { + applicationLabels := map[string]string{ + "app.kubernetes.io/name": "bookinfo", + "app.kubernetes.io/version": "1", } - }) - // Add Tests for OpenAPI validation (or additonal CRD features) specified in - // your API definition. - // Avoid adding tests for vanilla CRUD operations because they would - // test Kubernetes API server, which isn't the goal here. - Context("Application Controller", func() { - It("Should create successfully", func() { + BeforeEach(func() { + By("create deployment,service,application objects") + service := newService(serviceName, ns.Name, applicationLabels) + deployments := []*v1.Deployment{newDeployments(serviceName, ns.Name, applicationLabels, "v1")} + app := newApplication(applicationName, ns.Name, applicationLabels) - By("Reconcile Application successfully") - // application should have "kubesphere.io/last-updated" annotation - Eventually(func() bool { - app := &v1beta1.Application{} - _ = k8sClient.Get(ctx, types.NamespacedName{Name: service.Labels[servicemesh.ApplicationNameLabel], Namespace: metav1.NamespaceDefault}, app) - time, ok := app.Annotations["kubesphere.io/last-updated"] - return len(time) > 0 && ok - }, timeout, interval).Should(BeTrue()) + Expect(k8sClient.Create(ctx, service.DeepCopy())).Should(Succeed()) + for i := range deployments { + deployment := deployments[i] + Expect(k8sClient.Create(ctx, deployment.DeepCopy())).Should(Succeed()) + } + Expect(k8sClient.Create(ctx, app)).Should(Succeed()) + }) + + Context("Application Controller", func() { + It("Should not reconcile application", func() { + By("update application labels") + application := &v1beta1.Application{} + + err := k8sClient.Get(ctx, types.NamespacedName{Name: applicationName, Namespace: ns.Name}, application) + Expect(err).Should(Succeed()) + + updateApplication := func(object interface{}) { + newApp := object.(*v1beta1.Application) + newApp.Labels["kubesphere.io/creator"] = "" + } + + updated, err := updateWithRetries(k8sClient, ctx, application.Namespace, applicationName, updateApplication, 1 * time.Second, 5 * time.Second) + Expect(updated).Should(BeTrue()) + + Eventually(func() bool { + + err = k8sClient.Get(ctx, types.NamespacedName{Name: applicationName, Namespace: ns.Name}, application) + + // application status field should not be populated with selected deployments and services + return len(application.Status.ComponentList.Objects) == 0 + }, timeout, interval).Should(BeTrue()) + + }) + + It("Should reconcile application successfully", func() { + + By("check if application status been updated by controller") + application := &v1beta1.Application{} + + Eventually(func() bool { + err := k8sClient.Get(ctx, types.NamespacedName{Name: applicationName, Namespace: ns.Name}, application) + Expect(err).Should(Succeed()) + + // application status field should be populated by controller + return len(application.Status.ComponentList.Objects) > 0 + }, timeout, interval).Should(BeTrue()) + + }) }) }) }) -func newDeployments(service *corev1.Service, version string) *v1.Deployment { - lbs := service.Labels - lbs["version"] = version +type UpdateObjectFunc func(obj interface{}) + +func updateWithRetries(client client.Client, ctx context.Context, namespace, name string, updateFunc UpdateObjectFunc, interval, timeout time.Duration)(bool, error) { + var updateErr error + + pollErr := wait.PollImmediate(interval, timeout, func() (done bool, err error) { + app := &v1beta1.Application{} + if err = client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, app); err != nil { + return false, err + } + + updateFunc(app) + if err = client.Update(ctx, app); err == nil { + return true, nil + } + + updateErr = err + return false, nil + }) + + if pollErr == wait.ErrWaitTimeout { + pollErr = fmt.Errorf("couldn't apply the provided update to object %q: %v", name, updateErr) + return false, pollErr + } + return true, nil +} + +func newDeployments(deploymentName, namespace string, labels map[string]string, version string) *v1.Deployment { + labels["app"] = deploymentName + labels["version"] = version deployment := &v1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s", service.Name, version), - Namespace: metav1.NamespaceDefault, - Labels: lbs, + Name: fmt.Sprintf("%s-%s", deploymentName, version), + Namespace: namespace, + Labels: labels, Annotations: map[string]string{servicemesh.ServiceMeshEnabledAnnotation: "true"}, }, Spec: v1.DeploymentSpec{ Replicas: &replicas, Selector: &metav1.LabelSelector{ - MatchLabels: lbs, + MatchLabels: labels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: lbs, - Annotations: service.Annotations, + Labels: labels, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ @@ -130,16 +194,14 @@ func newDeployments(service *corev1.Service, version string) *v1.Deployment { return deployment } -func newService(name string) *corev1.Service { +func newService(serviceName, namesapce string, labels map[string]string) *corev1.Service { + labels["app"] = serviceName + svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: metav1.NamespaceDefault, - Labels: map[string]string{ - "app.kubernetes.io/name": "bookinfo", - "app.kubernetes.io/version": "1", - "app": name, - }, + Name: serviceName, + Namespace: namesapce, + Labels: labels, Annotations: map[string]string{ "servicemesh.kubesphere.io/enabled": "true", }, @@ -162,11 +224,7 @@ func newService(name string) *corev1.Service { Protocol: corev1.ProtocolTCP, }, }, - Selector: map[string]string{ - "app.kubernetes.io/name": "bookinfo", - "app.kubernetes.io/version": "1", - "app": "foo", - }, + Selector: labels, Type: corev1.ServiceTypeClusterIP, }, Status: corev1.ServiceStatus{}, @@ -174,12 +232,12 @@ func newService(name string) *corev1.Service { return svc } -func newAppliation(service *corev1.Service) *v1beta1.Application { +func newApplication(applicationName, namespace string, labels map[string]string) *v1beta1.Application { app := &v1beta1.Application{ ObjectMeta: metav1.ObjectMeta{ - Name: service.Labels[servicemesh.ApplicationNameLabel], - Namespace: metav1.NamespaceDefault, - Labels: service.Labels, + Name: applicationName, + Namespace: namespace, + Labels: labels, Annotations: map[string]string{servicemesh.ServiceMeshEnabledAnnotation: "true"}, }, Spec: v1beta1.ApplicationSpec{ @@ -193,6 +251,9 @@ func newAppliation(service *corev1.Service) *v1beta1.Application { Kind: "Deployment", }, }, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, AddOwnerRef: true, }, } diff --git a/pkg/controller/application/application_suit_test.go b/pkg/controller/application/application_suit_test.go index 4885d1a4f..40feef74f 100644 --- a/pkg/controller/application/application_suit_test.go +++ b/pkg/controller/application/application_suit_test.go @@ -17,12 +17,15 @@ limitations under the License. package application import ( - "github.com/onsi/gomega/gexec" + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/kubernetes/scheme" - "k8s.io/klog" + "k8s.io/client-go/rest" "k8s.io/klog/klogr" "kubesphere.io/kubesphere/pkg/apis" - "os" + "math/rand" "path/filepath" appv1beta1 "sigs.k8s.io/application/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" @@ -40,8 +43,8 @@ import ( // These tests use Ginkgo (BDD-style Go testing framework). Refer to // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. +var cfg *rest.Config var k8sClient client.Client -var k8sManager ctrl.Manager var testEnv *envtest.Environment func TestApplicationController(t *testing.T) { @@ -55,44 +58,23 @@ var _ = BeforeSuite(func(done Done) { logf.SetLogger(klogr.New()) By("bootstrapping test environment") - t := true - if os.Getenv("TEST_USE_EXISTING_CLUSTER") == "true" { - testEnv = &envtest.Environment{ - UseExistingCluster: &t, - } - } else { - testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, - AttachControlPlaneOutput: false, - } + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crds")}, + AttachControlPlaneOutput: false, } - sch := scheme.Scheme - err := appv1beta1.AddToScheme(sch) - Expect(err).NotTo(HaveOccurred()) - err = apis.AddToScheme(sch) - Expect(err).NotTo(HaveOccurred()) - - cfg, err := testEnv.Start() + var err error + cfg, err = testEnv.Start() Expect(err).ToNot(HaveOccurred()) Expect(cfg).ToNot(BeNil()) - k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{ - Scheme: sch, - MetricsBindAddress: "0", - }) + err = appv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + err = apis.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).ToNot(HaveOccurred()) - - err = Add(k8sManager) - Expect(err).ToNot(HaveOccurred()) - - go func() { - err = k8sManager.Start(ctrl.SetupSignalHandler()) - klog.Error(err) - Expect(err).ToNot(HaveOccurred()) - }() - - k8sClient = k8sManager.GetClient() Expect(k8sClient).ToNot(BeNil()) close(done) @@ -100,7 +82,69 @@ var _ = BeforeSuite(func(done Done) { var _ = AfterSuite(func() { By("tearing down the test environment") - gexec.KillAndWait(5 * time.Second) err := testEnv.Stop() Expect(err).ToNot(HaveOccurred()) }) + +// SetupTest will setup a testing environment. +// This includes: +// * creating a Namespace to be used during the test +// * starting application controller +// * stopping application controller after the test ends +// Call this function at the start of each of your tests. +func SetupTest(ctx context.Context) *corev1.Namespace { + var stopCh chan struct{} + ns := &corev1.Namespace{} + + BeforeEach(func() { + stopCh = make(chan struct{}) + *ns = corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: "testns-" + randStringRunes(5)}, + } + + err := k8sClient.Create(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to create a test namespace") + + mgr, err := ctrl.NewManager(cfg, ctrl.Options{}) + Expect(err).NotTo(HaveOccurred(), "failed to create a manager") + + selector, _ := labels.Parse("app.kubernetes.io/name,!kubesphere.io/creator") + + reconciler := &ApplicationReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Mapper: mgr.GetRESTMapper(), + ApplicationSelector: selector, + } + err = reconciler.SetupWithManager(mgr) + Expect(err).NotTo(HaveOccurred(), "failed to setup application reconciler") + + go func() { + err = mgr.Start(stopCh) + Expect(err).NotTo(HaveOccurred(), "failed to start manager") + }() + }) + + AfterEach(func() { + close(stopCh) + + err := k8sClient.Delete(ctx, ns) + Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace") + }) + + return ns +} + +func init() { + rand.Seed(time.Now().UnixNano()) +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") + +func randStringRunes(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[rand.Intn(len(letterRunes))] + } + return string(b) +} \ No newline at end of file diff --git a/vendor/sigs.k8s.io/application/controllers/condition.go b/pkg/controller/application/condition.go similarity index 99% rename from vendor/sigs.k8s.io/application/controllers/condition.go rename to pkg/controller/application/condition.go index 5abd50ac5..e590addca 100644 --- a/vendor/sigs.k8s.io/application/controllers/condition.go +++ b/pkg/controller/application/condition.go @@ -1,7 +1,7 @@ // Copyright 2020 The Kubernetes Authors. // SPDX-License-Identifier: Apache-2.0 -package controllers +package application import ( corev1 "k8s.io/api/core/v1" diff --git a/vendor/sigs.k8s.io/application/controllers/status.go b/pkg/controller/application/status.go similarity index 99% rename from vendor/sigs.k8s.io/application/controllers/status.go rename to pkg/controller/application/status.go index ae375cf91..709f95322 100644 --- a/vendor/sigs.k8s.io/application/controllers/status.go +++ b/pkg/controller/application/status.go @@ -1,7 +1,7 @@ // Copyright 2020 The Kubernetes Authors. // SPDX-License-Identifier: Apache-2.0 -package controllers +package application import ( "strings" diff --git a/vendor/kubesphere.io/client-go b/vendor/kubesphere.io/client-go deleted file mode 120000 index 0bee05eff..000000000 --- a/vendor/kubesphere.io/client-go +++ /dev/null @@ -1 +0,0 @@ -../../staging/src/kubesphere.io/client-go \ No newline at end of file diff --git a/vendor/modules.txt b/vendor/modules.txt index e6993045b..af18ba87e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -652,7 +652,7 @@ go.uber.org/zap/internal/bufferpool go.uber.org/zap/internal/color go.uber.org/zap/internal/exit go.uber.org/zap/zapcore -# golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de => golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 +# golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 => golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 golang.org/x/crypto/bcrypt golang.org/x/crypto/blowfish golang.org/x/crypto/cast5 @@ -701,7 +701,7 @@ golang.org/x/oauth2/internal # golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 => golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sync/errgroup golang.org/x/sync/singleflight -# golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 => golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e +# golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c => golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e golang.org/x/sys/cpu golang.org/x/sys/unix golang.org/x/sys/windows @@ -1555,7 +1555,6 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client # sigs.k8s.io/application v0.8.4-0.20201016185654-c8e2959e57a0 => sigs.k8s.io/application v0.8.4-0.20201016185654-c8e2959e57a0 sigs.k8s.io/application/api/v1beta1 -sigs.k8s.io/application/controllers # sigs.k8s.io/controller-runtime v0.6.4 => sigs.k8s.io/controller-runtime v0.6.4 sigs.k8s.io/controller-runtime sigs.k8s.io/controller-runtime/pkg/builder diff --git a/vendor/sigs.k8s.io/application/controllers/application_controller.go b/vendor/sigs.k8s.io/application/controllers/application_controller.go deleted file mode 100644 index a3350bc80..000000000 --- a/vendor/sigs.k8s.io/application/controllers/application_controller.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright 2020 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 - -package controllers - -import ( - "context" - "fmt" - - "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/api/equality" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - utilerrors "k8s.io/apimachinery/pkg/util/errors" - "k8s.io/client-go/util/retry" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - - appv1beta1 "sigs.k8s.io/application/api/v1beta1" -) - -const ( - loggerCtxKey = "logger" -) - -// ApplicationReconciler reconciles a Application object -type ApplicationReconciler struct { - client.Client - Mapper meta.RESTMapper - Log logr.Logger - Scheme *runtime.Scheme -} - -// +kubebuilder:rbac:groups=app.k8s.io,resources=applications,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups=app.k8s.io,resources=applications/status,verbs=get;update;patch -// +kubebuilder:rbac:groups=*,resources=*,verbs=list;get;update;patch;watch - -func (r *ApplicationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) { - rootCtx := context.Background() - logger := r.Log.WithValues("application", req.NamespacedName) - ctx := context.WithValue(rootCtx, loggerCtxKey, logger) - - var app appv1beta1.Application - err := r.Get(ctx, req.NamespacedName, &app) - if err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{}, nil - } - return ctrl.Result{}, err - } - - // Application is in the process of being deleted, so no need to do anything. - if app.DeletionTimestamp != nil { - return ctrl.Result{}, nil - } - - resources, errs := r.updateComponents(ctx, &app) - newApplicationStatus := r.getNewApplicationStatus(ctx, &app, resources, &errs) - - newApplicationStatus.ObservedGeneration = app.Generation - if equality.Semantic.DeepEqual(newApplicationStatus, &app.Status) { - return ctrl.Result{}, nil - } - - err = r.updateApplicationStatus(ctx, req.NamespacedName, newApplicationStatus) - return ctrl.Result{}, err -} - -func (r *ApplicationReconciler) updateComponents(ctx context.Context, app *appv1beta1.Application) ([]*unstructured.Unstructured, []error) { - var errs []error - resources := r.fetchComponentListResources(ctx, app.Spec.ComponentGroupKinds, app.Spec.Selector, app.Namespace, &errs) - - if app.Spec.AddOwnerRef { - ownerRef := metav1.NewControllerRef(app, appv1beta1.GroupVersion.WithKind("Application")) - *ownerRef.Controller = false - if err := r.setOwnerRefForResources(ctx, *ownerRef, resources); err != nil { - errs = append(errs, err) - } - } - return resources, errs -} - -func (r *ApplicationReconciler) getNewApplicationStatus(ctx context.Context, app *appv1beta1.Application, resources []*unstructured.Unstructured, errList *[]error) *appv1beta1.ApplicationStatus { - objectStatuses := r.objectStatuses(ctx, resources, errList) - errs := utilerrors.NewAggregate(*errList) - - aggReady, countReady := aggregateReady(objectStatuses) - - newApplicationStatus := app.Status.DeepCopy() - newApplicationStatus.ComponentList = appv1beta1.ComponentList{ - Objects: objectStatuses, - } - newApplicationStatus.ComponentsReady = fmt.Sprintf("%d/%d", countReady, len(objectStatuses)) - if errs != nil { - setReadyUnknownCondition(newApplicationStatus, "ComponentsReadyUnknown", "failed to aggregate all components' statuses, check the Error condition for details") - } else if aggReady { - setReadyCondition(newApplicationStatus, "ComponentsReady", "all components ready") - } else { - setNotReadyCondition(newApplicationStatus, "ComponentsNotReady", fmt.Sprintf("%d components not ready", len(objectStatuses)-countReady)) - } - - if errs != nil { - setErrorCondition(newApplicationStatus, "ErrorSeen", errs.Error()) - } else { - clearErrorCondition(newApplicationStatus) - } - - return newApplicationStatus -} - -func (r *ApplicationReconciler) fetchComponentListResources(ctx context.Context, groupKinds []metav1.GroupKind, selector *metav1.LabelSelector, namespace string, errs *[]error) []*unstructured.Unstructured { - logger := getLoggerOrDie(ctx) - var resources []*unstructured.Unstructured - - if selector == nil { - logger.Info("No selector is specified") - return resources - } - - for _, gk := range groupKinds { - mapping, err := r.Mapper.RESTMapping(schema.GroupKind{ - Group: appv1beta1.StripVersion(gk.Group), - Kind: gk.Kind, - }) - if err != nil { - logger.Info("NoMappingForGK", "gk", gk.String()) - continue - } - - list := &unstructured.UnstructuredList{} - list.SetGroupVersionKind(mapping.GroupVersionKind) - if err = r.Client.List(ctx, list, client.InNamespace(namespace), client.MatchingLabels(selector.MatchLabels)); err != nil { - logger.Error(err, "unable to list resources for GVK", "gvk", mapping.GroupVersionKind) - *errs = append(*errs, err) - continue - } - - for _, u := range list.Items { - resource := u - resources = append(resources, &resource) - } - } - return resources -} - -func (r *ApplicationReconciler) setOwnerRefForResources(ctx context.Context, ownerRef metav1.OwnerReference, resources []*unstructured.Unstructured) error { - logger := getLoggerOrDie(ctx) - for _, resource := range resources { - ownerRefs := resource.GetOwnerReferences() - ownerRefFound := false - for i, refs := range ownerRefs { - if ownerRef.Kind == refs.Kind && - ownerRef.APIVersion == refs.APIVersion && - ownerRef.Name == refs.Name { - ownerRefFound = true - if ownerRef.UID != refs.UID { - ownerRefs[i] = ownerRef - } - } - } - - if !ownerRefFound { - ownerRefs = append(ownerRefs, ownerRef) - } - resource.SetOwnerReferences(ownerRefs) - err := r.Client.Update(ctx, resource) - if err != nil { - // We log this error, but we continue and try to set the ownerRefs on the other resources. - logger.Error(err, "ErrorSettingOwnerRef", "gvk", resource.GroupVersionKind().String(), - "namespace", resource.GetNamespace(), "name", resource.GetName()) - } - } - return nil -} - -func (r *ApplicationReconciler) objectStatuses(ctx context.Context, resources []*unstructured.Unstructured, errs *[]error) []appv1beta1.ObjectStatus { - logger := getLoggerOrDie(ctx) - var objectStatuses []appv1beta1.ObjectStatus - for _, resource := range resources { - os := appv1beta1.ObjectStatus{ - Group: resource.GroupVersionKind().Group, - Kind: resource.GetKind(), - Name: resource.GetName(), - Link: resource.GetSelfLink(), - } - s, err := status(resource) - if err != nil { - logger.Error(err, "unable to compute status for resource", "gvk", resource.GroupVersionKind().String(), - "namespace", resource.GetNamespace(), "name", resource.GetName()) - *errs = append(*errs, err) - } - os.Status = s - objectStatuses = append(objectStatuses, os) - } - return objectStatuses -} - -func aggregateReady(objectStatuses []appv1beta1.ObjectStatus) (bool, int) { - countReady := 0 - for _, os := range objectStatuses { - if os.Status == StatusReady { - countReady++ - } - } - if countReady == len(objectStatuses) { - return true, countReady - } - return false, countReady -} - -func (r *ApplicationReconciler) updateApplicationStatus(ctx context.Context, nn types.NamespacedName, status *appv1beta1.ApplicationStatus) error { - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - original := &appv1beta1.Application{} - if err := r.Get(ctx, nn, original); err != nil { - return err - } - original.Status = *status - if err := r.Client.Status().Update(ctx, original); err != nil { - return err - } - return nil - }); err != nil { - return fmt.Errorf("failed to update status of Application %s/%s: %v", nn.Namespace, nn.Name, err) - } - return nil -} - -func (r *ApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - For(&appv1beta1.Application{}). - Complete(r) -} - -func getLoggerOrDie(ctx context.Context) logr.Logger { - logger, ok := ctx.Value(loggerCtxKey).(logr.Logger) - if !ok { - panic("context didn't contain logger") - } - return logger -}