Merge pull request #1920 from wansir/1919

fix: role binding change

.gitattributes

@@ -1,2 +1,6 @@
 pkg/cmd/api/spec/api.swagger.json linguist-generated=true
 pkg/cmd/api/spec/static.go linguist-generated=true
+pkg/client/* linguist-generated=true
+config/crds/* linguist-generated=true
+config/rbac/* linguist-generated=true
+zz_generated.deepcopy.go linguist-generated=true

.github/ISSUE_TEMPLATE/installation_failure.md

@@ -0,0 +1,16 @@
+---
+name: 安装问题
+about: Create a report to help us improve
+---
+
+[备注]: <> (请补全下面信息，帮助我们更快地定位问题。提交问题前预览下issue，看下是否有格式错误)
+
+**问题描述**
+
+**安装环境的硬件配置**
+
+[备注]: <> (请说明节点的运行环境，是否是物理机，云主机，VMware虚拟机)
+
+**错误信息或截图**
+
+**Installer版本**

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,38 @@
+**What type of PR is this?**
+> Uncomment only one ` /kind <>` line, hit enter to put that in a new line, and remove leading whitespaces from that line:
+>
+> /kind api-change
+> /kind bug
+> /kind cleanup
+> /kind design
+> /kind documentation
+> /kind failing-test
+> /kind feature
+> /kind flake
+
+**What this PR does / why we need it**:
+
+**Which issue(s) this PR fixes**:
+<!--
+Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
+_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
+-->
+Fixes #
+
+**Special notes for reviewers**:
+```
+```
+
+**Additional documentation, usage docs, etc.**:
+
+<!--
+This section can be blank if this pull request does not require a release note.
+Please use the following format for linking documentation or pass the
+section below:
+- [KEP]: <link>
+- [Usage]: <link>
+- [Other doc]: <link>
+-->
+```docs
+
+```

.gitignore

@@ -4,6 +4,7 @@
 *.dll
 *.so
 *.dylib
+*.a
 
 # Test binary, build with `go test -c`
 *.test
@@ -21,6 +22,11 @@ bin/
 
 tmp/
 
+apiserver.local.config
+
 # OSX trash
 .DS_Store
 api.json
+*.coverprofile
+
+kustomize/network/etcd

.travis.yml

@@ -3,27 +3,39 @@ services:
 
 language: go
 
+dist: xenial
+
 git:
   depth: false
 
 go:
-  - 1.12
-
-go_import_path: kubesphere.io/kubesphere
-
-before_install:
-  - go get -u github.com/golang/dep/cmd/dep
+  - "1.13.x"
+env:
+  - GO111MODULE=on
+cache:
+  directories:
+    - $HOME/gopath/pkg/mod
 
 before_script:
   - docker --version
   - bash hack/install_kubebuilder.sh
 
 script:
-  - make all 
+  - diff -u <(echo -n) <(gofmt -d ./pkg ./cmd ./tools)
+  - make openapi
+  - make all
+
+install:
+  - go get golang.org/x/lint/golint
 
 deploy:
-  skip_cleanup: true
-  provider: script
-  script: bash hack/docker_build.sh 
-  on:
-    branch: master
+  - skip_cleanup: true
+    provider: script
+    script: bash hack/docker_build.sh
+    on:
+      branch: master
+  - skip_cleanup: true
+    provider: script
+    script: bash hack/docker_build.sh
+    on:
+      branch: release-2.1

CONTRIBUTING.md

@@ -0,0 +1,221 @@
+# Development Guide
+
+This document walks you through how to get started developing KubeSphere and development workflow.
+
+## Preparing the environment
+
+### Go
+
+KubeSphere development is based on [Kubernetes](https://github.com/kubernetes/kubernetes), both of them are written in [Go](http://golang.org/). If you don't have a Go development environment, please [set one up](http://golang.org/doc/code.html).
+
+| Kubernetes     | requires Go |
+|----------------|-------------|
+| 1.13+          | >= 1.12     |
+
+> Tips:
+> - Ensure your GOPATH and PATH have been configured in accordance with the Go
+environment instructions.
+> - It's recommended to install [macOS GNU tools](https://www.topbug.net/blog/2013/04/14/install-and-use-gnu-command-line-tools-in-mac-os-x) for Mac OS.
+
+### Docker
+
+KubeSphere components are often deployed as containers in Kubernetes. If you need to rebuild the KubeSphere components in the Kubernetes cluster, you will need to [install Docker](https://docs.docker.com/install/).
+
+
+### Dependency management
+
+KubeSphere uses [Go Modules](https://github.com/golang/go/wiki/Modules) to manage dependencies in the `vendor/` tree.
+
+#### Dependencies
+
+[kubesphere/kubesphere](https://github.com/kubesphere/kubesphere) repository contains the source code . If you're looking for its dependent components, they live in their own repositories since they can be individual and universal.
+
+- [Alert](https://github.com/kubesphere/alert): Alert is an enterprise-grade general-purpose high-performance alerting system.
+- [Notification](https://github.com/openpitrix/notification): Notification is an enterprise-grade general-purpose high-performance notification system, it provides email notification service for KubeSphere currently.
+- [OpenPitrix](https://github.com/openpitrix/openpitrix): Application management platform on multi-cloud environment, it provides application template and application management for KubeSphere currently.
+- [SonarQube](https://github.com/SonarSource/sonarqube): Integrated in KubeSphere DevOps, it provides the capability to not only show health of an application but also to highlight issues newly introduced.
+
+## Building KubeSphere on a local OS/shell environment
+
+### For Quick Taste Binary
+
+```bash
+mkdir ks-tmp
+cd ks-tmp
+echo 'module kubesphere' > go.mod
+echo 'replace (
+        github.com/Sirupsen/logrus v1.4.1 => github.com/sirupsen/logrus v1.4.1
+      	github.com/kiali/kiali => github.com/kubesphere/kiali v0.15.1-0.20190407071308-6b5b818211c3
+      	github.com/kubernetes-sigs/application => github.com/kubesphere/application v0.0.0-20190518133311-b9d9eb0b5cf7
+      )' >> go.mod
+
+GO111MODULE=on go get kubesphere.io/kubesphere@d649e3d0bbc64bfba18816c904819e4850d021e0
+GO111MODULE=on go build -o ks-apiserver kubesphere.io/kubesphere/cmd/ks-apiserver # build ks-apiserver
+GO111MODULE=on go build -o ks-apigateway kubesphere.io/kubesphere/cmd/ks-apigateway # build ks-apigateway
+GO111MODULE=on go build -o ks-controller-manager kubesphere.io/kubesphere/cmd/controller-manager # build ks-controller-manager
+GO111MODULE=on go build -o ks-iam kubesphere.io/kubesphere/cmd/ks-iam # build ks-iam
+```
+
+### For Building KubeSphere Images
+
+KubeSphere components are often deployed as a container in a kubernetes cluster, you may need to build a Docker image locally.
+
+1. Clone repo to local
+
+```bash
+git clone https://github.com/kubesphere/kubesphere.git
+```
+
+2. Run Docker command to build image
+
+```bash
+# $REPO is the docker registry to push to
+# $Tag is the tag name of the docker image
+# The full go build process will be executed in the Dockerfile, so you may need to set GOPROXY in it.
+docker build -f build/ks-apigateway/Dockerfile -t $REPO/ks-apigateway:$TAG .
+docker build -f build/ks-apiserver/Dockerfile -t $REPO/ks-apiserver:$TAG .
+docker build -f build/ks-iam/Dockerfile -t $REPO/ks-account:$TAG .
+docker build -f build/ks-controller-manager/Dockerfile -t $REPO/ks-controller-manager:$TAG .
+docker build -f ./pkg/db/Dockerfile -t $REPO/ks-devops:flyway-$TAG ./pkg/db/
+```
+
+### Test
+
+In the development process, it is recommended to use local Kubernetes clusters, such as [minikube](https://kubernetes.io/docs/tasks/tools/install-minikube/), or to install an single-node [all-in-one](https://github.com/kubesphere/kubesphere#all-in-one) environment (Kubernetes-based) for quick testing.
+
+> Tip: It also supports to use Docker for Desktop ships with Kubernetes as the test environment.
+
+## Development Workflow
+
+![ks-workflow](docs/images/ks-workflow.png)
+
+### 1 Fork in the cloud
+
+1. Visit https://github.com/kubesphere/kubesphere
+2. Click `Fork` button to establish a cloud-based fork.
+
+### 2 Clone fork to local storage
+
+Per Go's [workspace instructions](https://golang.org/doc/code.html#Workspaces), place KubeSphere' code on your `GOPATH` using the following cloning procedure.
+
+1. Define a local working directory:
+
+```bash
+$ export working_dir=$GOPATH/src/kubesphere.io
+$ export user={your github profile name}
+```
+
+2. Create your clone locally:
+
+```bash
+$ mkdir -p $working_dir
+$ cd $working_dir
+$ git clone https://github.com/$user/kubesphere.git
+$ cd $working_dir/kubesphere
+$ git remote add upstream https://github.com/kubesphere/kubesphere.git
+
+# Never push to upstream master
+$ git remote set-url --push upstream no_push
+
+# Confirm that your remotes make sense:
+$ git remote -v
+```
+
+### 3 Keep your branch in sync
+
+```bash
+git fetch upstream
+git checkout master
+git rebase upstream/master
+```
+
+### 4 Add new features or fix issues
+
+Branch from it:
+
+```bash
+$ git checkout -b myfeature
+```
+
+Then edit code on the myfeature branch.
+
+**Test and build**
+
+Currently, make rules only contain simple checks such as vet, unit test, will add e2e tests soon.
+
+**Using KubeBuilder**
+
+- For Linux OS, you can download and execute this [KubeBuilder script](https://raw.githubusercontent.com/kubesphere/kubesphere/master/hack/install_kubebuilder.sh).
+
+- For MacOS, you can install KubeBuilder by following this [guide](https://book.kubebuilder.io/quick-start.html).
+
+**Run and test**
+
+```bash
+$ make all
+# Run every unit test
+$ make test
+```
+
+Run `make help` for additional information on these make targets.
+
+### 5 Development in new branch
+
+**Sync with upstream**
+
+After the test is completed, suggest you to keep your local in sync with upstream which can avoid conflicts.
+
+```
+# Rebase your the master branch of your local repo.
+$ git checkout master
+$ git rebase upstream/master
+
+# Then make your development branch in sync with master branch
+git checkout new_feature
+git rebase -i master
+```
+**Commit local changes**
+
+```bash
+$ git add <file>
+$ git commit -s -m "add your description"
+```
+
+### 6 Push to your folk
+
+When ready to review (or just to establish an offsite backup or your work), push your branch to your fork on github.com:
+
+```
+$ git push -f ${your_remote_name} myfeature
+```
+
+### 7 Create a PR
+
+- Visit your fork at https://github.com/$user/kubesphere
+- Click the` Compare & Pull Request` button next to your myfeature branch.
+- Check out the [pull request process](pull-request.md) for more details and advice.
+
+
+## CI/CD
+
+KubeSphere uses [Travis CI](https://travis-ci.org/) as a CI/CD tool.
+
+The components of KubeSphere need to be compiled and build include following:
+
+`ks-apiserver, ks-controller-manager, ks-account, ks-apigateway, ks-devops`
+
+After your PR is merged，Travis CI will compile the entire project and build the image, and push the image `kubespheredev/[component-name]:latest` to Dockerhub (e.g. `kubespheredev/ks-apiserver:latest`)
+
+## API Reference
+
+KubeSphere provides standard RESTFul API and detailed API documentations for developers, see [KubeSphere API Reference](https://docs.kubesphere.io/advanced-v2.0/zh-CN/api-reference/api-docs/) for more information.
+
+## Code conventions
+
+Please reference [Code conventions](https://github.com/kubernetes/community/blob/master/contributors/guide/coding-conventions.md) and follow with the rules.
+
+**Note:**
+
+> - All new packages and most new significant functionality must come with unit tests
+> - Comment your code in English, see [Go's commenting conventions
+](http://blog.golang.org/godoc-documenting-go-code)

Gopkg.toml

@@ -1,135 +0,0 @@
-required = [
-    "github.com/emicklei/go-restful",
-    "github.com/onsi/ginkgo", # for test framework
-    "github.com/onsi/gomega", # for test matchers
-    "k8s.io/gengo/examples/defaulter-gen/generators",
-    "k8s.io/gengo/examples/deepcopy-gen/generators",
-    "k8s.io/client-go/plugin/pkg/client/auth/gcp", # for development against gcp
-    "k8s.io/code-generator/cmd/client-gen",
-    "sigs.k8s.io/controller-tools/cmd/controller-gen", # for crd/rbac generation
-    "sigs.k8s.io/controller-runtime/pkg/client/config",
-    "sigs.k8s.io/controller-runtime/pkg/controller",
-    "sigs.k8s.io/controller-runtime/pkg/handler",
-    "sigs.k8s.io/controller-runtime/pkg/manager",
-    "sigs.k8s.io/controller-runtime/pkg/runtime/signals",
-    "sigs.k8s.io/controller-runtime/pkg/source",
-    "sigs.k8s.io/testing_frameworks/integration", # for integration testing
-    "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned",
-    "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions",
-    "github.com/kubesphere/s2ioperator/pkg/apis/devops/v1alpha1"
-]
-
-[[constraint]]
-  name = "github.com/docker/docker"
-  version = "v17.05.0-ce"
-
-[[override]]
-  name = "k8s.io/api"
-  version = "kubernetes-1.13.1"
-
-[[override]]
-  name = "k8s.io/apimachinery"
-  version = "kubernetes-1.13.1"
-
-[[override]]
-  name = "k8s.io/apiserver"
-  version = "kubernetes-1.13.1"
-
-[[constraint]]
-  name = "k8s.io/code-generator"
-  version = "kubernetes-1.13.1"
-
-[[override]]
-  name = "k8s.io/client-go"
-  version = "kubernetes-1.13.1"
-
-[[constraint]]
-  name = "k8s.io/kubernetes"
-  version = "1.13.1"
-
-[[constraint]]
-  name="sigs.k8s.io/controller-runtime"
-  version="v0.1.7"
-
-[[constraint]]
-  name="sigs.k8s.io/controller-tools"
-  version="v0.1.7"
-
-[[constraint]]
-  name="github.com/kubesphere/s2ioperator"
-  version="v0.0.8"
-
-[[override]]
-  name="github.com/bifurcation/mint"
-  revision="824af65410658916142a7600349144e1289f2110"
-
-[prune]
-  go-tests = true
-  unused-packages = true
-  non-go = true
-
-[[prune.project]]
-  name = "k8s.io/code-generator"
-  unused-packages = false
-  non-go = false
-
-
-# To use reference package:
-#   vendor/github.com/docker/docker/client/container_commit.go:17: undefined: reference.ParseNormalizedNamed
-#   vendor/github.com/docker/docker/client/container_commit.go:25: undefined: reference.TagNameOnly
-#   vendor/github.com/docker/docker/client/container_commit.go:30: undefined: reference.FamiliarNam
-[[override]]
-  name = "github.com/docker/distribution"
-  branch = "master"
-
-# To use reference package:
-#   vendor/github.com/docker/docker/registry/registry.go:30: cannot call non-function tlsconfig.ServerDefault (type tls.Config)
-#   vendor/github.com/docker/docker/registry/registry.go:66: undefined: tlsconfig.SystemCertPool
-#   vendor/github.com/docker/docker/registry/registry.go:168: cannot call non-function tlsconfig.ServerDefault (type tls.Config)
-#   vendor/github.com/docker/docker/registry/service_v2.go:11: cannot call non-function tlsconfig.ServerDefault (type tls.Config)
-[[override]]
-  name = "github.com/docker/go-connections"
-  version = "0.4.0"
-
-# For dependency below: Refer to issue https://github.com/golang/dep/issues/1799
-[[override]]
-  name = "gopkg.in/fsnotify.v1"
-  source = "https://github.com/fsnotify/fsnotify.git"
-  version = "v1.4.7"
-
-[[override]]
-  name = "github.com/russross/blackfriday"
-  version = "v1.5.2"
-
-# offical application controller doesn't limit observe scope to namespace
-# use our own version instead
-[[constraint]]
-  name = "sigs.k8s.io/application"
-  source = "https://github.com/kubesphere/application"
-  branch = "kubesphere"
-
-[[constraint]]
-  name = "github.com/kiali/kiali"
-  source = "https://github.com/kubesphere/kiali"
-  branch = "kubesphere"
-
-[[constraint]]
-  name = "github.com/kubernetes-sigs/application"
-  source = "https://github.com/kubesphere/application"
-  branch = "kubesphere"
-
-[[constraint]]
-  name = "github.com/knative/pkg"
-  revision = "cd278f2d3394c865fda66bca12459e879e0279b8"
-
-[[constraint]]
-  name = "gopkg.in/igm/sockjs-go.v2"
-  version = "2.0.0"
-
-[[constraint]]
-  name = "github.com/gocraft/dbr"
-  revision = "a0fd650918f6287ffe111d1c7b66bb755ff3be4a"
-
-[[constraint]]
-  name = "github.com/kubesphere/sonargo"
-  version = "0.0.2"

Makefile

@@ -5,9 +5,20 @@
 # The binary to build 
 BIN ?= ks-apiserver
 
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
+CRD_OPTIONS ?= "crd:trivialVersions=true"
+
+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+
 IMG ?= kubespheredev/ks-apiserver
 OUTPUT_DIR=bin
-
+GOFLAGS=-mod=vendor
 define ALL_HELP_INFO
 # Build code.
 #
@@ -28,7 +39,7 @@ define ALL_HELP_INFO
 #           debugging tools like delve.
 endef
 .PHONY: all
-all: test ks-apiserver ks-apigateway ks-iam controller-manager
+all: hypersphere ks-apiserver ks-apigateway ks-iam controller-manager
 
 # Build ks-apiserver binary
 ks-apiserver: test
@@ -46,42 +57,78 @@ ks-iam: test
 controller-manager: test
 	hack/gobuild.sh cmd/controller-manager
 
+# Build hypersphere binary
+hypersphere: test
+	hack/gobuild.sh cmd/hypersphere
+
 # Run go fmt against code 
-fmt:
-	go fmt ./pkg/... ./cmd/...
+fmt: generate
+	gofmt -w ./pkg ./cmd ./tools ./api
 
 # Run go vet against code
-vet:
+vet: generate
 	go vet ./pkg/... ./cmd/...
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests:
-	go run vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go all
+	go run ./vendor/sigs.k8s.io/controller-tools/cmd/controller-gen/main.go all
 
 deploy: manifests
 	kubectl apply -f config/crds
 	kustomize build config/default | kubectl apply -f -
 
-# Generate DeepCopy to implement runtime.Object
-deepcopy:
-	./vendor/k8s.io/code-generator/generate-groups.sh all kubesphere.io/kubesphere/pkg/client kubesphere.io/kubesphere/pkg/apis "servicemesh:v1alpha2 tenant:v1alpha1"
-
-# Generate code
+# generate will generate crds' deepcopy & go openapi structs
+# Futher more about go:genreate . https://blog.golang.org/generate
 generate:
-ifndef GOPATH
-	$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
-endif
 	go generate ./pkg/... ./cmd/...
 
+deepcopy:
+	GO111MODULE=on go install -mod=vendor k8s.io/code-generator/cmd/deepcopy-gen
+	${GOPATH}/bin/deepcopy-gen -i kubesphere.io/kubesphere/pkg/apis/... -h ./hack/boilerplate.go.txt -O zz_generated.deepcopy
+
+openapi:
+	go run ./vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go -O openapi_generated -i ./vendor/k8s.io/apimachinery/pkg/apis/meta/v1,./pkg/apis/tenant/v1alpha1 -p kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1 -h ./hack/boilerplate.go.txt --report-filename ./api/api-rules/violation_exceptions.list
+	go run ./vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go -O openapi_generated -i ./vendor/k8s.io/apimachinery/pkg/apis/meta/v1,./pkg/apis/servicemesh/v1alpha2 -p kubesphere.io/kubesphere/pkg/apis/servicemesh/v1alpha2 -h ./hack/boilerplate.go.txt --report-filename ./api/api-rules/violation_exceptions.list
+	go run ./vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go -O openapi_generated -i ./vendor/k8s.io/api/networking/v1,./vendor/k8s.io/apimachinery/pkg/apis/meta/v1,./pkg/apis/network/v1alpha1 -p kubesphere.io/kubesphere/pkg/apis/network/v1alpha1 -h ./hack/boilerplate.go.txt --report-filename ./api/api-rules/violation_exceptions.list
+	go run ./vendor/k8s.io/kube-openapi/cmd/openapi-gen/openapi-gen.go -O openapi_generated -i ./vendor/k8s.io/apimachinery/pkg/apis/meta/v1,./pkg/apis/devops/v1alpha1 -p kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1 -h ./hack/boilerplate.go.txt --report-filename ./api/api-rules/violation_exceptions.list
+	go run ./tools/cmd/crd-doc-gen/main.go
 # Build the docker image
 docker-build: all
 	docker build . -t ${IMG}
 
 # Run tests
-test: generate fmt vet
-	go test ./pkg/... ./cmd/... -coverprofile cover.out
+test: fmt vet
+	export KUBEBUILDER_CONTROLPLANE_START_TIMEOUT=1m; go test ./pkg/... ./cmd/... -coverprofile cover.out
 
 .PHONY: clean
 clean:
 	-make -C ./pkg/version clean
 	@echo "ok"
+
+# find or download controller-gen
+# download controller-gen if necessary
+clientset: 
+	./hack/generate_client.sh
+
+
+# Currently in the upgrade phase of controller tools.
+# But the new controller tools are not compatible with the old version.
+# With these commands you may need to manually modify the generated code
+# So don't use it unless you know it very deeply
+internal-crds:
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./pkg/apis/network/..." output:crd:artifacts:config=config/crd/bases
+
+internal-generate-apis: internal-controller-gen
+	$(CONTROLLER_GEN) object:headerFile=./hack/boilerplate.go.txt paths=./pkg/apis/network/...
+
+internal-controller-gen:
+ifeq (, $(shell which controller-gen))
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.0-beta.4
+CONTROLLER_GEN=$(GOBIN)/controller-gen
+else
+CONTROLLER_GEN=$(shell which controller-gen)
+endif
+
+network-rbac:
+	$(CONTROLLER_GEN) paths=./pkg/controller/network/provider/ paths=./pkg/controller/network/ rbac:roleName=network-manager output:rbac:artifacts:config=kustomize/network/calico-k8s
+	$(CONTROLLER_GEN) paths=./pkg/controller/network/ rbac:roleName=network-manager output:rbac:artifacts:config=kustomize/network/calico-etcd

OWNERS

@@ -0,0 +1,20 @@
+approvers:
+    - zryfish #oncall
+    - rayzhou2017
+
+reviewers:
+    - rayzhou2017
+    - zryfish
+    - benjaminhuo
+    - calvinyv
+    - FeynmanZhou
+    - huanggze
+    - huojiao2006
+    - Ma-Dan
+    - magicsong
+    - pixiake
+    - runzexia
+    - wansir
+    - wnxn
+    - zheng1
+    - soulseen

README.md

@@ -1,18 +1,50 @@
 # KubeSphere
 [![License](http://img.shields.io/badge/license-apache%20v2-blue.svg)](https://github.com/KubeSphere/KubeSphere/blob/master/LICENSE)
 [![Build Status](https://travis-ci.org/kubesphere/kubesphere.svg?branch=master)](https://travis-ci.org/kubesphere/kubesphere)
+[![KubeSphere release](https://img.shields.io/github/release/kubesphere/kubesphere.svg?color=release&label=release&logo=release&logoColor=release)](https://github.com/kubesphere/kubesphere/releases/tag/advanced-2.0.2)
+
+![logo](docs/images/kubesphere-logo.png)
 
 ----
 
 ## What is KubeSphere
 
-[KubeSphere](https://kubesphere.io/) is an enterprise-grade multi-tenant container management platform that built on [Kubernetes](https://kubernetes.io). It provides an easy-to-use UI enables creation of computing resources with a few clicks and one-click deployment, which reduces the learning curve and empower the DevOps teams. It greatly reduces the complexity of the daily work of development, testing, operation and maintenance, aiming to solve the pain spots of Kubernetes' storage, network, security and ease of use, etc.
+> English | [中文](README_zh.md)
 
-> See this [document](https://docs.kubesphere.io/advanced-v2.0/zh-CN/introduction/intro/) that describes the KubeSphere landscape and details.
+[KubeSphere](https://kubesphere.io/) is an enterprise-grade multi-tenant container management platform that built on [Kubernetes](https://kubernetes.io). It provides an easy-to-use UI for users to manage computing resources with a few clicks, which reduces the learning curve and empowers the DevOps teams. It greatly reduces the complexity of the daily work of development, testing, operation and maintenance, aiming to alleviate the pain points of Kubernetes' storage, network, security and ease of use, etc.
+
+
+## Screenshots
+
+> Note: See the [Screenshots](docs/screenshots.md) of KubeSphere to have a most intuitive understanding of KubeSphere dashboard and features.
+
+
+<table>
+  <tr>
+      <td width="50%" align="center"><b>KubeSphere Dashboard</b></td>
+      <td width="50%" align="center"><b>Project Resources</b></td>
+  </tr>
+  <tr>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112094014.png"/></td>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112094426.png"/></td>
+  </tr>
+  <tr>
+      <td width="50%" align="center"><b>CI/CD Pipeline</b></td>
+      <td width="50%" align="center"><b>Application Store</b></td>
+  </tr>
+  <tr>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20190925000712.png"/></td>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112095006.png"/></td>
+  </tr>
+</table>
+
+## Video on Youtube
+
+[![KubeSphere](https://pek3b.qingstor.com/kubesphere-docs/png/20191112093503.png)](https://youtu.be/u5lQvhi_Xlc)
 
 ## Features
 
-KubeSphere Advanced Edition 2.0.0 provides an easy-to-use console with the awesome user experience that allows you to quickly get started with a container management platform. KubeSphere provides and supports following core features:
+KubeSphere provides an easy-to-use console with the awesome user experience that allows you to quickly get started with a container management platform. KubeSphere provides and supports following core features:
 
 
 - Workload management
@@ -31,35 +63,41 @@ KubeSphere Advanced Edition 2.0.0 provides an easy-to-use console with the aweso
 
 It also supports multiple open source storage and high-performance cloud storage as the persistent storage services, as well as supports multiple open source network plugins.
 
-> See this [document](https://docs.kubesphere.io/advanced-v2.0/zh-CN/introduction/features/) that elaborates on the KubeSphere features and services from a professional point of view.
+> Note: See this [document](https://docs.kubesphere.io/advanced-v2.0/zh-CN/introduction/features/) that elaborates on the KubeSphere features and services from a professional point of view.
 
 ----
 
+## Architecture
+
+KubeSphere adopts the separation of front and back ends, each component is drawn in the architecture diagram below. KubeSphere can run anywhere from on-premise datacenter to any cloud to edge. In addition, it can be deployed on any Kubernetes distribution.
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190810073322.png)
+
 ## Latest Release
 
-KubeSphere Advanced Edition 2.0.0 was released on **May 18th, 2019**. See the [Release Notes For 2.0.0](https://docs.kubesphere.io/advanced-v2.0/release/release-v200/) to preview the updates.
+KubeSphere 2.1.0 was released on **November 12nd, 2019**. See the [Release Notes For 2.1.0](https://kubesphere.io/docs/v2.1/zh-CN/release/release-v210/) to preview the updates.
 
 ## Installation
 
-KubeSphere installation supports following 2 kinds of installation, please reference the following guides on how to get KubeSphere up and running.
+> Attention: Following section is only used for minimal installation by default, KubeSphere has decoupled some core components in v2.1.0, for more pluggable components installation, see `Enable Pluggable Components` below.
 
-### All-in-One
+### Deploy On Kubernetes
 
-[All-in-One](https://docs.kubesphere.io/advanced-v2.0/zh-CN/installation/all-in-one/): For those who are new to KubeSphere and looking for the fastest way to install and experience the dashboard. 
+**Prerequisites**
 
-Just download the installer and execute the `install.sh` under `/scripts` folder, choose `"1) All-in-one"` to trigger the installation. Generally, you can install it directly without any modification, for details please reference [All-in-One](https://docs.kubesphere.io/advanced-v2.0/zh-CN/installation/all-in-one/).
+> - `Kubernetes version`： `1.13.0 ≤ K8s version < 1.16`;
+> - `Helm version` >= `2.10.0`，see [Install and Configure Helm in Kubernetes](https://devopscube.com/install-configure-helm-kubernetes/);
+> - CPU > 1 Core，Memory > 2 G;
+> - An existing Storage Class in your Kubernetes clusters, use `kubectl get sc` to verify it.
 
-```bash
-$ curl -L https://kubesphere.io/download/stable/advanced-2.0.0 > advanced-2.0.0.tar.gz
-$ tar -zxf advanced-2.0.0.tar.gz
+When all Pods of KubeSphere are running, it means the installation is successsful. Then you can use `http://IP:30880` to access the dashboard with default account `admin/P@88w0rd`.
+
+```yaml
+$ kubectl apply -f https://raw.githubusercontent.com/kubesphere/ks-installer/master/kubesphere-minimal.yaml
 ```
 
-### Multi-Node
 
-[Multi-Node](https://docs.kubesphere.io/advanced-v2.0/zh-CN/installation/multi-node/) is used for installing KubeSphere on multiple instances, supports for installing a highly available cluster which is able to use in a formal environment.
-
-
-### Minimum Requirements
+### Deploy on Linux
 
 - Operating Systems
    - CentOS 7.5 (64 bit)
@@ -67,29 +105,73 @@ $ tar -zxf advanced-2.0.0.tar.gz
    - Red Hat Enterprise Linux Server 7.4 (64 bit)
    - Debian Stretch 9.5 (64 bit)
 - Hardware
-   - CPU：8 Core,  Memory：16 G, Disk Space：100 G
+   - CPU：2 Core,  Memory：4 G, Disk Space：100 G
 
-## Quick Start
+### All-in-One
 
-The [Quick Start Guide](https://docs.kubesphere.io/advanced-v2.0/quick-start/admin-quick-start/) provides 12 quick-start examples to walk you through the process and common manipulation in KubeSphere, with a quick overview of the core features of KubeSphere that helps you to get familiar with it.
+For those who are new to KubeSphere and looking for the fastest way to install and experience the dashboard. Execute following commands to download and install KubeSphere in a single node.
 
+```bash
+$ curl -L https://kubesphere.io/download/stable/v2.1.0 > installer.tar.gz \
+&& tar -zxf installer.tar.gz && cd kubesphere-all-v2.1.0/scripts
+$ ./install.sh
+```
+
+Choose `"1) All-in-one"` to trigger the installation. Generally, you can install it directly without any configuration..
+
+> Note: In a formal environment, it's highly recommended to install KubeSphere with Multi-Node Installation.
+
+### Enable Pluggable Components
+
+The above two methods is only used for minimal installation by default, execute following command to enable more pluggable components installation, make sure your cluster has enough CPU and memory in advance.
+
+```
+$ kubectl edit cm -n kubesphere-system ks-installer
+```
+
+## To start using KubeSphere
+
+### Quick Start
+
+KubeSphere provides 12 quick-start tutorials to walk you through the process and common manipulation, with a quick overview of the core features of KubeSphere that helps you to get familiar with it.
+
+- [Get Started - En](https://github.com/kubesphere/kubesphere.github.io/tree/master/blog/advanced-2.0/en)
+- [Get Started - 中](https://kubesphere.io/docs/advanced-v2.0/zh-CN/quick-start/quick-start-guide/)
+
+
+### Documentation
+
+- [KubeSphere Documentation (En/中) ](https://kubesphere.io/docs)
+- [API Documentation](https://kubesphere.io/docs/advanced-v2.0/zh-CN/api-reference/api-docs/)
+
+
+## To start developing KubeSphere
+
+The [development guide](CONTRIBUTING.md) hosts all information about building KubeSphere from source, git workflow, how to contribute code and how to test.
 
 ## RoadMap
 
-Currently, KubeSphere has released the following 4 major editions. Advanced Edition 2.0.0 was released on May 18, 2019. The future releases will include Big data, AI, Multicluster, QingCloud SDN, etc.
+Currently, KubeSphere has released the following 4 major editions. The future releases will include Multicluster, Big data, AI, SDN, etc.
 
-**Community Edition** => **Express Edition** => **Advanced Edition 1.0.0** => **Advanced Edition 2.0.0**
+**Express Edition** => **v1.0.x** => **v2.0.x**  => **v2.1.0**
 
-![Roadmap](docs/images/roadmap-en.png)
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190926000413.png)
 
-## Documentation
+## Landscapes
+
+<p align="center">
+<br/><br/>
+<img src="https://landscape.cncf.io/images/left-logo.svg" width="150"/>&nbsp;&nbsp;<img src="https://landscape.cncf.io/images/right-logo.svg" width="200"/>&nbsp;&nbsp;<img src="https://www.cncf.io/wp-content/uploads/2017/11/certified_kubernetes_color.png" height="40" width="30"/>
+<br/><br/>
+KubeSphere is a member of CNCF and a <a href="https://www.cncf.io/certification/software-conformance/#logos">Kubernetes Conformance Certified platform
+</a>, which enriches the <a href="https://landscape.cncf.io/landscape=observability-and-analysis&license=apache-license-2-0">CNCF CLOUD NATIVE Landscape.
+</a>
+</p>
 
-- [KubeSphere Documentation (En/中) ](https://docs.kubesphere.io/)
-- [KubeSphere Docementation (PDF)](https://docs.kubesphere.io/KubeSphere-advanced-v2.0.pdf)
 
 ## Support, Discussion, and Community
 
-If you need any help with KubeSphere, please join us at [Slack channel](http://kubesphere.slack.com/) where most of our team hangs out at.
+If you need any help with KubeSphere, please join us at [Slack Channel](https://join.slack.com/t/kubesphere/shared_invite/enQtNTE3MDIxNzUxNzQ0LTZkNTdkYWNiYTVkMTM5ZThhODY1MjAyZmVlYWEwZmQ3ODQ1NmM1MGVkNWEzZTRhNzk0MzM5MmY4NDc3ZWVhMjE).
 
 Please submit any KubeSphere bugs, issues, and feature requests to [KubeSphere GitHub Issue](https://github.com/kubesphere/kubesphere/issues).
 
@@ -98,7 +180,3 @@ Please submit any KubeSphere bugs, issues, and feature requests to [KubeSphere G
 All members of the KubeSphere community must abide by [Code of Conduct](docs/code-of-conduct.md). Only by respecting each other can we develop a productive, collaborative community.
 
 How to submit a pull request to KubeSphere? See [Pull Request Instruction](docs/pull-requests.md).
-
-You can then find out more detail [here](docs/welcome-to-KubeSphere-new-developer-guide.md).
-
-

README_zh.md

@@ -0,0 +1,172 @@
+# KubeSphere
+[![License](http://img.shields.io/badge/license-apache%20v2-blue.svg)](https://github.com/KubeSphere/KubeSphere/blob/master/LICENSE)
+[![Build Status](https://travis-ci.org/kubesphere/kubesphere.svg?branch=master)](https://travis-ci.org/kubesphere/kubesphere)
+[![KubeSphere release](https://img.shields.io/github/release/kubesphere/kubesphere.svg?color=release&label=release&logo=release&logoColor=release)](https://github.com/kubesphere/kubesphere/releases/tag/advanced-2.0.2)
+
+![logo](docs/images/kubesphere-logo.png)
+
+----
+
+## KubeSphere 是什么
+
+> [English](README.md) | 中文
+
+[KubeSphere](https://kubesphere.io/) 是在 [Kubernetes](https://kubernetes.io) 之上构建的以**应用为中心的**多租户**容器管理平台**，支持部署和运行在**任何基础设施之上**，提供**简单易用的操作界面**以及**向导式操作**方式，在降低用户使用容器调度平台学习成本的同时，极大减轻开发、测试、运维的日常工作的复杂度，旨在解决 Kubernetes 本身存在的存储、网络、安全和易用性等痛点。帮助企业轻松应对**敏捷开发、自动化运维、应用快速交付、微服务治理、多租户管理、监控日志告警、服务与网络管理、镜像仓库**等业务场景。
+
+KubeSphere 已大规模服务于社区用户，广泛地应用在以容器为中心的开发测试及生产环境，大量服务平稳地运行在 KubeSphere 之上。
+
+> 说明：KubeSphere 目前最新的版本为高级版 2.0.2，并且所有版本 100% 开源，关于 KubeSphere 更详细的介绍与说明请参阅 [产品介绍](https://docs.kubesphere.io/advanced-v2.0/zh-CN/introduction/intro/)。
+
+
+点击 [KubeSphere 快览](docs/screenshots.md) 快速查看 KubeSphere UI；
+
+<table>
+  <tr>
+      <td width="50%" align="center"><b>KubeSphere Dashboard</b></td>
+      <td width="50%" align="center"><b>Project Resources</b></td>
+  </tr>
+  <tr>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112094014.png"/></td>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112094426.png"/></td>
+  </tr>
+  <tr>
+      <td width="50%" align="center"><b>CI/CD Pipeline</b></td>
+      <td width="50%" align="center"><b>Application Store</b></td>
+  </tr>
+  <tr>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20190925000712.png"/></td>
+     <td><img src="https://pek3b.qingstor.com/kubesphere-docs/png/20191112095006.png"/></td>
+  </tr>
+</table>
+
+## Video on Youtube
+
+[![KubeSphere](https://pek3b.qingstor.com/kubesphere-docs/png/20191112093503.png)](https://youtu.be/u5lQvhi_Xlc)
+
+## 核心功能
+
+- Kubernetes 资源管理：纳管多种类型的 K8s 资源，提供易用友好的向导式 UI
+- 应用编排与管理：包括**一键部署应用**、**Helm Chart 可视化管理**、**应用生命周期管理**，后续将支持计量计费
+- 微服务治理：基于 Istio 提供可视化无代码侵入的**灰度发布、熔断、流量管控、Tracing**，兼容**Spring Cloud & Dubbo**
+- 一站式 DevOps：提供**可视化编辑 CI/CD 流水线**，包括从开发测试到持续部署上线的**全流程管理**，提供 [S2i](https://kubesphere.io/docs/v2.0/zh-CN/quick-start/source-to-image/)、[B2i](https://kubesphere.io/docs/v2.1/zh-CN/quick-start/b2i-war/)
+- 多租户管理：提供基于角色的细粒度 [多租户统一认证](https://kubesphere.io/docs/v2.1/zh-CN/multi-tenant/intro/)，支持**对接企业 LDAP/AD**，提供多层级的权限管理满足多组织架构的企业用户
+- 日志查询与收集：提供基于多租户和多维度的 [日志查询](https://kubesphere.io/docs/v2.1/zh-CN/toolbox/log-search/)，并支持快速对接多种日志收集平台
+- 多维度监控：提供集群与应用级别多项监控指标，提供按节点、企业空间、项目等资源用量的排行
+- 多租户告警系统：支持基于多租户、多维度的告警，提供丰富的监控告警指标，可自定义告警策略，支持邮件通知
+- 基础设施管理：提供主机管理、存储类型管理、CPU 与内存等资源配额管理
+- 支持多种网络方案：支持 Calico、Flannel，提供面向物理部署 Kubernetes 的 LB 插件 [Porter](https://github.com/kubesphere/porter) 和云上[负载均衡器插件](https://github.com/yunify/qingcloud-cloud-controller-manager)
+- 支持多种存储：支持 GlusterFS、CephRBD、NFS，支持 [企业级分布式存储 NeonSAN](https://www.qingcloud.com/products/qingstor-neonsan/) 和 [QingCloud 云平台块存储](https://docs.qingcloud.com/product/storage/volume/)
+- 支持 GPU 节点
+
+
+> 更多详细的功能解读与说明，请查阅 [产品功能](https://kubesphere.io/docs/v2.1/zh-CN/introduction/features/)。
+
+----
+
+## 架构
+
+KubeSphere 采用了前后端分离的架构设计，后端的各个功能组件可通过 REST API 对接外部系统，KubeSphere 可以运行在任何 Kubernetes、私有云、公有云、VM 或物理环境之上。
+
+![](docs/images/architecture-zh.png)
+
+## 最新发布
+
+KubeSphere 2.1.0 已于 2019 年 11 月 12 日 正式发布，点击 [Release Notes For 2.1.0](https://kubesphere.io/docs/v2.1/zh-CN/release/release-v210/) 查看 2.1.0 版本的更新详情。
+
+## 快速安装
+
+### 部署在 Linux
+
+- 操作系统
+   - CentOS 7.5 (64 bit)
+   - Ubuntu 16.04/18.04 LTS (64 bit)
+   - Red Hat Enterprise Linux Server 7.4 (64 bit)
+   - Debian Stretch 9.5 (64 bit)
+- 配置规格（最低）
+   - CPU：2 Core， 内存：4 G， 硬盘：100 G
+
+#### All-in-One
+
+[All-in-One](https://kubesphere.io/docs/v2.1/zh-CN/installation/all-in-one/): 对于首次接触 KubeSphere 高级版的用户，想寻找一个最快安装和体验 KubeSphere 高级版核心功能的方式，All-in-one 模式支持一键安装 KubeSphere 至一台目标机器，建议使用干净的机器安装。
+
+```bash
+$ curl -L https://kubesphere.io/download/stable/v2.1.0 > installer.tar.gz \
+&& tar -zxf installer.tar.gz && cd kubesphere-all-v2.1.0/scripts
+$ ./install.sh
+```
+
+直接选择 `"1) All-in-one"` 即可开始快速安装。
+
+> 注意：All-in-One 仅适用于**测试体验**，**正式环境** 安装和使用请参考 [安装说明](https://kubesphere.io/docs/v2.1/zh-CN/installation/intro/#%E6%AD%A3%E5%BC%8F%E7%8E%AF%E5%A2%83%E5%AE%89%E8%A3%85)。
+
+### 部署在 Kubernetes
+
+**前提条件**
+
+> - `Kubernetes` 版本： `1.13.0 ≤ K8s version < 1.16`；
+> - `Helm`，版本 `>= 2.10.0`，且已安装了 Tiller，参考 [如何安装与配置 Helm](https://devopscube.com/install-configure-helm-kubernetes/)；
+> - 集群的可用 CPU > 1 C，可用内存 > 2 G；且集群能够访问外网
+> - 集群已有存储类型（StorageClass）；
+
+可参考 [前提条件](https://kubesphere.io/docs/v2.1/zh-CN/installation/prerequisites/) 验证，若待安装的环境满足以上条件则可以开始部署 KubeSphere，当 KubeSphere 的所有 Pod 都为 RRunning 则说明安装成功。使用 `http://IP:30880` 访问 Dashboard，默认账号为 `admin/P@88w0rd`。
+
+```yaml
+$ kubectl apply -f https://raw.githubusercontent.com/kubesphere/ks-installer/master/kubesphere-minimal.yaml
+```
+
+注意，以上两种安装方式默认是 **最小化安装**，由于 2.1.0 已对大部分功能组件解耦，实现了功能组件的可插拔，因此可在 **安装完成后** 执行以下命令开启可插拔功能组件的安装，开启安装前确认您的机器资源已符合 [资源最低要求](https://kubesphere.io/docs/v2.1/zh-CN/installation/intro/#%E5%8F%AF%E6%8F%92%E6%8B%94%E5%8A%9F%E8%83%BD%E7%BB%84%E4%BB%B6%E5%88%97%E8%A1%A8)。
+
+```
+$ kubectl edit cm -n kubesphere-system ks-installer
+```
+
+## 开始使用 KubeSphere
+
+### 快速入门
+
+[KubeSphere 快速入门](https://kubesphere.io/docs/v2.1/zh-CN/quick-start/quick-start-guide/) 通过 14 个 Step-by-Step 的快速入门的示例教程帮助您了解 KubeSphere 容器平台的基本使用流程，带您快速上手 KubeSphere。
+
+### 文档
+
+- [KubeSphere 文档中心 (En/中) ](https://kubesphere.io/docs/)
+- [API 文档](https://kubesphere.io/docs/advanced-v2.0/zh-CN/api-reference/api-docs/)
+
+
+## 开发 KubeSphere
+
+[开发指南](CONTRIBUTING.md) 详细说明了如何从源码编译、KubeSphere 的 GitHub 工作流、如何贡献代码以及如何测试等。
+
+
+## 路线图
+
+目前，KubeSphere 已发布了 4 个大版本和 3 个小版本，所有版本都是完全开源的，为 KubeSphere 社区用户提供服务。
+
+**Express Edition** => **v1.0.x** => **v2.0.x**  => **v2.1.0**
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190926000514.png)
+
+## Landscapes
+
+<p align="center">
+<br/><br/>
+<img src="https://landscape.cncf.io/images/left-logo.svg" width="150"/>&nbsp;&nbsp;<img src="https://landscape.cncf.io/images/right-logo.svg" width="200"/>&nbsp;&nbsp;<img src="https://www.cncf.io/wp-content/uploads/2017/11/certified_kubernetes_color.png" height="40" width="30"/>
+<br/><br/>
+KubeSphere 是 CNCF 基金会成员并且通过了 <a href="https://www.cncf.io/certification/software-conformance/#logos">Kubernetes 一致性认证
+</a>，进一步丰富了 <a href="https://landscape.cncf.io/landscape=observability-and-analysis&license=apache-license-2-0">CNCF 云原生的生态。
+</a>
+</p>
+
+## 技术社区
+
+- [Slack Channel](https://join.slack.com/t/kubesphere/shared_invite/enQtNTE3MDIxNzUxNzQ0LTZkNTdkYWNiYTVkMTM5ZThhODY1MjAyZmVlYWEwZmQ3ODQ1NmM1MGVkNWEzZTRhNzk0MzM5MmY4NDc3ZWVhMjE)
+
+- [技术论坛](https://kubesphere.io/forum/)
+
+- 微信群 (与工程师和用户们交流云原生技术，请备注 “公司 - 名字”）
+
+<img width="150px" src="https://pek3b.qingstor.com/kubesphere-docs/png/20190902002055.png" />
+
+
+## Bug 与建议反馈
+
+KubeSphere 的日益完善与快速发展离不开社区用户的支持，KubeSphere 也一直在反哺社区，为开源用户提供更多的支持。若您安装使用时有任何建议问题、反馈或发现的 Bug，欢迎在 [GitHub Issue](https://github.com/kubesphere/kubesphere/issues) 提交 Issue。

api/api-rules/violation_exceptions.list

@@ -0,0 +1,8 @@
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,APIResourceList,APIResources
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Duration,Duration
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Object
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,InternalEvent,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,MicroTime,Time
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,StatusCause,Type
+API rule violation: names_match,k8s.io/apimachinery/pkg/apis/meta/v1,Time,Time
+API rule violation: names_match,kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1,S2iBinarySpec,MD5

build/hypersphere/Dockerfile

@@ -0,0 +1,18 @@
+# Copyright 2018 The KubeSphere Authors. All rights reserved.
+# Use of this source code is governed by a Apache license
+# that can be found in the LICENSE file.
+
+# Copyright 2018 The KubeSphere Authors. All rights reserved.
+# Use of this source code is governed by a Apache license
+# that can be found in the LICENSE file.
+FROM golang:1.12 as hypersphere-builder
+
+COPY / /go/src/kubesphere.io/kubesphere
+
+WORKDIR /go/src/kubesphere.io/kubesphere
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o hypersphere cmd/hypersphere/hypersphere.go
+
+FROM alpine:3.9
+RUN apk add --update ca-certificates && update-ca-certificates
+COPY --from=hypersphere-builder /go/src/kubesphere.io/kubesphere/hypersphere /usr/local/bin/
+CMD ["sh"]

build/ks-apigateway/Dockerfile

@@ -10,11 +10,11 @@ FROM golang:1.12 as ks-apigateway-builder
 
 COPY / /go/src/kubesphere.io/kubesphere
 WORKDIR /go/src/kubesphere.io/kubesphere
-RUN CGO_ENABLED=0 GO111MODULE=off GOOS=linux GOARCH=amd64 go build -i -ldflags '-w -s' -o ks-apigateway cmd/ks-apigateway/apiserver.go && \
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o ks-apigateway cmd/ks-apigateway/apiserver.go && \
     go run tools/cmd/doc-gen/main.go --output=install/swagger-ui/api.json
 
 FROM alpine:3.9
 RUN apk add --update ca-certificates && update-ca-certificates
 COPY --from=ks-apigateway-builder /go/src/kubesphere.io/kubesphere/ks-apigateway /usr/local/bin/
 COPY --from=ks-apigateway-builder /go/src/kubesphere.io/kubesphere/install/swagger-ui /var/static/swagger-ui
-CMD ["sh"]
+CMD ["sh"]

build/ks-apiserver/Dockerfile

@@ -10,7 +10,7 @@ FROM golang:1.12 as ks-apiserver-builder
 COPY / /go/src/kubesphere.io/kubesphere
 
 WORKDIR /go/src/kubesphere.io/kubesphere
-RUN CGO_ENABLED=0 GO111MODULE=off GOOS=linux GOARCH=amd64 go build -i -ldflags '-w -s' -o ks-apiserver cmd/ks-apiserver/apiserver.go
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o ks-apiserver cmd/ks-apiserver/apiserver.go
 
 FROM alpine:3.9
 RUN apk add --update ca-certificates && update-ca-certificates

build/ks-controller-manager/Dockerfile

@@ -10,7 +10,7 @@ FROM golang:1.12 as controller-manager-builder
 COPY / /go/src/kubesphere.io/kubesphere
 WORKDIR /go/src/kubesphere.io/kubesphere
 
-RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build --ldflags "-extldflags -static" -o controller-manager ./cmd/controller-manager/
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build --ldflags "-extldflags -static" -o controller-manager ./cmd/controller-manager/
 
 FROM alpine:3.7
 RUN apk add --update ca-certificates && update-ca-certificates

build/ks-iam/Dockerfile

@@ -10,7 +10,7 @@ FROM golang:1.12 as ks-iam-builder
 COPY / /go/src/kubesphere.io/kubesphere
 
 WORKDIR /go/src/kubesphere.io/kubesphere
-RUN CGO_ENABLED=0 GO111MODULE=off GOOS=linux GOARCH=amd64 go build -i -ldflags '-w -s' -o ks-iam cmd/ks-iam/apiserver.go
+RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o ks-iam cmd/ks-iam/apiserver.go
 
 FROM alpine:3.9
 RUN apk add --update ca-certificates && update-ca-certificates

build/ks-network/Dockerfile

@@ -0,0 +1,4 @@
+FROM gcr.io/distroless/static:latest
+WORKDIR /
+COPY ks-network .
+ENTRYPOINT ["/ks-network"]

cmd/controller-manager/app/controllers.go

@@ -24,6 +24,9 @@ import (
 	"kubesphere.io/kubesphere/pkg/controller/application"
 	"kubesphere.io/kubesphere/pkg/controller/destinationrule"
 	"kubesphere.io/kubesphere/pkg/controller/job"
+	"kubesphere.io/kubesphere/pkg/controller/s2ibinary"
+	"kubesphere.io/kubesphere/pkg/controller/s2irun"
+	"kubesphere.io/kubesphere/pkg/controller/storage/expansion"
 
 	//"kubesphere.io/kubesphere/pkg/controller/job"
 	"kubesphere.io/kubesphere/pkg/controller/virtualservice"
@@ -34,10 +37,12 @@ import (
 
 	istioclientset "github.com/knative/pkg/client/clientset/versioned"
 	istioinformers "github.com/knative/pkg/client/informers/externalversions"
-	applicationclientset "github.com/kubernetes-sigs/application/pkg/client/clientset/versioned"
-	applicationinformers "github.com/kubernetes-sigs/application/pkg/client/informers/externalversions"
-	servicemeshclientset "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
-	servicemeshinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	s2iclientset "github.com/kubesphere/s2ioperator/pkg/client/clientset/versioned"
+	s2iinformers "github.com/kubesphere/s2ioperator/pkg/client/informers/externalversions"
+	kubesphereclientset "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
+	kubesphereinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions"
+	applicationclientset "sigs.k8s.io/application/pkg/client/clientset/versioned"
+	applicationinformers "sigs.k8s.io/application/pkg/client/informers/externalversions"
 )
 
 const defaultResync = 600 * time.Second
@@ -62,56 +67,82 @@ func AddControllers(mgr manager.Manager, cfg *rest.Config, stopCh <-chan struct{
 		log.Error(err, "create application client failed")
 		return err
 	}
+	s2iclient, err := s2iclientset.NewForConfig(cfg)
+	if err != nil {
+		log.Error(err, "create s2i client failed")
+		return err
+	}
+	kubesphereclient, err := kubesphereclientset.NewForConfig(cfg)
+	if err != nil {
+		log.Error(err, "create kubesphere client failed")
+		return err
+	}
 
 	informerFactory := informers.NewSharedInformerFactory(kubeClient, defaultResync)
 	istioInformer := istioinformers.NewSharedInformerFactory(istioclient, defaultResync)
 	applicationInformer := applicationinformers.NewSharedInformerFactory(applicationClient, defaultResync)
+	s2iInformer := s2iinformers.NewSharedInformerFactory(s2iclient, defaultResync)
 
-	servicemeshclient, err := servicemeshclientset.NewForConfig(cfg)
-	if err != nil {
-		log.Error(err, "create servicemesh client failed")
-		return err
-	}
-
-	servicemeshInformer := servicemeshinformers.NewSharedInformerFactory(servicemeshclient, defaultResync)
+	kubesphereInformer := kubesphereinformers.NewSharedInformerFactory(kubesphereclient, defaultResync)
 
 	vsController := virtualservice.NewVirtualServiceController(informerFactory.Core().V1().Services(),
 		istioInformer.Networking().V1alpha3().VirtualServices(),
 		istioInformer.Networking().V1alpha3().DestinationRules(),
-		servicemeshInformer.Servicemesh().V1alpha2().Strategies(),
+		kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
 		kubeClient,
 		istioclient,
-		servicemeshclient)
+		kubesphereclient)
 
 	drController := destinationrule.NewDestinationRuleController(informerFactory.Apps().V1().Deployments(),
 		istioInformer.Networking().V1alpha3().DestinationRules(),
 		informerFactory.Core().V1().Services(),
-		servicemeshInformer.Servicemesh().V1alpha2().ServicePolicies(),
+		kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
 		kubeClient,
 		istioclient,
-		servicemeshclient)
+		kubesphereclient)
 
 	apController := application.NewApplicationController(informerFactory.Core().V1().Services(),
 		informerFactory.Apps().V1().Deployments(),
 		informerFactory.Apps().V1().StatefulSets(),
-		servicemeshInformer.Servicemesh().V1alpha2().Strategies(),
-		servicemeshInformer.Servicemesh().V1alpha2().ServicePolicies(),
+		kubesphereInformer.Servicemesh().V1alpha2().Strategies(),
+		kubesphereInformer.Servicemesh().V1alpha2().ServicePolicies(),
 		applicationInformer.App().V1beta1().Applications(),
 		kubeClient,
 		applicationClient)
 
 	jobController := job.NewJobController(informerFactory.Batch().V1().Jobs(), kubeClient)
 
-	servicemeshInformer.Start(stopCh)
+	s2iBinaryController := s2ibinary.NewController(kubesphereclient,
+		kubeClient,
+		kubesphereInformer.Devops().V1alpha1().S2iBinaries())
+
+	s2iRunController := s2irun.NewController(kubesphereclient, s2iclient, kubeClient,
+		kubesphereInformer.Devops().V1alpha1().S2iBinaries(),
+		s2iInformer.Devops().V1alpha1().S2iRuns())
+
+	volumeExpansionController := expansion.NewVolumeExpansionController(
+		kubeClient,
+		informerFactory.Core().V1().PersistentVolumeClaims(),
+		informerFactory.Storage().V1().StorageClasses(),
+		informerFactory.Core().V1().Pods(),
+		informerFactory.Apps().V1().Deployments(),
+		informerFactory.Apps().V1().ReplicaSets(),
+		informerFactory.Apps().V1().StatefulSets())
+
+	kubesphereInformer.Start(stopCh)
 	istioInformer.Start(stopCh)
 	informerFactory.Start(stopCh)
 	applicationInformer.Start(stopCh)
+	s2iInformer.Start(stopCh)
 
 	controllers := map[string]manager.Runnable{
 		"virtualservice-controller":  vsController,
 		"destinationrule-controller": drController,
 		"application-controller":     apController,
 		"job-controller":             jobController,
+		"s2ibinary-controller":       s2iBinaryController,
+		"s2irun-controller":          s2iRunController,
+		"volumeexpansion-controller": volumeExpansionController,
 	}
 
 	for name, ctrl := range controllers {

cmd/controller-manager/app/options/options.go

@@ -0,0 +1,82 @@
+package options
+
+import (
+	"flag"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
+	kubesphereconfig "kubesphere.io/kubesphere/pkg/server/config"
+	"kubesphere.io/kubesphere/pkg/simple/client/devops"
+	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
+	"kubesphere.io/kubesphere/pkg/simple/client/s2is3"
+	"strings"
+	"time"
+)
+
+type KubeSphereControllerManagerOptions struct {
+	KubernetesOptions *k8s.KubernetesOptions
+	DevopsOptions     *devops.DevopsOptions
+	S3Options         *s2is3.S3Options
+	OpenPitrixOptions *openpitrix.OpenPitrixOptions
+
+	LeaderElection *apiserverconfig.LeaderElectionConfiguration
+}
+
+func NewKubeSphereControllerManagerOptions() *KubeSphereControllerManagerOptions {
+	s := &KubeSphereControllerManagerOptions{
+		KubernetesOptions: k8s.NewKubernetesOptions(),
+		DevopsOptions:     devops.NewDevopsOptions(),
+		S3Options:         s2is3.NewS3Options(),
+		OpenPitrixOptions: openpitrix.NewOpenPitrixOptions(),
+		LeaderElection: &apiserverconfig.LeaderElectionConfiguration{
+			LeaderElect:   false,
+			LeaseDuration: v1.Duration{Duration: 30 * time.Second},
+			RenewDeadline: v1.Duration{Duration: 15 * time.Second},
+			RetryPeriod:   v1.Duration{Duration: 5 * time.Second},
+			ResourceLock:  "ks-controller-manager-leader-election",
+		},
+	}
+
+	return s
+}
+
+func (s *KubeSphereControllerManagerOptions) ApplyTo(conf *kubesphereconfig.Config) {
+	s.S3Options.ApplyTo(conf.S3Options)
+	s.KubernetesOptions.ApplyTo(conf.KubernetesOptions)
+	s.DevopsOptions.ApplyTo(conf.DevopsOptions)
+	s.OpenPitrixOptions.ApplyTo(conf.OpenPitrixOptions)
+}
+
+func (s *KubeSphereControllerManagerOptions) Flags() cliflag.NamedFlagSets {
+	fss := cliflag.NamedFlagSets{}
+
+	s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"))
+	s.DevopsOptions.AddFlags(fss.FlagSet("devops"))
+	s.S3Options.AddFlags(fss.FlagSet("s3"))
+	s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"))
+
+	fs := fss.FlagSet("leaderelection")
+	leaderelectionconfig.BindFlags(s.LeaderElection, fs)
+
+	kfs := fss.FlagSet("klog")
+	local := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(local)
+	local.VisitAll(func(fl *flag.Flag) {
+		fl.Name = strings.Replace(fl.Name, "_", "-", -1)
+		kfs.AddGoFlag(fl)
+	})
+
+	return fss
+}
+
+func (s *KubeSphereControllerManagerOptions) Validate() []error {
+	var errs []error
+	errs = append(errs, s.DevopsOptions.Validate()...)
+	errs = append(errs, s.KubernetesOptions.Validate()...)
+	errs = append(errs, s.S3Options.Validate()...)
+	errs = append(errs, s.OpenPitrixOptions.Validate()...)
+	return errs
+}

cmd/controller-manager/app/server.go

@@ -0,0 +1,213 @@
+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package app
+
+import (
+	"context"
+	"fmt"
+	"github.com/spf13/cobra"
+	v1 "k8s.io/api/core/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/tools/record"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/cmd/controller-manager/app/options"
+	"kubesphere.io/kubesphere/pkg/apis"
+	"kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme"
+	"kubesphere.io/kubesphere/pkg/controller"
+	controllerconfig "kubesphere.io/kubesphere/pkg/server/config"
+	"kubesphere.io/kubesphere/pkg/simple/client"
+	"kubesphere.io/kubesphere/pkg/utils/term"
+	"os"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+)
+
+func NewControllerManagerCommand() *cobra.Command {
+	s := options.NewKubeSphereControllerManagerOptions()
+
+	cmd := &cobra.Command{
+		Use:  "controller-manager",
+		Long: `KubeSphere controller manager is a daemon that`,
+		Run: func(cmd *cobra.Command, args []string) {
+
+			err := controllerconfig.Load()
+			if err != nil {
+				klog.Fatal(err)
+				os.Exit(1)
+			}
+
+			s = Complete(s)
+
+			if errs := s.Validate(); len(errs) != 0 {
+				klog.Error(utilerrors.NewAggregate(errs))
+				os.Exit(1)
+			}
+
+			if err = Run(s, signals.SetupSignalHandler()); err != nil {
+				os.Exit(1)
+			}
+		},
+	}
+
+	fs := cmd.Flags()
+	namedFlagSets := s.Flags()
+
+	for _, f := range namedFlagSets.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
+	usageFmt := "Usage:\n  %s\n"
+	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
+	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
+		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
+		cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
+	})
+	return cmd
+}
+
+func Complete(s *options.KubeSphereControllerManagerOptions) *options.KubeSphereControllerManagerOptions {
+	conf := controllerconfig.Get()
+
+	conf.Apply(&controllerconfig.Config{
+		DevopsOptions:     s.DevopsOptions,
+		KubernetesOptions: s.KubernetesOptions,
+		S3Options:         s.S3Options,
+		OpenPitrixOptions: s.OpenPitrixOptions,
+	})
+
+	out := &options.KubeSphereControllerManagerOptions{
+		KubernetesOptions: conf.KubernetesOptions,
+		DevopsOptions:     conf.DevopsOptions,
+		S3Options:         conf.S3Options,
+		OpenPitrixOptions: conf.OpenPitrixOptions,
+		LeaderElection:    s.LeaderElection,
+	}
+
+	return out
+}
+
+func CreateClientSet(conf *controllerconfig.Config, stopCh <-chan struct{}) error {
+	csop := &client.ClientSetOptions{}
+
+	csop.SetKubernetesOptions(conf.KubernetesOptions).
+		SetDevopsOptions(conf.DevopsOptions).
+		SetS3Options(conf.S3Options).
+		SetOpenPitrixOptions(conf.OpenPitrixOptions).
+		SetKubeSphereOptions(conf.KubeSphereOptions)
+	client.NewClientSetFactory(csop, stopCh)
+
+	return nil
+}
+
+func Run(s *options.KubeSphereControllerManagerOptions, stopCh <-chan struct{}) error {
+	err := CreateClientSet(controllerconfig.Get(), stopCh)
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+
+	config := client.ClientSets().K8s().Config()
+
+	run := func(ctx context.Context) {
+		klog.V(0).Info("setting up manager")
+		mgr, err := manager.New(config, manager.Options{})
+		if err != nil {
+			klog.Fatalf("unable to set up overall controller manager: %v", err)
+		}
+
+		klog.V(0).Info("setting up scheme")
+		if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
+			klog.Fatalf("unable add APIs to scheme: %v", err)
+		}
+
+		klog.V(0).Info("Setting up controllers")
+		if err := controller.AddToManager(mgr); err != nil {
+			klog.Fatalf("unable to register controllers to the manager: %v", err)
+		}
+
+		if err := AddControllers(mgr, config, stopCh); err != nil {
+			klog.Fatalf("unable to register controllers to the manager: %v", err)
+		}
+
+		klog.V(0).Info("Starting the Cmd.")
+		if err := mgr.Start(stopCh); err != nil {
+			klog.Fatalf("unable to run the manager: %v", err)
+		}
+
+		select {}
+	}
+
+	if !s.LeaderElection.LeaderElect {
+		run(context.TODO())
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go func() {
+		<-stopCh
+		cancel()
+	}()
+
+	id, err := os.Hostname()
+	if err != nil {
+		return err
+	}
+
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id = id + "_" + string(uuid.NewUUID())
+
+	// TODO: change lockType to lease
+	// once we finished moving to Kubernetes v1.16+, we
+	// change lockType to lease
+	lock, err := resourcelock.New("endpoints",
+		"kubesphere-system",
+		s.LeaderElection.ResourceLock,
+		client.ClientSets().K8s().Kubernetes().CoreV1(),
+		resourcelock.ResourceLockConfig{
+			Identity: id,
+			EventRecorder: record.NewBroadcaster().NewRecorder(scheme.Scheme, v1.EventSource{
+				Component: "ks-controller-manager",
+			}),
+		})
+
+	if err != nil {
+		klog.Fatalf("error creating lock: %v", err)
+	}
+
+	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Lock:          lock,
+		LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
+		RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
+		RetryPeriod:   s.LeaderElection.RetryPeriod.Duration,
+		Callbacks: leaderelection.LeaderCallbacks{
+			OnStartedLeading: run,
+			OnStoppedLeading: func() {
+				klog.Errorf("leadership lost")
+				os.Exit(0)
+			},
+		},
+	})
+
+	return nil
+}

cmd/controller-manager/controller-manager.go

@@ -1,89 +1,14 @@
-/*
-
- Copyright 2019 The KubeSphere Authors.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-*/
-
 package main
 
 import (
-	"flag"
-	"k8s.io/client-go/tools/clientcmd"
 	"kubesphere.io/kubesphere/cmd/controller-manager/app"
-	"kubesphere.io/kubesphere/pkg/apis"
-	"kubesphere.io/kubesphere/pkg/controller"
 	"os"
-	"sigs.k8s.io/controller-runtime/pkg/manager"
-	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
-	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
-var (
-	masterURL   string
-	kubeconfig  string
-	metricsAddr string
-)
-
-func init() {
-	flag.StringVar(&masterURL, "master-url", "", "only need if out of cluster")
-	flag.StringVar(&kubeconfig, "kubeconfig", "", "only need if out of cluster")
-	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
-}
-
 func main() {
-	flag.Parse()
+	command := app.NewControllerManagerCommand()
 
-	logf.SetLogger(logf.ZapLogger(false))
-	log := logf.Log.WithName("controller-manager")
-
-	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
-	if err != nil {
-		log.Error(err, "failed to build kubeconfig")
+	if err := command.Execute(); err != nil {
 		os.Exit(1)
 	}
-
-	stopCh := signals.SetupSignalHandler()
-
-	log.Info("setting up manager")
-	mgr, err := manager.New(cfg, manager.Options{})
-	if err != nil {
-		log.Error(err, "unable to set up overall controller manager")
-		os.Exit(1)
-	}
-
-	log.Info("setting up scheme")
-	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
-		log.Error(err, "unable add APIs to scheme")
-		os.Exit(1)
-	}
-
-	log.Info("Setting up controllers")
-	if err := controller.AddToManager(mgr); err != nil {
-		log.Error(err, "unable to register controllers to the manager")
-		os.Exit(1)
-	}
-
-	if err := app.AddControllers(mgr, cfg, stopCh); err != nil {
-		log.Error(err, "unable to register controllers to the manager")
-		os.Exit(1)
-	}
-
-	log.Info("Starting the Cmd.")
-	if err := mgr.Start(stopCh); err != nil {
-		log.Error(err, "unable to run the manager")
-		os.Exit(1)
-	}
-
 }

cmd/hypersphere/hypersphere.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	goflag "flag"
+	cliflag "k8s.io/component-base/cli/flag"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	controllermanager "kubesphere.io/kubesphere/cmd/controller-manager/app"
+	ksapigateway "kubesphere.io/kubesphere/cmd/ks-apigateway/app"
+	ksapiserver "kubesphere.io/kubesphere/cmd/ks-apiserver/app"
+	ksaiam "kubesphere.io/kubesphere/cmd/ks-iam/app"
+	"os"
+)
+
+func main() {
+	hypersphereCommand, allCommandFns := NewHyperSphereCommand()
+
+	pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+
+	basename := filepath.Base(os.Args[0])
+	if err := commandFor(basename, hypersphereCommand, allCommandFns).Execute(); err != nil {
+		os.Exit(1)
+	}
+}
+
+func commandFor(basename string, defaultCommand *cobra.Command, commands []func() *cobra.Command) *cobra.Command {
+	for _, commandFn := range commands {
+		command := commandFn()
+		if command.Name() == basename {
+			return command
+		}
+
+		for _, alias := range command.Aliases {
+			if alias == basename {
+				return command
+			}
+		}
+	}
+
+	return defaultCommand
+}
+
+func NewHyperSphereCommand() (*cobra.Command, []func() *cobra.Command) {
+	apiserver := func() *cobra.Command { return ksapiserver.NewAPIServerCommand() }
+	controllermanager := func() *cobra.Command { return controllermanager.NewControllerManagerCommand() }
+	iam := func() *cobra.Command { return ksaiam.NewAPIServerCommand() }
+	apigateway := func() *cobra.Command { return ksapigateway.NewAPIGatewayCommand() }
+
+	commandFns := []func() *cobra.Command{
+		apiserver,
+		controllermanager,
+		iam,
+		apigateway,
+	}
+
+	cmd := &cobra.Command{
+		Use:   "hypersphere",
+		Short: "Request a new project",
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) != 0 {
+				cmd.Help()
+				os.Exit(0)
+			}
+		},
+	}
+
+	for i := range commandFns {
+		cmd.AddCommand(commandFns[i]())
+	}
+
+	return cmd, commandFns
+}

cmd/ks-apigateway/apiserver.go

@@ -18,18 +18,15 @@
 package main
 
 import (
-	"github.com/mholt/caddy/caddy/caddymain"
-	"github.com/mholt/caddy/caddyhttp/httpserver"
-
-	// Install apis
-	_ "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authenticate"
-	_ "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authentication"
-	_ "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/swagger"
+	"kubesphere.io/kubesphere/cmd/ks-apigateway/app"
+	"os"
 )
 
 func main() {
-	httpserver.RegisterDevDirective("authenticate", "jwt")
-	httpserver.RegisterDevDirective("authentication", "jwt")
-	httpserver.RegisterDevDirective("swagger", "jwt")
-	caddymain.Run()
+
+	cmd := app.NewAPIGatewayCommand()
+
+	if err := cmd.Execute(); err != nil {
+		os.Exit(1)
+	}
 }

cmd/ks-apigateway/app/server.go

@@ -0,0 +1,53 @@
+package app
+
+import (
+	"flag"
+	"github.com/mholt/caddy/caddy/caddymain"
+	"github.com/mholt/caddy/caddyhttp/httpserver"
+	"github.com/spf13/cobra"
+	apiserverconfig "kubesphere.io/kubesphere/pkg/server/config"
+	"kubesphere.io/kubesphere/pkg/simple/client"
+	"kubesphere.io/kubesphere/pkg/utils/signals"
+
+	"kubesphere.io/kubesphere/pkg/apigateway"
+)
+
+func NewAPIGatewayCommand() *cobra.Command {
+
+	cmd := &cobra.Command{
+		Use: "ks-apigateway",
+		Long: `The KubeSphere API Gateway, which is responsible 
+for proxy request to the right backend. API Gateway also proxy 
+Kubernetes API Server for KubeSphere authorization purpose.
+`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+
+			err := apiserverconfig.Load()
+			if err != nil {
+				return err
+			}
+
+			apigateway.RegisterPlugins()
+
+			return Run(signals.SetupSignalHandler())
+		},
+	}
+
+	cmd.Flags().AddGoFlagSet(flag.CommandLine)
+
+	return cmd
+}
+
+func Run(stopCh <-chan struct{}) error {
+
+	csop := &client.ClientSetOptions{}
+	csop.SetKubernetesOptions(apiserverconfig.Get().KubernetesOptions)
+	client.NewClientSetFactory(csop, stopCh)
+
+	httpserver.RegisterDevDirective("authenticate", "jwt")
+	httpserver.RegisterDevDirective("authentication", "jwt")
+	httpserver.RegisterDevDirective("swagger", "jwt")
+	caddymain.Run()
+
+	return nil
+}

cmd/ks-apiserver/apiserver.go

@@ -20,15 +20,6 @@ package main
 import (
 	"kubesphere.io/kubesphere/cmd/ks-apiserver/app"
 	"log"
-	// Install apis
-	_ "kubesphere.io/kubesphere/pkg/apis/devops/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/logging/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/monitoring/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/operations/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/resources/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/servicemesh/metrics/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/tenant/install"
-	_ "kubesphere.io/kubesphere/pkg/apis/terminal/install"
 )
 
 func main() {

cmd/ks-apiserver/app/options/options.go

@@ -1,45 +1,74 @@
 package options
 
 import (
-	"github.com/spf13/pflag"
-	genericoptions "kubesphere.io/kubesphere/pkg/options"
+	"flag"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	genericoptions "kubesphere.io/kubesphere/pkg/server/options"
+	"kubesphere.io/kubesphere/pkg/simple/client/devops"
+	esclient "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch"
+	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/mysql"
+	"kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
+	"kubesphere.io/kubesphere/pkg/simple/client/prometheus"
+	"kubesphere.io/kubesphere/pkg/simple/client/s2is3"
+	"kubesphere.io/kubesphere/pkg/simple/client/servicemesh"
+	"kubesphere.io/kubesphere/pkg/simple/client/sonarqube"
+	"strings"
 )
 
 type ServerRunOptions struct {
 	GenericServerRunOptions *genericoptions.ServerRunOptions
 
-	// istio pilot discovery service url
-	IstioPilotServiceURL string
-
-	// jaeger query service url
-	JaegerQueryServiceUrl string
-
-	// prometheus service url for servicemesh metrics
-	ServicemeshPrometheusServiceUrl string
-
-	// openpitrix api gateway service url
-	OpenPitrixServer string
-
-	// openpitrix service token
-	OpenPitrixProxyToken string
+	KubernetesOptions  *k8s.KubernetesOptions
+	DevopsOptions      *devops.DevopsOptions
+	SonarQubeOptions   *sonarqube.SonarQubeOptions
+	ServiceMeshOptions *servicemesh.ServiceMeshOptions
+	MySQLOptions       *mysql.MySQLOptions
+	MonitoringOptions  *prometheus.PrometheusOptions
+	S3Options          *s2is3.S3Options
+	OpenPitrixOptions  *openpitrix.OpenPitrixOptions
+	LoggingOptions     *esclient.ElasticSearchOptions
 }
 
 func NewServerRunOptions() *ServerRunOptions {
 
 	s := ServerRunOptions{
 		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
-		IstioPilotServiceURL:    "http://istio-pilot.istio-system.svc:8080/version",
-		JaegerQueryServiceUrl:   "http://jaeger-query.istio-system.svc:16686/jaeger",
+		KubernetesOptions:       k8s.NewKubernetesOptions(),
+		DevopsOptions:           devops.NewDevopsOptions(),
+		SonarQubeOptions:        sonarqube.NewSonarQubeOptions(),
+		ServiceMeshOptions:      servicemesh.NewServiceMeshOptions(),
+		MySQLOptions:            mysql.NewMySQLOptions(),
+		MonitoringOptions:       prometheus.NewPrometheusOptions(),
+		S3Options:               s2is3.NewS3Options(),
+		OpenPitrixOptions:       openpitrix.NewOpenPitrixOptions(),
+		LoggingOptions:          esclient.NewElasticSearchOptions(),
 	}
 
 	return &s
 }
 
-func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
+func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 
-	s.GenericServerRunOptions.AddFlags(fs)
+	s.GenericServerRunOptions.AddFlags(fss.FlagSet("generic"))
+	s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"))
+	s.MySQLOptions.AddFlags(fss.FlagSet("mysql"))
+	s.DevopsOptions.AddFlags(fss.FlagSet("devops"))
+	s.SonarQubeOptions.AddFlags(fss.FlagSet("sonarqube"))
+	s.S3Options.AddFlags(fss.FlagSet("s3"))
+	s.OpenPitrixOptions.AddFlags(fss.FlagSet("openpitrix"))
+	s.ServiceMeshOptions.AddFlags(fss.FlagSet("servicemesh"))
+	s.MonitoringOptions.AddFlags(fss.FlagSet("monitoring"))
+	s.LoggingOptions.AddFlags(fss.FlagSet("logging"))
 
-	fs.StringVar(&s.IstioPilotServiceURL, "istio-pilot-service-url", "http://istio-pilot.istio-system.svc:8080/version", "istio pilot discovery service url")
-	fs.StringVar(&s.JaegerQueryServiceUrl, "jaeger-query-service-url", "http://jaeger-query.istio-system.svc:16686/jaeger", "jaeger query service url")
-	fs.StringVar(&s.ServicemeshPrometheusServiceUrl, "servicemesh-prometheus-service-url", "http://prometheus-k8s-system.kubesphere-monitoring-system.svc:9090", "prometheus service for servicemesh")
+	fs := fss.FlagSet("klog")
+	local := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(local)
+	local.VisitAll(func(fl *flag.Flag) {
+		fl.Name = strings.Replace(fl.Name, "_", "-", -1)
+		fs.AddGoFlag(fl)
+	})
+
+	return fss
 }

cmd/ks-apiserver/app/options/validation.go

@@ -0,0 +1,19 @@
+package options
+
+// Validate validates server run options, to find
+// options' misconfiguration
+func (s *ServerRunOptions) Validate() []error {
+	var errors []error
+
+	errors = append(errors, s.DevopsOptions.Validate()...)
+	errors = append(errors, s.KubernetesOptions.Validate()...)
+	errors = append(errors, s.MySQLOptions.Validate()...)
+	errors = append(errors, s.ServiceMeshOptions.Validate()...)
+	errors = append(errors, s.MonitoringOptions.Validate()...)
+	errors = append(errors, s.SonarQubeOptions.Validate()...)
+	errors = append(errors, s.S3Options.Validate()...)
+	errors = append(errors, s.OpenPitrixOptions.Validate()...)
+	errors = append(errors, s.LoggingOptions.Validate()...)
+
+	return errors
+}

cmd/ks-apiserver/app/server.go

@@ -18,181 +18,336 @@
 package app
 
 import (
-	goflag "flag"
 	"fmt"
-	"github.com/golang/glog"
-	"github.com/json-iterator/go"
 	kconfig "github.com/kiali/kiali/config"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
 	"kubesphere.io/kubesphere/cmd/ks-apiserver/app/options"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
 	"kubesphere.io/kubesphere/pkg/apiserver/servicemesh/tracing"
-	"kubesphere.io/kubesphere/pkg/filter"
 	"kubesphere.io/kubesphere/pkg/informers"
-	"kubesphere.io/kubesphere/pkg/models/devops"
-	logging "kubesphere.io/kubesphere/pkg/models/log"
 	"kubesphere.io/kubesphere/pkg/server"
-	"kubesphere.io/kubesphere/pkg/signals"
-	"kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins"
-	"kubesphere.io/kubesphere/pkg/simple/client/devops_mysql"
-	"log"
+	apiserverconfig "kubesphere.io/kubesphere/pkg/server/config"
+	"kubesphere.io/kubesphere/pkg/server/filter"
+	"kubesphere.io/kubesphere/pkg/simple/client"
+	"kubesphere.io/kubesphere/pkg/utils/signals"
+	"kubesphere.io/kubesphere/pkg/utils/term"
 	"net/http"
-)
 
-var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
+	"kubesphere.io/kubesphere/pkg/apis"
+)
 
 func NewAPIServerCommand() *cobra.Command {
 	s := options.NewServerRunOptions()
 
 	cmd := &cobra.Command{
 		Use: "ks-apiserver",
-		Long: `The KubeSphere API server validates and configures data
-for the api objects. The API Server services REST operations and provides the frontend to the
+		Long: `The KubeSphere API server validates and configures data for the api objects. 
+The API Server services REST operations and provides the frontend to the
 cluster's shared state through which all other components interact.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Run(s)
+			err := apiserverconfig.Load()
+			if err != nil {
+				return err
+			}
+
+			err = Complete(s)
+			if err != nil {
+				return err
+			}
+
+			if errs := s.Validate(); len(errs) != 0 {
+				return utilerrors.NewAggregate(errs)
+			}
+
+			return Run(s, signals.SetupSignalHandler())
 		},
 	}
 
-	s.AddFlags(cmd.Flags())
-	cmd.Flags().AddGoFlagSet(goflag.CommandLine)
-	glog.CopyStandardLogTo("INFO")
+	fs := cmd.Flags()
+	namedFlagSets := s.Flags()
 
+	for _, f := range namedFlagSets.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
+	usageFmt := "Usage:\n  %s\n"
+	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
+	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
+		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
+		cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
+	})
 	return cmd
 }
 
-func Run(s *options.ServerRunOptions) error {
+func Run(s *options.ServerRunOptions, stopCh <-chan struct{}) error {
 
-	pflag.VisitAll(func(flag *pflag.Flag) {
-		log.Printf("FLAG: --%s=%q", flag.Name, flag.Value)
-	})
-
-	var err error
-
-	waitForResourceSync()
-
-	container := runtime.Container
-	container.DoNotRecover(false)
-	container.Filter(filter.Logging)
-	container.RecoverHandler(server.LogStackOnRecover)
-	for _, webservice := range container.RegisteredWebServices() {
-		for _, route := range webservice.Routes() {
-			log.Println(route.Method, route.Path)
-		}
+	err := CreateClientSet(apiserverconfig.Get(), stopCh)
+	if err != nil {
+		return err
+	}
+
+	err = WaitForResourceSync(stopCh)
+	if err != nil {
+		return err
 	}
 
-	initializeAdminJenkins()
-	initializeDevOpsDatabase()
-	initializeESClientConfig()
 	initializeServicemeshConfig(s)
 
-	if s.GenericServerRunOptions.InsecurePort != 0 {
-		log.Printf("Server listening on %d.", s.GenericServerRunOptions.InsecurePort)
-		err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container)
+	err = CreateAPIServer(s)
+	if err != nil {
+		return err
 	}
 
-	if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 {
-		log.Printf("Server listening on %d.", s.GenericServerRunOptions.SecurePort)
-		err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container)
-	}
-
-	return err
-}
-
-func initializeAdminJenkins() {
-	devops.JenkinsInit()
-	admin_jenkins.Client()
-}
-
-func initializeDevOpsDatabase() {
-	devops_mysql.OpenDatabase()
+	return nil
 }
 
 func initializeServicemeshConfig(s *options.ServerRunOptions) {
 	// Initialize kiali config
 	config := kconfig.NewConfig()
 
-	tracing.JaegerQueryUrl = s.JaegerQueryServiceUrl
+	tracing.JaegerQueryUrl = s.ServiceMeshOptions.JaegerQueryHost
 
 	// Exclude system namespaces
 	config.API.Namespaces.Exclude = []string{"istio-system", "kubesphere*", "kube*"}
 	config.InCluster = true
 
 	// Set default prometheus service url
-	config.ExternalServices.PrometheusServiceURL = s.ServicemeshPrometheusServiceUrl
+	config.ExternalServices.PrometheusServiceURL = s.ServiceMeshOptions.ServicemeshPrometheusHost
 	config.ExternalServices.PrometheusCustomMetricsURL = config.ExternalServices.PrometheusServiceURL
 
 	// Set istio pilot discovery service url
-	config.ExternalServices.Istio.UrlServiceVersion = s.IstioPilotServiceURL
+	config.ExternalServices.Istio.UrlServiceVersion = s.ServiceMeshOptions.IstioPilotHost
 
 	kconfig.Set(config)
 }
 
-func initializeESClientConfig() {
+//
+func CreateAPIServer(s *options.ServerRunOptions) error {
+	var err error
 
-	// List all outputs
-	outputs, err := logging.GetFluentbitOutputFromConfigMap()
-	if err != nil {
-		glog.Errorln(err)
-		return
-	}
+	container := runtime.Container
+	container.DoNotRecover(false)
+	container.Filter(filter.Logging)
+	container.RecoverHandler(server.LogStackOnRecover)
 
-	// Iterate the outputs to get elasticsearch configs
-	for _, output := range outputs {
-		if configs := logging.ParseEsOutputParams(output.Parameters); configs != nil {
-			configs.WriteESConfigs()
-			return
+	apis.InstallAPIs(container)
+
+	// install config api
+	apiserverconfig.InstallAPI(container)
+
+	if s.GenericServerRunOptions.InsecurePort != 0 {
+		err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container)
+		if err == nil {
+			klog.V(0).Infof("Server listening on insecure port %d.", s.GenericServerRunOptions.InsecurePort)
 		}
 	}
+
+	if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 {
+		err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container)
+		if err == nil {
+			klog.V(0).Infof("Server listening on secure port %d.", s.GenericServerRunOptions.SecurePort)
+		}
+	}
+
+	return err
 }
 
-func waitForResourceSync() {
-	stopChan := signals.SetupSignalHandler()
+func CreateClientSet(conf *apiserverconfig.Config, stopCh <-chan struct{}) error {
+	csop := &client.ClientSetOptions{}
+
+	csop.SetDevopsOptions(conf.DevopsOptions).
+		SetSonarQubeOptions(conf.SonarQubeOptions).
+		SetKubernetesOptions(conf.KubernetesOptions).
+		SetMySQLOptions(conf.MySQLOptions).
+		SetLdapOptions(conf.LdapOptions).
+		SetS3Options(conf.S3Options).
+		SetOpenPitrixOptions(conf.OpenPitrixOptions).
+		SetPrometheusOptions(conf.MonitoringOptions).
+		SetKubeSphereOptions(conf.KubeSphereOptions).
+		SetElasticSearchOptions(conf.LoggingOptions)
+
+	client.NewClientSetFactory(csop, stopCh)
+
+	return nil
+}
+
+func WaitForResourceSync(stopCh <-chan struct{}) error {
+	klog.V(0).Info("Start cache objects")
+
+	discoveryClient := client.ClientSets().K8s().Discovery()
+	apiResourcesList, err := discoveryClient.ServerResources()
+	if err != nil {
+		return err
+	}
+
+	isResourceExists := func(resource schema.GroupVersionResource) bool {
+		for _, apiResource := range apiResourcesList {
+			if apiResource.GroupVersion == resource.GroupVersion().String() {
+				for _, rsc := range apiResource.APIResources {
+					if rsc.Name == resource.Resource {
+						return true
+					}
+				}
+			}
+		}
+		return false
+	}
 
 	informerFactory := informers.SharedInformerFactory()
-	informerFactory.Rbac().V1().Roles().Lister()
-	informerFactory.Rbac().V1().RoleBindings().Lister()
-	informerFactory.Rbac().V1().ClusterRoles().Lister()
-	informerFactory.Rbac().V1().ClusterRoleBindings().Lister()
 
-	informerFactory.Storage().V1().StorageClasses().Lister()
+	// resources we have to create informer first
+	k8sGVRs := []schema.GroupVersionResource{
+		{Group: "", Version: "v1", Resource: "namespaces"},
+		{Group: "", Version: "v1", Resource: "nodes"},
+		{Group: "", Version: "v1", Resource: "resourcequotas"},
+		{Group: "", Version: "v1", Resource: "pods"},
+		{Group: "", Version: "v1", Resource: "services"},
+		{Group: "", Version: "v1", Resource: "persistentvolumeclaims"},
+		{Group: "", Version: "v1", Resource: "secrets"},
+		{Group: "", Version: "v1", Resource: "configmaps"},
 
-	informerFactory.Core().V1().Namespaces().Lister()
-	informerFactory.Core().V1().Nodes().Lister()
-	informerFactory.Core().V1().ResourceQuotas().Lister()
-	informerFactory.Core().V1().Pods().Lister()
-	informerFactory.Core().V1().Services().Lister()
-	informerFactory.Core().V1().PersistentVolumeClaims().Lister()
-	informerFactory.Core().V1().Secrets().Lister()
-	informerFactory.Core().V1().ConfigMaps().Lister()
+		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "roles"},
+		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "rolebindings"},
+		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"},
+		{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"},
 
-	informerFactory.Apps().V1().ControllerRevisions().Lister()
-	informerFactory.Apps().V1().StatefulSets().Lister()
-	informerFactory.Apps().V1().Deployments().Lister()
-	informerFactory.Apps().V1().DaemonSets().Lister()
-	informerFactory.Apps().V1().ReplicaSets().Lister()
+		{Group: "apps", Version: "v1", Resource: "deployments"},
+		{Group: "apps", Version: "v1", Resource: "daemonsets"},
+		{Group: "apps", Version: "v1", Resource: "replicasets"},
+		{Group: "apps", Version: "v1", Resource: "statefulsets"},
+		{Group: "apps", Version: "v1", Resource: "controllerrevisions"},
 
-	informerFactory.Batch().V1().Jobs().Lister()
-	informerFactory.Batch().V1beta1().CronJobs().Lister()
-	informerFactory.Extensions().V1beta1().Ingresses().Lister()
+		{Group: "storage.k8s.io", Version: "v1", Resource: "storageclasses"},
 
-	informerFactory.Start(stopChan)
-	informerFactory.WaitForCacheSync(stopChan)
+		{Group: "batch", Version: "v1", Resource: "jobs"},
+		{Group: "batch", Version: "v1beta1", Resource: "cronjobs"},
+
+		{Group: "extensions", Version: "v1beta1", Resource: "ingresses"},
+
+		{Group: "autoscaling", Version: "v2beta2", Resource: "horizontalpodautoscalers"},
+	}
+
+	for _, gvr := range k8sGVRs {
+		if !isResourceExists(gvr) {
+			klog.Warningf("resource %s not exists in the cluster", gvr)
+		} else {
+			_, err := informerFactory.ForResource(gvr)
+			if err != nil {
+				klog.Errorf("cannot create informer for %s", gvr)
+				return err
+			}
+		}
+	}
+
+	informerFactory.Start(stopCh)
+	informerFactory.WaitForCacheSync(stopCh)
 
 	s2iInformerFactory := informers.S2iSharedInformerFactory()
-	s2iInformerFactory.Devops().V1alpha1().S2iBuilderTemplates().Lister()
-	s2iInformerFactory.Devops().V1alpha1().S2iRuns().Lister()
-	s2iInformerFactory.Devops().V1alpha1().S2iBuilders().Lister()
 
-	s2iInformerFactory.Start(stopChan)
-	s2iInformerFactory.WaitForCacheSync(stopChan)
+	s2iGVRs := []schema.GroupVersionResource{
+		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuildertemplates"},
+		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2iruns"},
+		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibuilders"},
+	}
+
+	for _, gvr := range s2iGVRs {
+		if !isResourceExists(gvr) {
+			klog.Warningf("resource %s not exists in the cluster", gvr)
+		} else {
+			_, err := s2iInformerFactory.ForResource(gvr)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	s2iInformerFactory.Start(stopCh)
+	s2iInformerFactory.WaitForCacheSync(stopCh)
 
 	ksInformerFactory := informers.KsSharedInformerFactory()
-	ksInformerFactory.Tenant().V1alpha1().Workspaces().Lister()
 
-	ksInformerFactory.Start(stopChan)
-	ksInformerFactory.WaitForCacheSync(stopChan)
+	ksGVRs := []schema.GroupVersionResource{
+		{Group: "tenant.kubesphere.io", Version: "v1alpha1", Resource: "workspaces"},
+		{Group: "devops.kubesphere.io", Version: "v1alpha1", Resource: "s2ibinaries"},
+
+		{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "strategies"},
+		{Group: "servicemesh.kubesphere.io", Version: "v1alpha2", Resource: "servicepolicies"},
+	}
+
+	for _, gvr := range ksGVRs {
+		if !isResourceExists(gvr) {
+			klog.Warningf("resource %s not exists in the cluster", gvr)
+		} else {
+			_, err := ksInformerFactory.ForResource(gvr)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	ksInformerFactory.Start(stopCh)
+	ksInformerFactory.WaitForCacheSync(stopCh)
+
+	appInformerFactory := informers.AppSharedInformerFactory()
+
+	appGVRs := []schema.GroupVersionResource{
+		{Group: "app.k8s.io", Version: "v1beta1", Resource: "applications"},
+	}
+
+	for _, gvr := range appGVRs {
+		if !isResourceExists(gvr) {
+			klog.Warningf("resource %s not exists in the cluster", gvr)
+		} else {
+			_, err := appInformerFactory.ForResource(gvr)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	appInformerFactory.Start(stopCh)
+	appInformerFactory.WaitForCacheSync(stopCh)
+
+	klog.V(0).Info("Finished caching objects")
+
+	return nil
 
-	log.Println("resources sync success")
+}
+
+// apply server run options to configuration
+func Complete(s *options.ServerRunOptions) error {
+
+	// loading configuration file
+	conf := apiserverconfig.Get()
+
+	conf.Apply(&apiserverconfig.Config{
+		MySQLOptions:       s.MySQLOptions,
+		DevopsOptions:      s.DevopsOptions,
+		SonarQubeOptions:   s.SonarQubeOptions,
+		KubernetesOptions:  s.KubernetesOptions,
+		ServiceMeshOptions: s.ServiceMeshOptions,
+		MonitoringOptions:  s.MonitoringOptions,
+		S3Options:          s.S3Options,
+		OpenPitrixOptions:  s.OpenPitrixOptions,
+		LoggingOptions:     s.LoggingOptions,
+	})
+
+	*s = options.ServerRunOptions{
+		GenericServerRunOptions: s.GenericServerRunOptions,
+		KubernetesOptions:       conf.KubernetesOptions,
+		DevopsOptions:           conf.DevopsOptions,
+		SonarQubeOptions:        conf.SonarQubeOptions,
+		ServiceMeshOptions:      conf.ServiceMeshOptions,
+		MySQLOptions:            conf.MySQLOptions,
+		MonitoringOptions:       conf.MonitoringOptions,
+		S3Options:               conf.S3Options,
+		OpenPitrixOptions:       conf.OpenPitrixOptions,
+		LoggingOptions:          conf.LoggingOptions,
+	}
+
+	return nil
 }

cmd/ks-iam/apiserver.go

@@ -20,8 +20,6 @@ package main
 import (
 	"kubesphere.io/kubesphere/cmd/ks-iam/app"
 	"log"
-	// Install apis
-	_ "kubesphere.io/kubesphere/pkg/apis/iam/install"
 )
 
 func main() {

cmd/ks-iam/app/options/options.go

@@ -18,29 +18,69 @@
 package options
 
 import (
-	"github.com/spf13/pflag"
-	genericoptions "kubesphere.io/kubesphere/pkg/options"
+	"flag"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
+	genericoptions "kubesphere.io/kubesphere/pkg/server/options"
+	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
+	"kubesphere.io/kubesphere/pkg/simple/client/ldap"
+	"kubesphere.io/kubesphere/pkg/simple/client/mysql"
+	"kubesphere.io/kubesphere/pkg/simple/client/redis"
+	"strings"
+	"time"
 )
 
 type ServerRunOptions struct {
 	GenericServerRunOptions *genericoptions.ServerRunOptions
+	KubernetesOptions       *k8s.KubernetesOptions
+	LdapOptions             *ldap.LdapOptions
+	RedisOptions            *redis.RedisOptions
+	MySQLOptions            *mysql.MySQLOptions
 	AdminEmail              string
 	AdminPassword           string
-	TokenExpireTime         string
+	TokenIdleTimeout        time.Duration
 	JWTSecret               string
+	AuthRateLimit           string
+	EnableMultiLogin        bool
+	GenerateKubeConfig      bool
 }
 
 func NewServerRunOptions() *ServerRunOptions {
 	s := &ServerRunOptions{
 		GenericServerRunOptions: genericoptions.NewServerRunOptions(),
+		KubernetesOptions:       k8s.NewKubernetesOptions(),
+		LdapOptions:             ldap.NewLdapOptions(),
+		MySQLOptions:            mysql.NewMySQLOptions(),
+		RedisOptions:            redis.NewRedisOptions(),
 	}
 	return s
 }
 
-func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
+func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
+
+	fs := fss.FlagSet("generic")
+
+	s.GenericServerRunOptions.AddFlags(fs)
 	fs.StringVar(&s.AdminEmail, "admin-email", "admin@kubesphere.io", "default administrator's email")
 	fs.StringVar(&s.AdminPassword, "admin-password", "passw0rd", "default administrator's password")
-	fs.StringVar(&s.TokenExpireTime, "token-expire-time", "2h", "token expire time,valid time units are \"ns\",\"us\",\"ms\",\"s\",\"m\",\"h\"")
+	fs.DurationVar(&s.TokenIdleTimeout, "token-idle-timeout", 30*time.Minute, "tokens that are idle beyond that time will expire,0s means the token has no expiration time. valid time units are \"ns\",\"us\",\"ms\",\"s\",\"m\",\"h\"")
 	fs.StringVar(&s.JWTSecret, "jwt-secret", "", "jwt secret")
-	s.GenericServerRunOptions.AddFlags(fs)
+	fs.StringVar(&s.AuthRateLimit, "auth-rate-limit", "5/30m", "specifies the maximum number of authentication attempts permitted and time interval,valid time units are \"s\",\"m\",\"h\"")
+	fs.BoolVar(&s.EnableMultiLogin, "enable-multi-login", false, "allow one account to have multiple sessions")
+	fs.BoolVar(&s.GenerateKubeConfig, "generate-kubeconfig", true, "generate kubeconfig for new users, kubeconfig is required in devops pipeline, set to false if you don't need devops.")
+
+	s.KubernetesOptions.AddFlags(fss.FlagSet("kubernetes"))
+	s.LdapOptions.AddFlags(fss.FlagSet("ldap"))
+	s.RedisOptions.AddFlags(fss.FlagSet("redis"))
+	s.MySQLOptions.AddFlags(fss.FlagSet("mysql"))
+
+	kfs := fss.FlagSet("klog")
+	local := flag.NewFlagSet("klog", flag.ExitOnError)
+	klog.InitFlags(local)
+	local.VisitAll(func(fl *flag.Flag) {
+		fl.Name = strings.Replace(fl.Name, "_", "-", -1)
+		kfs.AddGoFlag(fl)
+	})
+
+	return fss
 }

cmd/ks-iam/app/options/validation.go

@@ -0,0 +1,11 @@
+package options
+
+func (s *ServerRunOptions) Validate() []error {
+	errs := []error{}
+
+	errs = append(errs, s.KubernetesOptions.Validate()...)
+	errs = append(errs, s.GenericServerRunOptions.Validate()...)
+	errs = append(errs, s.LdapOptions.Validate()...)
+
+	return errs
+}

cmd/ks-iam/app/server.go

@@ -18,24 +18,24 @@
 package app
 
 import (
-	goflag "flag"
 	"fmt"
-	"github.com/golang/glog"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	cliflag "k8s.io/component-base/cli/flag"
+	"k8s.io/klog"
 	"kubesphere.io/kubesphere/cmd/ks-iam/app/options"
+	"kubesphere.io/kubesphere/pkg/apis"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
-	"kubesphere.io/kubesphere/pkg/filter"
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/models/iam"
 	"kubesphere.io/kubesphere/pkg/server"
-	"kubesphere.io/kubesphere/pkg/signals"
-	"kubesphere.io/kubesphere/pkg/simple/client/admin_jenkins"
-	"kubesphere.io/kubesphere/pkg/simple/client/devops_mysql"
+	apiserverconfig "kubesphere.io/kubesphere/pkg/server/config"
+	"kubesphere.io/kubesphere/pkg/server/filter"
+	"kubesphere.io/kubesphere/pkg/simple/client"
 	"kubesphere.io/kubesphere/pkg/utils/jwtutil"
-	"log"
+	"kubesphere.io/kubesphere/pkg/utils/signals"
+	"kubesphere.io/kubesphere/pkg/utils/term"
 	"net/http"
-	"time"
 )
 
 func NewAPIServerCommand() *cobra.Command {
@@ -43,39 +43,59 @@ func NewAPIServerCommand() *cobra.Command {
 
 	cmd := &cobra.Command{
 		Use: "ks-iam",
-		Long: `The KubeSphere API server validates and configures data
+		Long: `The KubeSphere account server validates and configures data
 for the api objects. The API Server services REST operations and provides the frontend to the
 cluster's shared state through which all other components interact.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return Run(s)
+
+			err := apiserverconfig.Load()
+			if err != nil {
+				return err
+			}
+
+			err = Complete(s)
+			if err != nil {
+				return err
+			}
+
+			if errs := s.Validate(); len(errs) != 0 {
+				return utilerrors.NewAggregate(errs)
+			}
+
+			return Run(s, signals.SetupSignalHandler())
 		},
 	}
-	s.AddFlags(cmd.Flags())
-	cmd.Flags().AddGoFlagSet(goflag.CommandLine)
-	glog.CopyStandardLogTo("INFO")
+
+	fs := cmd.Flags()
+	namedFlagSets := s.Flags()
+
+	for _, f := range namedFlagSets.FlagSets {
+		fs.AddFlagSet(f)
+	}
+
+	usageFmt := "Usage:\n  %s\n"
+	cols, _, _ := term.TerminalSize(cmd.OutOrStdout())
+	cmd.SetHelpFunc(func(cmd *cobra.Command, args []string) {
+		fmt.Fprintf(cmd.OutOrStdout(), "%s\n\n"+usageFmt, cmd.Long, cmd.UseLine())
+		cliflag.PrintSections(cmd.OutOrStdout(), namedFlagSets, cols)
+	})
 
 	return cmd
 }
 
-func Run(s *options.ServerRunOptions) error {
-	pflag.VisitAll(func(flag *pflag.Flag) {
-		log.Printf("FLAG: --%s=%q", flag.Name, flag.Value)
-	})
+func Run(s *options.ServerRunOptions, stopChan <-chan struct{}) error {
+	csop := client.NewClientSetOptions()
+	csop.SetKubernetesOptions(s.KubernetesOptions).
+		SetLdapOptions(s.LdapOptions).
+		SetRedisOptions(s.RedisOptions).
+		SetMySQLOptions(s.MySQLOptions)
 
-	var err error
+	client.NewClientSetFactory(csop, stopChan)
 
-	expireTime, err := time.ParseDuration(s.TokenExpireTime)
+	waitForResourceSync(stopChan)
 
-	if err != nil {
-		return err
-	}
+	err := iam.Init(s.AdminEmail, s.AdminPassword, s.AuthRateLimit, s.TokenIdleTimeout, s.EnableMultiLogin, s.GenerateKubeConfig)
 
-	waitForResourceSync()
-
-	initializeAdminJenkins()
-	initializeDevOpsDatabase()
-
-	err = iam.Init(s.AdminEmail, s.AdminPassword, expireTime)
 	jwtutil.Setup(s.JWTSecret)
 
 	if err != nil {
@@ -87,27 +107,40 @@ func Run(s *options.ServerRunOptions) error {
 	container.DoNotRecover(false)
 	container.RecoverHandler(server.LogStackOnRecover)
 
-	for _, webservice := range container.RegisteredWebServices() {
-		for _, route := range webservice.Routes() {
-			log.Println(route.Method, route.Path)
-		}
-	}
+	apis.InstallAuthorizationAPIs(container)
 
 	if s.GenericServerRunOptions.InsecurePort != 0 {
-		log.Printf("Server listening on %d.", s.GenericServerRunOptions.InsecurePort)
+		klog.Infof("Server listening on %s:%d ", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort)
 		err = http.ListenAndServe(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.InsecurePort), container)
 	}
 
 	if s.GenericServerRunOptions.SecurePort != 0 && len(s.GenericServerRunOptions.TlsCertFile) > 0 && len(s.GenericServerRunOptions.TlsPrivateKey) > 0 {
-		log.Printf("Server listening on %d.", s.GenericServerRunOptions.SecurePort)
+		klog.Infof("Server listening on %s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort)
 		err = http.ListenAndServeTLS(fmt.Sprintf("%s:%d", s.GenericServerRunOptions.BindAddress, s.GenericServerRunOptions.SecurePort), s.GenericServerRunOptions.TlsCertFile, s.GenericServerRunOptions.TlsPrivateKey, container)
 	}
 
 	return err
 }
 
-func waitForResourceSync() {
-	stopChan := signals.SetupSignalHandler()
+func Complete(s *options.ServerRunOptions) error {
+	conf := apiserverconfig.Get()
+
+	conf.Apply(&apiserverconfig.Config{
+		KubernetesOptions: s.KubernetesOptions,
+		LdapOptions:       s.LdapOptions,
+		RedisOptions:      s.RedisOptions,
+		MySQLOptions:      s.MySQLOptions,
+	})
+
+	s.KubernetesOptions = conf.KubernetesOptions
+	s.LdapOptions = conf.LdapOptions
+	s.RedisOptions = conf.RedisOptions
+	s.MySQLOptions = conf.MySQLOptions
+
+	return nil
+}
+
+func waitForResourceSync(stopCh <-chan struct{}) {
 
 	informerFactory := informers.SharedInformerFactory()
 	informerFactory.Rbac().V1().Roles().Lister()
@@ -117,21 +150,12 @@ func waitForResourceSync() {
 
 	informerFactory.Core().V1().Namespaces().Lister()
 
-	informerFactory.Start(stopChan)
-	informerFactory.WaitForCacheSync(stopChan)
+	informerFactory.Start(stopCh)
+	informerFactory.WaitForCacheSync(stopCh)
 
 	ksInformerFactory := informers.KsSharedInformerFactory()
 	ksInformerFactory.Tenant().V1alpha1().Workspaces().Lister()
 
-	ksInformerFactory.Start(stopChan)
-	ksInformerFactory.WaitForCacheSync(stopChan)
-	log.Println("resources sync success")
-}
-
-func initializeAdminJenkins() {
-	admin_jenkins.Client()
-}
-
-func initializeDevOpsDatabase() {
-	devops_mysql.OpenDatabase()
+	ksInformerFactory.Start(stopCh)
+	ksInformerFactory.WaitForCacheSync(stopCh)
 }

cmd/ks-network/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"flag"
+
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/controller/network/runoption"
+)
+
+var opt runoption.RunOption
+
+func init() {
+	flag.StringVar(&opt.ProviderName, "np-provider", "calico", "specify the network policy provider, k8s or calico")
+	flag.BoolVar(&opt.AllowInsecureEtcd, "allow-insecure-etcd", false, "specify allow connect to etcd using insecure http")
+	flag.StringVar(&opt.DataStoreType, "datastore-type", "k8s", "specify the datastore type of calico")
+	//TODO add more flags
+}
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Set("logtostderr", "true")
+	flag.Parse()
+	klog.V(1).Info("Preparing kubernetes client")
+	klog.Fatal(opt.Run())
+}

config/crds/devops_v1alpha1_s2ibinary.yaml

@@ -0,0 +1,73 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: s2ibinaries.devops.kubesphere.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .spec.fileName
+    name: FileName
+    type: string
+  - JSONPath: .spec.md5
+    name: MD5
+    type: string
+  - JSONPath: .spec.size
+    name: Size
+    type: string
+  - JSONPath: .status.phase
+    name: Phase
+    type: string
+  group: devops.kubesphere.io
+  names:
+    kind: S2iBinary
+    plural: s2ibinaries
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          properties:
+            downloadURL:
+              description: DownloadURL in KubeSphere
+              type: string
+            fileName:
+              description: FileName is filename of binary
+              type: string
+            md5:
+              description: MD5 is Binary's MD5 Hash
+              type: string
+            size:
+              description: Size is the file size of file
+              type: string
+            uploadTimeStamp:
+              description: UploadTime is last upload time
+              format: date-time
+              type: string
+          type: object
+        status:
+          properties:
+            phase:
+              description: Phase is status of S2iBinary . Possible value is "Ready","UnableToDownload"
+              type: string
+          type: object
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

config/crds/servicemesh_v1alpha2_servicepolicy.yaml

@@ -83,6 +83,16 @@ spec:
                                   http:
                                     description: HTTP connection pool settings.
                                     properties:
+                                      http1MaxPendingRequests:
+                                        description: Maximum number of pending HTTP
+                                          requests to a destination. Default 1024.
+                                        format: int32
+                                        type: integer
+                                      http2MaxRequests:
+                                        description: Maximum number of requests to
+                                          a backend. Default 1024.
+                                        format: int32
+                                        type: integer
                                       maxRequestsPerConnection:
                                         description: Maximum number of requests per
                                           connection to a backend. Setting this parameter
@@ -211,6 +221,17 @@ spec:
                                         http:
                                           description: HTTP connection pool settings.
                                           properties:
+                                            http1MaxPendingRequests:
+                                              description: Maximum number of pending
+                                                HTTP requests to a destination. Default
+                                                1024.
+                                              format: int32
+                                              type: integer
+                                            http2MaxRequests:
+                                              description: Maximum number of requests
+                                                to a backend. Default 1024.
+                                              format: int32
+                                              type: integer
                                             maxRequestsPerConnection:
                                               description: Maximum number of requests
                                                 per connection to a backend. Setting
@@ -464,6 +485,16 @@ spec:
                             http:
                               description: HTTP connection pool settings.
                               properties:
+                                http1MaxPendingRequests:
+                                  description: Maximum number of pending HTTP requests
+                                    to a destination. Default 1024.
+                                  format: int32
+                                  type: integer
+                                http2MaxRequests:
+                                  description: Maximum number of requests to a backend.
+                                    Default 1024.
+                                  format: int32
+                                  type: integer
                                 maxRequestsPerConnection:
                                   description: Maximum number of requests per connection
                                     to a backend. Setting this parameter to 1 disables
@@ -586,6 +617,16 @@ spec:
                                   http:
                                     description: HTTP connection pool settings.
                                     properties:
+                                      http1MaxPendingRequests:
+                                        description: Maximum number of pending HTTP
+                                          requests to a destination. Default 1024.
+                                        format: int32
+                                        type: integer
+                                      http2MaxRequests:
+                                        description: Maximum number of requests to
+                                          a backend. Default 1024.
+                                        format: int32
+                                        type: integer
                                       maxRequestsPerConnection:
                                         description: Maximum number of requests per
                                           connection to a backend. Setting this parameter

config/rbac/rbac_role.yaml

@@ -4,100 +4,140 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces/status
-    verbs:
-      - get
-      - update
-      - patch
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
-  - apiGroups:
-      - ""
-    resources:
-      - namespaces/status
-    verbs:
-      - get
-      - update
-      - patch
-  - apiGroups:
-      - tenant.kubesphere.io
-    resources:
-      - workspaces
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
-  - apiGroups:
-      - tenant.kubesphere.io
-    resources:
-      - workspaces/status
-    verbs:
-      - get
-      - update
-      - patch
-  - apiGroups:
-      - admissionregistration.k8s.io
-    resources:
-      - mutatingwebhookconfigurations
-      - validatingwebhookconfigurations
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
-  - apiGroups:
-      - ""
-    resources:
-      - secrets
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
-  - apiGroups:
-      - ""
-    resources:
-      - services
-    verbs:
-      - get
-      - list
-      - watch
-      - create
-      - update
-      - patch
-      - delete
+- apiGroups:
+  - core.kubesphere.io
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - core.kubesphere.io
+  resources:
+  - namespaces/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - core.kubesphere.io
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - core.kubesphere.io
+  resources:
+  - namespaces/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - apps
+  resources:
+  - deployments/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - devops.kubesphere.io
+  resources:
+  - s2ibinaries
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - devops.kubesphere.io
+  resources:
+  - s2ibinaries/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - tenant.kubesphere.io
+  resources:
+  - workspaces
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - tenant.kubesphere.io
+  resources:
+  - workspaces/status
+  verbs:
+  - get
+  - update
+  - patch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - update
+  - patch
+  - delete

config/samples/devops_v1alpha1_s2ibinary.yaml

@@ -0,0 +1,6 @@
+apiVersion: devops.kubesphere.io/v1alpha1
+kind: S2iBinary
+metadata:
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: s2ibinary-sample

config/samples/network_v1alpha1_namespacenetworkpolicy.yaml

@@ -0,0 +1,9 @@
+apiVersion: network.kubesphere.io/v1alpha1
+kind: NamespaceNetworkPolicy
+metadata:
+  labels:
+    controller-tools.k8s.io: "1.0"
+  name: namespacenetworkpolicy-sample
+spec:
+  # Add fields here
+  foo: bar

docs/code-of-conduct.md

@@ -2,7 +2,7 @@
 
 KubeSphere follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).  
 
-# Best practice of committing code
+# Best practices of committing code
 Besides following above conduct from CNCF, we also hope every contributor in this project could help us to improve the quality of code, something you should know before checking in any new code:
  - As gopher, make sure you already read [the conduct of Go language](https://golang.org/conduct) and [the instruction of writting Go](https://golang.org/doc/effective_go.html).  
  - Fork the project under your account and make the changes you want there.  
@@ -11,4 +11,4 @@ Besides following above conduct from CNCF, we also hope every contributor in thi
  - Every PR should only solve one problem or provide one feature, don't put several different fixes into one PR.  
  - At lease two code reviewers should involve into code reviewing process. 
  - Please introduce new third-party packages as little as possible to reduce the vendor dependency of this project. For example, don't import a full unit converting package but only use one function from it. For this case, you'd better write that function by yourself.
- - more.
+ - more.

docs/screenshots.md

@@ -0,0 +1,74 @@
+# KubeSphere Screenshots
+
+Screenshots of various KubeSphere dashboard views:
+
+## Workbench Overview
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111625.png)
+
+## Application Template
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111410.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111441.png)
+
+## Project Resources Management
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710112617.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710112653.png)
+
+## DevOps Project
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113123.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113153.png)
+
+## Service Mesh
+
+### Traffic Management
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002111.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002849.png)
+
+### Tracing
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002331.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002424.png)
+
+## Grayscale Release
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002554.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190713002641.png)
+
+## Node Management
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111701.png)
+
+## Service Components
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111734.png)
+
+## Workspace Management
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710111851.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710112018.png)
+
+## Monitoring Center
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113420.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113444.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113543.png)
+
+## Log and Alert
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113300.png)
+
+![](https://pek3b.qingstor.com/kubesphere-docs/png/20190710113650.png)
+

go.mod

@@ -0,0 +1,434 @@
+// This is a generated file. Do not edit directly.
+// Run hack/pin-dependency.sh to change pinned dependency versions.
+// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
+
+module kubesphere.io/kubesphere
+
+go 1.12
+
+require (
+	bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c // indirect
+	code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Azure/go-autorest/autorest v0.5.0 // indirect
+	github.com/Masterminds/semver v1.5.0 // indirect
+	github.com/Microsoft/go-winio v0.4.12 // indirect
+	github.com/NYTimes/gziphandler v1.1.1 // indirect
+	github.com/PuerkitoBio/goquery v1.5.0
+	github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf
+	github.com/aws/aws-sdk-go v1.22.2
+	github.com/beevik/etree v1.1.0
+	github.com/beorn7/perks v1.0.0 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/coreos/bbolt v1.3.3 // indirect
+	github.com/coreos/etcd v3.3.13+incompatible // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker v1.4.2-0.20190822205725-ed20165a37b4
+	github.com/docker/go-connections v0.3.0 // indirect
+	github.com/docker/go-units v0.3.3 // indirect
+	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
+	github.com/elastic/go-elasticsearch/v5 v5.6.1
+	github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/elastic/go-elasticsearch/v7 v7.3.0
+	github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect
+	github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 // indirect
+	github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 // indirect
+	github.com/emicklei/go-restful v2.9.5+incompatible
+	github.com/emicklei/go-restful-openapi v1.0.0
+	github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6 // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
+	github.com/fatih/structs v1.1.0
+	github.com/go-ldap/ldap v3.0.3+incompatible
+	github.com/go-logr/logr v0.1.0
+	github.com/go-openapi/jsonpointer v0.19.0 // indirect
+	github.com/go-openapi/jsonreference v0.19.0 // indirect
+	github.com/go-openapi/spec v0.19.0
+	github.com/go-openapi/strfmt v0.19.0
+	github.com/go-openapi/swag v0.19.0 // indirect
+	github.com/go-playground/locales v0.12.1 // indirect
+	github.com/go-playground/universal-translator v0.16.0 // indirect
+	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/gobuffalo/flect v0.1.5 // indirect
+	github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6
+	github.com/golang/example v0.0.0-20170904185048-46695d81d1fa
+	github.com/golang/protobuf v1.3.1
+	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/gofuzz v1.0.0 // indirect
+	github.com/google/uuid v1.1.1
+	github.com/googleapis/gnostic v0.2.0 // indirect
+	github.com/gophercloud/gophercloud v0.3.0 // indirect
+	github.com/gorilla/mux v1.7.1 // indirect
+	github.com/gorilla/websocket v1.4.0
+	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect
+	github.com/hashicorp/go-version v1.2.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.7 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.6
+	github.com/kelseyhightower/envconfig v1.4.0 // indirect
+	github.com/kiali/kiali v0.15.1-0.20200213040359-608aece2aa66
+	github.com/klauspost/cpuid v1.2.1 // indirect
+	github.com/knative/pkg v0.0.0-20190314204845-cd278f2d3394
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kubesphere/s2ioperator v0.0.13
+	github.com/kubesphere/sonargo v0.0.2
+	github.com/leodido/go-urn v1.1.0 // indirect
+	github.com/lib/pq v1.2.0 // indirect
+	github.com/lucas-clemente/quic-go v0.11.1 // indirect
+	github.com/mattn/go-sqlite3 v1.11.0 // indirect
+	github.com/mholt/caddy v1.0.0
+	github.com/mholt/certmagic v0.5.1 // indirect
+	github.com/miekg/dns v1.1.9 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
+	github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect
+	github.com/onsi/ginkgo v1.10.1
+	github.com/onsi/gomega v1.7.0
+	github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/openshift/api v3.9.0+incompatible // indirect
+	github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect
+	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba // indirect
+	github.com/projectcalico/go-yaml v0.0.0-20161201183616-955bc3e451ef // indirect
+	github.com/projectcalico/go-yaml-wrapper v0.0.0-20161127220527-598e54215bee // indirect
+	github.com/projectcalico/libcalico-go v0.0.0-20190708183129-ac36d966132f
+	github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 // indirect
+	github.com/prometheus/common v0.4.0
+	github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 // indirect
+	github.com/russross/blackfriday v1.5.2 // indirect
+	github.com/satori/go.uuid v1.2.0 // indirect
+	github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009
+	github.com/speps/go-hashids v2.0.0+incompatible
+	github.com/spf13/cobra v0.0.3
+	github.com/spf13/pflag v1.0.3
+	github.com/spf13/viper v1.4.0
+	github.com/stretchr/testify v1.3.0
+	github.com/xanzy/ssh-agent v0.2.1 // indirect
+	go.etcd.io/bbolt v1.3.3 // indirect
+	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
+	golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a // indirect
+	golang.org/x/tools v0.0.0-20190710153321-831012c29e42 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
+	google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 // indirect
+	google.golang.org/grpc v1.21.0
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
+	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
+	gopkg.in/go-playground/validator.v9 v9.29.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
+	gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
+	gopkg.in/src-d/go-git.v4 v4.11.0
+	gopkg.in/yaml.v2 v2.2.2
+	gotest.tools v2.2.0+incompatible // indirect
+	k8s.io/api v0.0.0-20190831074750-7364b6bdad65
+	k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476
+	k8s.io/apimachinery v0.0.0-20190831074630-461753078381
+	k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d
+	k8s.io/client-go v0.0.0-20190831074946-3fe2abece89e
+	k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b
+	k8s.io/component-base v0.0.0-20190831075413-37a093468564
+	k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a // indirect
+	k8s.io/klog v0.4.0
+	k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5
+	k8s.io/kubernetes v1.13.6
+	kubesphere.io/im v0.1.0 // indirect
+	openpitrix.io/iam v0.1.0 // indirect
+	openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c
+	sigs.k8s.io/application v0.0.0-20190404151855-67ae7f915d4e
+	sigs.k8s.io/controller-runtime v0.1.10
+	sigs.k8s.io/controller-tools v0.1.12
+	sigs.k8s.io/yaml v1.1.0 // indirect
+)
+
+replace (
+	bitbucket.org/ww/goautoneg => bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c
+	cloud.google.com/go => cloud.google.com/go v0.34.0
+	code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6
+	contrib.go.opencensus.io/exporter/ocagent => contrib.go.opencensus.io/exporter/ocagent v0.4.12
+	github.com/Azure/go-ansiterm => github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78
+	github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/autorest v0.5.0
+	github.com/Azure/go-autorest/autorest/adal => github.com/Azure/go-autorest/autorest/adal v0.2.0
+	github.com/Azure/go-autorest/autorest/date => github.com/Azure/go-autorest/autorest/date v0.1.0
+	github.com/Azure/go-autorest/autorest/mocks => github.com/Azure/go-autorest/autorest/mocks v0.1.0
+	github.com/Azure/go-autorest/logger => github.com/Azure/go-autorest/logger v0.1.0
+	github.com/Azure/go-autorest/tracing => github.com/Azure/go-autorest/tracing v0.1.0
+	github.com/BurntSushi/toml => github.com/BurntSushi/toml v0.3.1
+	github.com/Masterminds/semver => github.com/Masterminds/semver v1.5.0
+	github.com/Microsoft/go-winio => github.com/Microsoft/go-winio v0.4.12
+	github.com/NYTimes/gziphandler => github.com/NYTimes/gziphandler v1.1.1
+	github.com/PuerkitoBio/goquery => github.com/PuerkitoBio/goquery v1.5.0
+	github.com/PuerkitoBio/purell => github.com/PuerkitoBio/purell v1.1.0
+	github.com/PuerkitoBio/urlesc => github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578
+	github.com/Shopify/sarama => github.com/Shopify/sarama v1.19.0
+	github.com/Shopify/toxiproxy => github.com/Shopify/toxiproxy v2.1.4+incompatible
+	github.com/StackExchange/wmi => github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e
+	github.com/alcortesm/tgz => github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7
+	github.com/alecthomas/template => github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc
+	github.com/alecthomas/units => github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf
+	github.com/andybalholm/cascadia => github.com/andybalholm/cascadia v1.0.0
+	github.com/anmitsu/go-shlex => github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239
+	github.com/apache/thrift => github.com/apache/thrift v0.12.0
+	github.com/appscode/jsonpatch => github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30
+	github.com/armon/consul-api => github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6
+	github.com/asaskevich/govalidator => github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f
+	github.com/aws/aws-sdk-go => github.com/aws/aws-sdk-go v1.22.2
+	github.com/beevik/etree => github.com/beevik/etree v1.1.0
+	github.com/beorn7/perks => github.com/beorn7/perks v1.0.0
+	github.com/bitly/go-simplejson => github.com/bitly/go-simplejson v0.5.0
+	github.com/blang/semver => github.com/blang/semver v3.5.0+incompatible
+	github.com/bmizerany/assert => github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869
+	github.com/cenkalti/backoff => github.com/cenkalti/backoff v2.2.1+incompatible
+	github.com/census-instrumentation/opencensus-proto => github.com/census-instrumentation/opencensus-proto v0.2.0
+	github.com/cheekybits/genny => github.com/cheekybits/genny v1.0.0
+	github.com/client9/misspell => github.com/client9/misspell v0.3.4
+	github.com/coreos/bbolt => github.com/coreos/bbolt v1.3.3
+	github.com/coreos/etcd => github.com/coreos/etcd v3.3.13+incompatible
+	github.com/coreos/go-semver => github.com/coreos/go-semver v0.3.0
+	github.com/coreos/go-systemd => github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f
+	github.com/coreos/pkg => github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f
+	github.com/davecgh/go-spew => github.com/davecgh/go-spew v1.1.1
+	github.com/deckarep/golang-set => github.com/deckarep/golang-set v1.7.1
+	github.com/denisenkom/go-mssqldb => github.com/denisenkom/go-mssqldb v0.0.0-20190204142019-df6d76eb9289
+	github.com/dgrijalva/jwt-go => github.com/dgrijalva/jwt-go v3.2.0+incompatible
+	github.com/docker/distribution => github.com/docker/distribution v2.7.1+incompatible
+	github.com/docker/docker => github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4
+	github.com/docker/go-connections => github.com/docker/go-connections v0.3.0
+	github.com/docker/go-units => github.com/docker/go-units v0.3.3
+	github.com/docker/spdystream => github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c
+	github.com/dustin/go-humanize => github.com/dustin/go-humanize v1.0.0
+	github.com/eapache/go-resiliency => github.com/eapache/go-resiliency v1.1.0
+	github.com/eapache/go-xerial-snappy => github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21
+	github.com/eapache/queue => github.com/eapache/queue v1.1.0
+	github.com/elastic/go-elasticsearch/v5 => github.com/elastic/go-elasticsearch/v5 v5.6.1
+	github.com/elastic/go-elasticsearch/v6 => github.com/elastic/go-elasticsearch/v6 v6.8.2
+	github.com/elastic/go-elasticsearch/v7 => github.com/elastic/go-elasticsearch/v7 v7.3.0
+	github.com/elazarl/go-bindata-assetfs => github.com/elazarl/go-bindata-assetfs v1.0.0
+	github.com/elazarl/goproxy => github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2
+	github.com/elazarl/goproxy/ext => github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2
+	github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.9.5+incompatible
+	github.com/emicklei/go-restful-openapi => github.com/emicklei/go-restful-openapi v1.0.0
+	github.com/emicklei/go-restful-swagger12 => github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6
+	github.com/emirpasic/gods => github.com/emirpasic/gods v1.12.0
+	github.com/erikstmartin/go-testdb => github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5
+	github.com/evanphx/json-patch => github.com/evanphx/json-patch v4.2.0+incompatible
+	github.com/fatih/camelcase => github.com/fatih/camelcase v1.0.0
+	github.com/fatih/structs => github.com/fatih/structs v1.1.0
+	github.com/flynn/go-shlex => github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568
+	github.com/fsnotify/fsnotify => github.com/fsnotify/fsnotify v1.4.7
+	github.com/ghodss/yaml => github.com/ghodss/yaml v1.0.0
+	github.com/gliderlabs/ssh => github.com/gliderlabs/ssh v0.1.1
+	github.com/globalsign/mgo => github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb
+	github.com/go-acme/lego => github.com/go-acme/lego v2.5.0+incompatible
+	github.com/go-kit/kit => github.com/go-kit/kit v0.8.0
+	github.com/go-ldap/ldap => github.com/go-ldap/ldap v3.0.3+incompatible
+	github.com/go-logfmt/logfmt => github.com/go-logfmt/logfmt v0.3.0
+	github.com/go-logr/logr => github.com/go-logr/logr v0.1.0
+	github.com/go-logr/zapr => github.com/go-logr/zapr v0.1.1
+	github.com/go-ole/go-ole => github.com/go-ole/go-ole v1.2.1
+	github.com/go-openapi/errors => github.com/go-openapi/errors v0.17.0
+	github.com/go-openapi/jsonpointer => github.com/go-openapi/jsonpointer v0.19.0
+	github.com/go-openapi/jsonreference => github.com/go-openapi/jsonreference v0.19.0
+	github.com/go-openapi/spec => github.com/go-openapi/spec v0.19.0
+	github.com/go-openapi/strfmt => github.com/go-openapi/strfmt v0.19.0
+	github.com/go-openapi/swag => github.com/go-openapi/swag v0.19.0
+	github.com/go-playground/locales => github.com/go-playground/locales v0.12.1
+	github.com/go-playground/universal-translator => github.com/go-playground/universal-translator v0.16.0
+	github.com/go-redis/redis => github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-stack/stack => github.com/go-stack/stack v1.8.0
+	github.com/gobuffalo/flect => github.com/gobuffalo/flect v0.1.5
+	github.com/gocraft/dbr => github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6
+	github.com/gofrs/uuid => github.com/gofrs/uuid v3.2.0+incompatible
+	github.com/gogo/protobuf => github.com/gogo/protobuf v1.2.0
+	github.com/golang/example => github.com/golang/example v0.0.0-20170904185048-46695d81d1fa
+	github.com/golang/glog => github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/golang/groupcache => github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef
+	github.com/golang/mock => github.com/golang/mock v1.2.0
+	github.com/golang/protobuf => github.com/golang/protobuf v1.3.1
+	github.com/golang/snappy => github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db
+	github.com/google/btree => github.com/google/btree v1.0.0
+	github.com/google/go-cmp => github.com/google/go-cmp v0.2.0
+	github.com/google/go-querystring => github.com/google/go-querystring v1.0.0
+	github.com/google/gofuzz => github.com/google/gofuzz v1.0.0
+	github.com/google/gops => github.com/google/gops v0.3.6
+	github.com/google/uuid => github.com/google/uuid v1.1.1
+	github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.2.0
+	github.com/gophercloud/gophercloud => github.com/gophercloud/gophercloud v0.3.0
+	github.com/gorilla/context => github.com/gorilla/context v1.1.1
+	github.com/gorilla/mux => github.com/gorilla/mux v1.7.1
+	github.com/gorilla/websocket => github.com/gorilla/websocket v1.4.0
+	github.com/gregjones/httpcache => github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc
+	github.com/grpc-ecosystem/go-grpc-middleware => github.com/grpc-ecosystem/go-grpc-middleware v1.0.0
+	github.com/grpc-ecosystem/go-grpc-prometheus => github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/grpc-ecosystem/grpc-gateway => github.com/grpc-ecosystem/grpc-gateway v1.9.5
+	github.com/hashicorp/go-syslog => github.com/hashicorp/go-syslog v1.0.0
+	github.com/hashicorp/go-version => github.com/hashicorp/go-version v1.2.0
+	github.com/hashicorp/golang-lru => github.com/hashicorp/golang-lru v0.5.1
+	github.com/hashicorp/hcl => github.com/hashicorp/hcl v1.0.0
+	github.com/hpcloud/tail => github.com/hpcloud/tail v1.0.0
+	github.com/imdario/mergo => github.com/imdario/mergo v0.3.7
+	github.com/inconshreveable/mousetrap => github.com/inconshreveable/mousetrap v1.0.0
+	github.com/jbenet/go-context => github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99
+	github.com/jessevdk/go-flags => github.com/jessevdk/go-flags v1.4.0
+	github.com/jimstudt/http-authentication => github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a
+	github.com/jinzhu/gorm => github.com/jinzhu/gorm v1.9.2
+	github.com/jinzhu/inflection => github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a
+	github.com/jinzhu/now => github.com/jinzhu/now v1.0.0
+	github.com/jmespath/go-jmespath => github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af
+	github.com/jonboulle/clockwork => github.com/jonboulle/clockwork v0.1.0
+	github.com/json-iterator/go => github.com/json-iterator/go v1.1.6
+	github.com/julienschmidt/httprouter => github.com/julienschmidt/httprouter v1.2.0
+	github.com/kardianos/osext => github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1
+	github.com/kelseyhightower/envconfig => github.com/kelseyhightower/envconfig v1.4.0
+	github.com/kevinburke/ssh_config => github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e
+	github.com/keybase/go-ps => github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999
+	github.com/kiali/kiali => github.com/kubesphere/kiali v0.15.1-0.20200213040359-608aece2aa66
+	github.com/klauspost/cpuid => github.com/klauspost/cpuid v1.2.1
+	github.com/knative/pkg => github.com/knative/pkg v0.0.0-20190314204845-cd278f2d3394
+	github.com/koding/multiconfig => github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7
+	github.com/konsorten/go-windows-terminal-sequences => github.com/konsorten/go-windows-terminal-sequences v1.0.2
+	github.com/kr/logfmt => github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515
+	github.com/kr/pretty => github.com/kr/pretty v0.1.0
+	github.com/kr/pty => github.com/kr/pty v1.1.1
+	github.com/kr/text => github.com/kr/text v0.1.0
+	github.com/kubesphere/s2ioperator => github.com/kubesphere/s2ioperator v0.0.13
+	github.com/kubesphere/sonargo => github.com/kubesphere/sonargo v0.0.2
+	github.com/kylelemons/godebug => github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348
+	github.com/leodido/go-urn => github.com/leodido/go-urn v1.1.0
+	github.com/lib/pq => github.com/lib/pq v1.2.0
+	github.com/lucas-clemente/quic-go => github.com/lucas-clemente/quic-go v0.11.1
+	github.com/magiconair/properties => github.com/magiconair/properties v1.8.0
+	github.com/mailru/easyjson => github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329
+	github.com/marten-seemann/qtls => github.com/marten-seemann/qtls v0.2.3
+	github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 v1.11.0
+	github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions v1.0.1
+	github.com/mholt/caddy => github.com/mholt/caddy v1.0.0
+	github.com/mholt/certmagic => github.com/mholt/certmagic v0.5.1
+	github.com/miekg/dns => github.com/miekg/dns v1.1.9
+	github.com/mitchellh/go-homedir => github.com/mitchellh/go-homedir v1.1.0
+	github.com/mitchellh/mapstructure => github.com/mitchellh/mapstructure v1.1.2
+	github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
+	github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742
+	github.com/morikuni/aec => github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c
+	github.com/mwitkow/go-conntrack => github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223
+	github.com/naoina/go-stringutil => github.com/naoina/go-stringutil v0.1.0
+	github.com/naoina/toml => github.com/naoina/toml v0.1.1
+	github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.8.0
+	github.com/onsi/gomega => github.com/onsi/gomega v1.5.0
+	github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.0-rc1
+	github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.1
+	github.com/openshift/api => github.com/openshift/api v3.9.0+incompatible
+	github.com/openzipkin/zipkin-go => github.com/openzipkin/zipkin-go v0.1.6
+	github.com/pborman/uuid => github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709
+	github.com/pelletier/go-buffruneio => github.com/pelletier/go-buffruneio v0.2.0
+	github.com/pelletier/go-toml => github.com/pelletier/go-toml v1.2.0
+	github.com/peterbourgon/diskv => github.com/peterbourgon/diskv v2.0.1+incompatible
+	github.com/pierrec/lz4 => github.com/pierrec/lz4 v2.0.5+incompatible
+	github.com/pkg/errors => github.com/pkg/errors v0.8.1
+	github.com/pmezard/go-difflib => github.com/pmezard/go-difflib v1.0.0
+	github.com/projectcalico/go-json => github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba
+	github.com/projectcalico/go-yaml => github.com/projectcalico/go-yaml v0.0.0-20161201183616-955bc3e451ef
+	github.com/projectcalico/go-yaml-wrapper => github.com/projectcalico/go-yaml-wrapper v0.0.0-20161127220527-598e54215bee
+	github.com/projectcalico/libcalico-go => github.com/projectcalico/libcalico-go v0.0.0-20190708183129-ac36d966132f
+	github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
+	github.com/prometheus/client_model => github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90
+	github.com/prometheus/common => github.com/prometheus/common v0.4.0
+	github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084
+	github.com/rcrowley/go-metrics => github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a
+	github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af
+	github.com/rogpeppe/go-charset => github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4
+	github.com/russross/blackfriday => github.com/russross/blackfriday v1.5.2
+	github.com/satori/go.uuid => github.com/satori/go.uuid v1.2.0
+	github.com/sergi/go-diff => github.com/sergi/go-diff v1.0.0
+	github.com/shirou/gopsutil => github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7
+	github.com/shirou/w32 => github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4
+	github.com/sirupsen/logrus => github.com/sirupsen/logrus v1.2.0
+	github.com/soheilhy/cmux => github.com/soheilhy/cmux v0.1.4
+	github.com/sony/sonyflake => github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009
+	github.com/speps/go-hashids => github.com/speps/go-hashids v2.0.0+incompatible
+	github.com/spf13/afero => github.com/spf13/afero v1.2.2
+	github.com/spf13/cast => github.com/spf13/cast v1.3.0
+	github.com/spf13/cobra => github.com/spf13/cobra v0.0.3
+	github.com/spf13/jwalterweatherman => github.com/spf13/jwalterweatherman v1.0.0
+	github.com/spf13/pflag => github.com/spf13/pflag v1.0.3
+	github.com/spf13/viper => github.com/spf13/viper v1.4.0
+	github.com/src-d/gcfg => github.com/src-d/gcfg v1.4.0
+	github.com/stretchr/objx => github.com/stretchr/objx v0.1.1
+	github.com/stretchr/testify => github.com/stretchr/testify v1.3.0
+	github.com/tmc/grpc-websocket-proxy => github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5
+	github.com/ugorji/go => github.com/ugorji/go v1.1.4
+	github.com/urfave/cli => github.com/urfave/cli v1.20.0
+	github.com/xanzy/ssh-agent => github.com/xanzy/ssh-agent v0.2.1
+	github.com/xiang90/probing => github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
+	github.com/xlab/treeprint => github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6
+	github.com/xordataexchange/crypt => github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77
+	go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.3
+	go.opencensus.io => go.opencensus.io v0.20.2
+	go.uber.org/atomic => go.uber.org/atomic v1.4.0
+	go.uber.org/multierr => go.uber.org/multierr v1.1.0
+	go.uber.org/zap => go.uber.org/zap v1.10.0
+	golang.org/x/crypto => golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+	golang.org/x/exp => golang.org/x/exp v0.0.0-20190121172915-509febef88a4
+	golang.org/x/lint => golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f
+	golang.org/x/net => golang.org/x/net v0.0.0-20190620200207-3b0461eec859
+	golang.org/x/oauth2 => golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a
+	golang.org/x/sync => golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys => golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e
+	golang.org/x/text => golang.org/x/text v0.3.0
+	golang.org/x/time => golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
+	golang.org/x/tools => golang.org/x/tools v0.0.0-20190710153321-831012c29e42
+	google.golang.org/api => google.golang.org/api v0.3.1
+	google.golang.org/appengine => google.golang.org/appengine v1.5.0
+	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7
+	google.golang.org/grpc => google.golang.org/grpc v1.19.1
+	gopkg.in/alecthomas/kingpin.v2 => gopkg.in/alecthomas/kingpin.v2 v2.2.6
+	gopkg.in/asn1-ber.v1 => gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d
+	gopkg.in/check.v1 => gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127
+	gopkg.in/fsnotify.v1 => gopkg.in/fsnotify.v1 v1.4.7
+	gopkg.in/go-playground/assert.v1 => gopkg.in/go-playground/assert.v1 v1.2.1
+	gopkg.in/go-playground/validator.v9 => gopkg.in/go-playground/validator.v9 v9.29.1
+	gopkg.in/inf.v0 => gopkg.in/inf.v0 v0.9.1
+	gopkg.in/mcuadros/go-syslog.v2 => gopkg.in/mcuadros/go-syslog.v2 v2.2.1
+	gopkg.in/natefinch/lumberjack.v2 => gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	gopkg.in/resty.v1 => gopkg.in/resty.v1 v1.12.0
+	gopkg.in/square/go-jose.v2 => gopkg.in/square/go-jose.v2 v2.3.1
+	gopkg.in/src-d/go-billy.v4 => gopkg.in/src-d/go-billy.v4 v4.3.0
+	gopkg.in/src-d/go-git-fixtures.v3 => gopkg.in/src-d/go-git-fixtures.v3 v3.1.1
+	gopkg.in/src-d/go-git.v4 => gopkg.in/src-d/go-git.v4 v4.11.0
+	gopkg.in/tomb.v1 => gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7
+	gopkg.in/warnings.v0 => gopkg.in/warnings.v0 v0.1.2
+	gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.2.2
+	gotest.tools => gotest.tools v2.2.0+incompatible
+	honnef.co/go/tools => honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099
+	k8s.io/api => k8s.io/api v0.0.0-20181213150558-05914d821849
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476
+	k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
+	k8s.io/apiserver => k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d
+	k8s.io/client-go => k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31
+	k8s.io/code-generator => k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b
+	k8s.io/component-base => k8s.io/component-base v0.0.0-20190831075413-37a093468564
+	k8s.io/gengo => k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a
+	k8s.io/klog => k8s.io/klog v0.4.0
+	k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5
+	k8s.io/kubernetes => k8s.io/kubernetes v1.13.6
+	k8s.io/utils => k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5
+	kubesphere.io/im => kubesphere.io/im v0.1.0
+	openpitrix.io/iam => openpitrix.io/iam v0.1.0
+	openpitrix.io/logger v0.1.0 => github.com/openpitrix/logger v0.1.0
+	openpitrix.io/openpitrix => openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c
+	rsc.io/goversion => rsc.io/goversion v1.0.0
+	sigs.k8s.io/application => github.com/kubesphere/application v0.0.0-20200221140547-8beafe2fa7ef
+	sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.1.10
+	sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.1.12
+	sigs.k8s.io/testing_frameworks => sigs.k8s.io/testing_frameworks v0.1.1
+	sigs.k8s.io/yaml => sigs.k8s.io/yaml v1.1.0
+)

go.sum

@@ -0,0 +1,522 @@
+bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c h1:t+Ra932MCC0eeyD/vigXqMbZTzgZjd4JOfBJWC6VSMI=
+bitbucket.org/ww/goautoneg v0.0.0-20120707110453-75cd24fc2f2c/go.mod h1:1vhO7Mn/FZMgOgDVGLy5X1mE6rq1HbkBdkF/yj8zkcg=
+cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6 h1:tW+ztA4A9UT9xnco5wUjW1oNi35k22eUEn9tNpPYVwE=
+code.cloudfoundry.org/bytefmt v0.0.0-20190710193110-1eb035ffe2b6/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
+contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc=
+contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest/autorest v0.5.0 h1:Mlm9qy2fpQ9MvfyI41G2Zf5B4CsgjjNbLOWszfK6KrY=
+github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw=
+github.com/Azure/go-autorest/autorest/adal v0.2.0 h1:7IBDu1jgh+ADHXnEYExkV9RE/ztOOlxdACkkPRthGKw=
+github.com/Azure/go-autorest/autorest/adal v0.2.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E=
+github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0 h1:Kx+AUU2Te+A3JIyYn6Dfs+cFgx5XorQKuIXrZGoq/SI=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.1.0 h1:TRBxC5Pj/fIuh4Qob0ZpkggbfT8RC0SubHbpV3p4/Vc=
+github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww=
+github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
+github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiUOryc=
+github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
+github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
+github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk=
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/purell v1.1.0 h1:rmGxhojJlM0tuKtfdvliR84CFHljx9ag64t2xmVkjK4=
+github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
+github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
+github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/StackExchange/wmi v0.0.0-20170410192909-ea383cf3ba6e/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/andybalholm/cascadia v1.0.0 h1:hOCXnnZ5A+3eVDX8pvgl4kofXv2ELss0bKcqRySc45o=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSepE2OqVu0Pn1CbDw9IuMlONapol0zuwk=
+github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f h1:y2hSFdXeA1y5z5f0vfNO0Dg5qVY036qzlz3Pds0B92o=
+github.com/asaskevich/govalidator v0.0.0-20180315120708-ccb8e960c48f/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.22.2 h1:uYP58k2Cd9y1qBy8CxTe5ADmdi4kANm8Ul8ch3kkIcQ=
+github.com/aws/aws-sdk-go v1.22.2/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs=
+github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A=
+github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
+github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
+github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
+github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
+github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.13+incompatible h1:8F3hqu9fGYLBifCmRCJsicFqDx/D68Rt3q1JMazcgBQ=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c=
+github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set v1.7.1 h1:SCQV0S6gTtp6itiFrTqI+pfmJ4LN85S1YzhDf9rTHJQ=
+github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ=
+github.com/denisenkom/go-mssqldb v0.0.0-20190204142019-df6d76eb9289/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4 h1:+VAGRKyn9Ca+ckzV/PJsaRO7UXO9KQjFmSffcSDrWdE=
+github.com/docker/engine v1.4.2-0.20190822205725-ed20165a37b4/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY=
+github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o=
+github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
+github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s=
+github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
+github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
+github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
+github.com/elastic/go-elasticsearch/v5 v5.6.1 h1:RnL2wcXepOT5SdoKMMO1j1OBX0vxHYbBtkQNL2E3xs4=
+github.com/elastic/go-elasticsearch/v5 v5.6.1/go.mod h1:r7uV7HidpfkYh7D8SB4lkS13TNlNy3oa5GNmTZvuVqY=
+github.com/elastic/go-elasticsearch/v6 v6.8.2 h1:rp5DGrd63V5c6nHLjF6QEXUpZSvs0+QM3ld7m9VhV2g=
+github.com/elastic/go-elasticsearch/v6 v6.8.2/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI=
+github.com/elastic/go-elasticsearch/v7 v7.3.0 h1:H29Nqf9cB9dVxX6LwS+zTDC2D4t9s+8dK8ln4HPS9rw=
+github.com/elastic/go-elasticsearch/v7 v7.3.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
+github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk=
+github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
+github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2 h1:aZtFdDNWY/yH86JPR2WX/PN63635VsE/f/nXNPAbYxY=
+github.com/elazarl/goproxy v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2 h1:dWB6v3RcOy03t/bUadywsbyrQwCqZeNIEX6M1OtSZOM=
+github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8=
+github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
+github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/emicklei/go-restful-openapi v1.0.0 h1:ZFk3RuCl8ZmG1yUAF/mSbXRi5cuyA/k5+EpHayuuTXM=
+github.com/emicklei/go-restful-openapi v1.0.0/go.mod h1:Q+bHVYfUWv1fvC4FNTsz2AVvFSsXAC7RCiWjF1Sva1A=
+github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6 h1:V94anc0ZG3Pa/cAMwP2m1aQW3+/FF8Qmw/GsFyTJAp4=
+github.com/emicklei/go-restful-swagger12 v0.0.0-20170926063155-7524189396c6/go.mod h1:qr0VowGBT4CS4Q8vFF8BSeKz34PuqKGxs/L0IAQA9DQ=
+github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
+github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/evanphx/json-patch v4.2.0+incompatible h1:fUDGZCv/7iAN7u0puUVhvKCcsR6vRfwrJatElLBEf0I=
+github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
+github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
+github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
+github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.1.1 h1:j3L6gSLQalDETeEg/Jg0mGY0/y/N6zI2xX1978P0Uqw=
+github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb h1:D4uzjWwKYQ5XnAvUbuvHW93esHg7F8N/OYeBBcJoTr0=
+github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
+github.com/go-acme/lego v2.5.0+incompatible h1:5fNN9yRQfv8ymH3DSsxla+4aYeQt2IgfZqHKVnK8f0s=
+github.com/go-acme/lego v2.5.0+incompatible/go.mod h1:yzMNe9CasVUhkquNvti5nAtPmG94USbYxYrZfTkIn0M=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
+github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/zapr v0.1.1 h1:qXBXPDdNncunGs7XeEpsJt8wCjYBygluzfdLO0G5baE=
+github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-openapi/errors v0.17.0 h1:g5DzIh94VpuR/dd6Ff8KqyHNnw7yBa2xSHIPPzjRDUo=
+github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
+github.com/go-openapi/jsonpointer v0.19.0 h1:FTUMcX77w5rQkClIzDtTxvn6Bsa894CcrzNj2MMfeg8=
+github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
+github.com/go-openapi/jsonreference v0.19.0 h1:BqWKpV1dFd+AuiKlgtddwVIFQsuMpxfBDBHGfM2yNpk=
+github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
+github.com/go-openapi/spec v0.19.0 h1:A4SZ6IWh3lnjH0rG0Z5lkxazMGBECtrZcbyYQi+64k4=
+github.com/go-openapi/spec v0.19.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
+github.com/go-openapi/strfmt v0.19.0 h1:0Dn9qy1G9+UJfRU7TR8bmdGxb4uifB7HNrJjOnV0yPk=
+github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
+github.com/go-openapi/swag v0.19.0 h1:Kg7Wl7LkTPlmc393QZQ/5rQadPhi7pBVEMZxyTi0Ii8=
+github.com/go-openapi/swag v0.19.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
+github.com/go-playground/locales v0.12.1 h1:2FITxuFt/xuCNP1Acdhv62OzaCiviiE4kotfhkmOqEc=
+github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
+github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM=
+github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
+github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
+github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobuffalo/flect v0.1.5 h1:xpKq9ap8MbYfhuPCF0dBH854Gp9CxZjr/IocxELFflo=
+github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80=
+github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 h1:kumyNm8Vr8cbVm/aLQYTbDE3SKCbbn5HEVoDp/Dyyfc=
+github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6/go.mod h1:K/9g3pPouf13kP5K7pdriQEJAy272R9yXuWuDIEWJTM=
+github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
+github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang/example v0.0.0-20170904185048-46695d81d1fa h1:iqCQC2Z53KkwGgTN9szyL4q0OQHmuNjeoNnMT6lk66k=
+github.com/golang/example v0.0.0-20170904185048-46695d81d1fa/go.mod h1:tO/5UvQ/uKigUjQBPqzstj6uxd3fUIjddi19DxGJeWg=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gops v0.3.6/go.mod h1:RZ1rH95wsAGX4vMWKmqBOIWynmWisBf4QFdgT/k/xOI=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g=
+github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/gophercloud/gophercloud v0.3.0 h1:6sjpKIpVwRIIwmcEGp+WwNovNsem+c+2vm6oxshRpL8=
+github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/mux v1.7.1 h1:Dw4jY2nghMMRsh1ol8dv1axHkDwMQK2DHerMNJsIpJU=
+github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q=
+github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
+github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI=
+github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
+github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a h1:BcF8coBl0QFVhe8vAMMlD+CV8EISiu9MGKLoj6ZEyJA=
+github.com/jimstudt/http-authentication v0.0.0-20140401203705-3eca13d6893a/go.mod h1:wK6yTYYcgjHE1Z1QtXACPDjcFJyBskHEdagmnq3vsP8=
+github.com/jinzhu/gorm v1.9.2/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jinzhu/inflection v0.0.0-20180308033659-04140366298a/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.0.0/go.mod h1:oHTiXerJ20+SfYcrdlBO7rzZRJWGwSTQ0iUY2jI6Gfc=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kardianos/osext v0.0.0-20170510131534-ae77be60afb1/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
+github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8=
+github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/keybase/go-ps v0.0.0-20161005175911-668c8856d999/go.mod h1:hY+WOq6m2FpbvyrI93sMaypsttvaIL5nhVR92dTMUcQ=
+github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/knative/pkg v0.0.0-20190314204845-cd278f2d3394 h1:Lg2DviikLeZmY0rnPpVLMC77h7vqZG5mjh33apZl76o=
+github.com/knative/pkg v0.0.0-20190314204845-cd278f2d3394/go.mod h1:7Ijfhw7rfB+H9VtosIsDYvZQ+qYTz7auK3fHW/5z4ww=
+github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7 h1:SWlt7BoQNASbhTUD0Oy5yysI2seJ7vWuGUp///OM4TM=
+github.com/koding/multiconfig v0.0.0-20171124222453-69c27309b2d7/go.mod h1:Y2SaZf2Rzd0pXkLVhLlCiAXFCLSXAIbTKDivVgff/AM=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kubernetes-sigs/application v0.8.1 h1:tnIN+iijr0kDJxvOhnqjoXV+3hWydRNy8OHisMyO71U=
+github.com/kubernetes-sigs/application v0.8.1/go.mod h1:KqAScqo78PXUrmoFOH8z6P4xQDBdzyJH1FMqrhgJJLE=
+github.com/kubesphere/application v0.0.0-20190518133311-b9d9eb0b5cf7 h1:9dongD6mbssQJGb9pV4sE7IHyua6I/lCOu4h6rEfj20=
+github.com/kubesphere/application v0.0.0-20190518133311-b9d9eb0b5cf7/go.mod h1:xCs7b2bgA24oBSuZYf+5btESJC3xPs//ZTSK1ql+W6I=
+github.com/kubesphere/application v0.0.0-20200221140547-8beafe2fa7ef h1:0s/3VfJ9xP9cqLB7dKj1eXCfC+Nr8fy/5xUJhD2lojU=
+github.com/kubesphere/application v0.0.0-20200221140547-8beafe2fa7ef/go.mod h1:Sn/bPGEhZxJeByRvkBo3I+n343KJ+5PBbhdmCdoJZX8=
+github.com/kubesphere/kiali v0.15.1-0.20200213040359-608aece2aa66 h1:xrYcJkIEsE6RDmBprHn1SSMOoYcew/SAxJPm2+rHg1s=
+github.com/kubesphere/kiali v0.15.1-0.20200213040359-608aece2aa66/go.mod h1:Y1EqeixoXkKkU8I+yvOfhdh21+8+etFE6wYOVT2XFdI=
+github.com/kubesphere/s2ioperator v0.0.13 h1:K6RdjaFluhn/GterbnIykORrueAZcwR/Qj3MsVI4qQs=
+github.com/kubesphere/s2ioperator v0.0.13/go.mod h1:dv9L+zRYDlHvnKPp0j6VHRtlGB1BU+lloltW9SAWqVU=
+github.com/kubesphere/sonargo v0.0.2 h1:hsSRE3sv3mkPcUAeSABdp7rtfcNW2zzeHXzFa01CTkU=
+github.com/kubesphere/sonargo v0.0.2/go.mod h1:ww8n9ANlDXhX5PBZ18iaRnCgEkXN0GMml3/KZXOZ11w=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
+github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lucas-clemente/quic-go v0.11.1 h1:zasajC848Dqq/+WqfqBCkmPw+YHNe1MBts/z7y7nXf4=
+github.com/lucas-clemente/quic-go v0.11.1/go.mod h1:PpMmPfPKO9nKJ/psF49ESTAGQSdfXxlg1otPbEB2nOw=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic=
+github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/marten-seemann/qtls v0.2.3 h1:0yWJ43C62LsZt08vuQJDK1uC1czUc3FJeCLPoNAI4vA=
+github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mholt/caddy v1.0.0 h1:KI6RPGih2GFzWRPG8s9clKK28Ns4ZlVMKR/v7mxq6+c=
+github.com/mholt/caddy v1.0.0/go.mod h1:PzUpQ3yGCTuEuy0KSxEeB4TZOi3zBZ8BR/zY0RBP414=
+github.com/mholt/certmagic v0.5.1 h1:8Pf6Hwwlh5sbT3nwn3ovXyXWxHCEM54wvfLzTrQ+UiM=
+github.com/mholt/certmagic v0.5.1/go.mod h1:g4cOPxcjV0oFq3qwpjSA30LReKD8AoIfwAY9VvG35NY=
+github.com/miekg/dns v1.1.9 h1:OIdC9wT96RzuZMf2PfKRhFgsStHUUBZLM/lo1LqiM9E=
+github.com/miekg/dns v1.1.9/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8dQu6DMTwH4oIuGN8GJDAlqDdVE=
+github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/openpitrix/logger v0.1.0/go.mod h1:SV8Btt2cTSmeL9H/1XCkYmQ+WQ2upVY4e0wlr07RP28=
+github.com/openshift/api v3.9.0+incompatible h1:fJ/KsefYuZAjmrr3+5U9yZIZbTOpVkDDLDLFresAeYs=
+github.com/openshift/api v3.9.0+incompatible/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY=
+github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 h1:zNBQb37RGLmJybyMcs983HfUfpkw9OTFD9tbBfAViHE=
+github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34=
+github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA=
+github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba h1:aaF2byUCZhzszHsfPEr2M3qcU4ibtD/yk/il2R7T1PU=
+github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba/go.mod h1:q8EdCgBdMQzgiX/uk4GXLWLk+gIHd1a7mWUAamJKDb4=
+github.com/projectcalico/go-yaml v0.0.0-20161201183616-955bc3e451ef h1:Di9BaA9apb6DEstin8RdhKmlzQG76UMbmjPzjCVkMpc=
+github.com/projectcalico/go-yaml v0.0.0-20161201183616-955bc3e451ef/go.mod h1:1Ra2BftSa7Go38Gbq1q0bfmBFSSgUv+Cdc3SY8IL/C0=
+github.com/projectcalico/go-yaml-wrapper v0.0.0-20161127220527-598e54215bee h1:yVWsNSlAuYoJ0CznHsYRPiFgsotoj07k00k5rQvGlHM=
+github.com/projectcalico/go-yaml-wrapper v0.0.0-20161127220527-598e54215bee/go.mod h1:UgC0aTQ2KMDxlX3lU/stndk7DMUBJqzN40yFiILHgxc=
+github.com/projectcalico/libcalico-go v0.0.0-20190708183129-ac36d966132f h1:ccdS7T4NhdlHx8nXe6GiS7TAJUg6Gu/qEDJf1IJvcy8=
+github.com/projectcalico/libcalico-go v0.0.0-20190708183129-ac36d966132f/go.mod h1:0b/n/rPzNXjhn4ywFcEJuQdA/5olt9UxFIATz57xkbc=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA=
+github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.0 h1:7etb9YClo3a6HjLzfl6rIQaU+FDfi0VSX39io3aQ+DM=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
+github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009 h1:3wBL/e/qjpSYaXacpbIV+Bsj/nwQ4UO1llG/av54zzw=
+github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009/go.mod h1:dVvZuWJd174umvm5g8CmZD6S2GWwHKtpK/0ZPHswuNo=
+github.com/speps/go-hashids v2.0.0+incompatible h1:kSfxGfESueJKTx0mpER9Y/1XHl+FVQjtCqRyYcviFbw=
+github.com/speps/go-hashids v2.0.0+incompatible/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8=
+github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.4.0 h1:yXHLWeravcrgGyFSyCgdYpXQ9dR9c/WED3pg1RhxqEU=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
+github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
+github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
+github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk=
+go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.opencensus.io v0.20.2 h1:NAfh7zF0/3/HqtMvJNZ/RFrSlCE6ZTlHmKfhL/Dm1Jk=
+go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e h1:ZytStCyV048ZqDsWHiYDdoI2Vd4msMcrDECFxS+tL9c=
+golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20190710153321-831012c29e42 h1:4IOeC7p+OItq3+O5BWkcmVu2uBe3jekXau5S4QZX9DU=
+golang.org/x/tools v0.0.0-20190710153321-831012c29e42/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+google.golang.org/api v0.3.1 h1:oJra/lMfmtm13/rgY/8i3MzjFWYXvQIAKjQ3HqofMk8=
+google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.1 h1:TrBcJ1yqAl1G++wO39nD/qtgpsW9/1+QGrluyMGEYgM=
+google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v9 v9.29.1 h1:SvGtYmN60a5CVKTOzMSyfzWDeZRxRuGvRQyEAKbw1xc=
+gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
+gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/mcuadros/go-syslog.v2 v2.2.1 h1:60g8zx1BijSVSgLTzLCW9UC4/+i1Ih9jJ1DR5Tgp9vE=
+gopkg.in/mcuadros/go-syslog.v2 v2.2.1/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=
+gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek=
+gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk=
+gopkg.in/src-d/go-git-fixtures.v3 v3.1.1 h1:XWW/s5W18RaJpmo1l0IYGqXKuJITWRFuA45iOf1dKJs=
+gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
+gopkg.in/src-d/go-git.v4 v4.11.0 h1:cJwWgJ0DXifrNrXM6RGN1Y2yR60Rr1zQ9Q5DX5S9qgU=
+gopkg.in/src-d/go-git.v4 v4.11.0/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
+gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+k8s.io/api v0.0.0-20181213150558-05914d821849 h1:WZFcFPXmLR7g5CxQNmjWv0mg8qulJLxDghbzS4pQtzY=
+k8s.io/api v0.0.0-20181213150558-05914d821849/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA=
+k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476 h1:Ws9zfxsgV19Durts9ftyTG7TO0A/QLhmu98VqNWLiH8=
+k8s.io/apiextensions-apiserver v0.0.0-20181213153335-0fe22c71c476/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE=
+k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628 h1:UYfHH+KEF88OTg+GojQUwFTNxbxwmoktLwutUzR0GPg=
+k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0=
+k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d h1:tqGFwR5+/q757PJeccoTuGPAsscNEVdR32SLGWBqipc=
+k8s.io/apiserver v0.0.0-20190507070644-e9c02aff496d/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w=
+k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31 h1:OH3z6khCtxnJBAc0C5CMYWLl1CoK5R5fngX7wrwdN5c=
+k8s.io/client-go v0.0.0-20181213151034-8d9ed539ba31/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s=
+k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b h1:KH0fUlgdFZH8UMxJ/FDCYHpczfSQKefetq5NjL6BVF0=
+k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8=
+k8s.io/component-base v0.0.0-20190831075413-37a093468564 h1:mY4AxuX1h/hbjrwVkBBiTGnWeh41YGfEcFIFGb9Iabs=
+k8s.io/component-base v0.0.0-20190831075413-37a093468564/go.mod h1:pB3zmhcOR5xextKMKdxRr2XUCERS2UNFA/6Tr2WmSJs=
+k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a h1:QoHVuRquf80YZ+/bovwxoMO3Q/A3nt3yTgS0/0nejuk=
+k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.4.0 h1:lCJCxf/LIowc2IGS9TPjWDyXY4nOmdGdfcwwDQCOURQ=
+k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5 h1:MH8SvyTlIiLt8b1oHy4Dtp1zPpLGp6lTOjvfzPTkoQE=
+k8s.io/kube-openapi v0.0.0-20181109181836-c59034cc13d5/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
+k8s.io/kubernetes v1.13.6 h1:eUAUryzMLFmi4ZY8kMOUtLG5lHp2PUx5WOmy4RVaobk=
+k8s.io/kubernetes v1.13.6/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
+k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5 h1:VBM/0P5TWxwk+Nw6Z+lAw3DKgO76g90ETOiA6rfLV1Y=
+k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
+kubesphere.io/im v0.1.0 h1:Isu/WBOawUb4fzSlQeD1f6Vbq9pqFS0PmDg8v8iFYaY=
+kubesphere.io/im v0.1.0/go.mod h1:DHJj/JngMUFyaXecLjBPXj/zk5Oi7ifIixLRp0qJkyA=
+openpitrix.io/iam v0.1.0 h1:cb1mCusim7EGeoXEfuaVa1m7Co/pzim3keoxxKdv944=
+openpitrix.io/iam v0.1.0/go.mod h1:EcZE8CPBg+1fEKCDEhpsIZ8isWWO7javpu84mSqoVn4=
+openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c h1:r/wQNNzFE0Blbf42Dl8pRNSRc+YiYdgOclNcP3VMeTs=
+openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c/go.mod h1:8rZSFeUp2Np5kfKAXfRL/1HSh6BYywNCALt+ZnxFQ/4=
+rsc.io/goversion v1.0.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo=
+sigs.k8s.io/application v0.0.0-20190404151855-67ae7f915d4e h1:/TWUhUxC+Q5uMFUizxYzNAZjwbjlYXOsfnmSC2WpyuI=
+sigs.k8s.io/application v0.0.0-20190404151855-67ae7f915d4e/go.mod h1:9C86g0wiFn8jtZjgJepSx188uJeWLGWTbcCycu5p8mU=
+sigs.k8s.io/controller-runtime v0.1.10 h1:amLOmcekVdnsD1uIpmgRqfTbQWJ2qxvQkcdeFhcotn4=
+sigs.k8s.io/controller-runtime v0.1.10/go.mod h1:HFAYoOh6XMV+jKF1UjFwrknPbowfyHEHHRdJMf2jMX8=
+sigs.k8s.io/controller-tools v0.1.12 h1:LW8Tfywz+epjYiySSOYWFQl1O1y0os+ZWf22XJmsFww=
+sigs.k8s.io/controller-tools v0.1.12/go.mod h1:6g08p9m9G/So3sBc1AOQifHfhxH/mb6Sc4z0LMI8XMw=
+sigs.k8s.io/testing_frameworks v0.1.1 h1:cP2l8fkA3O9vekpy5Ks8mmA0NW/F7yBdXf8brkWhVrs=
+sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U=
+sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=

hack/boilerplate.go.txt

@@ -12,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-*/
+*/

hack/docker_build.sh

@@ -7,19 +7,22 @@ REPO=kubespheredev
 TAG=latest
 
 # check if build was triggered by a travis cronjob
-if [[ ! -v TRAVIS_EVENT_TYPE ]]; then
-    echo "TRAVIS_EVENT_TYPE is not set, treat as regular build"
-elif [[ -z "$TRAVIS_EVENT_TYPE" ]]; then
+if [[ -z "$TRAVIS_EVENT_TYPE" ]]; then
     echo "TRAVIS_EVENT_TYPE is empty, also normaly build"
-elif [ $TRAVIS_EVENT_TYPE == "cron" ]; then
+elif [[ $TRAVIS_EVENT_TYPE == "cron" ]]; then
     TAG=dev-$(date +%Y%m%d)
 fi
 
+if [[ $TRAVIS_BRANCH == "release-2.1" ]]; then
+    TAG=release-2.1
+fi
+
 
 docker build -f build/ks-apigateway/Dockerfile -t $REPO/ks-apigateway:$TAG .
 docker build -f build/ks-apiserver/Dockerfile -t $REPO/ks-apiserver:$TAG .
 docker build -f build/ks-iam/Dockerfile -t $REPO/ks-account:$TAG .
 docker build -f build/ks-controller-manager/Dockerfile -t $REPO/ks-controller-manager:$TAG .
+docker build -f build/hypersphere/Dockerfile -t $REPO/hypersphere:$TAG .
 docker build -f ./pkg/db/Dockerfile -t $REPO/ks-devops:flyway-$TAG ./pkg/db/
 
 # Push image to dockerhub, need to support multiple push
@@ -29,4 +32,5 @@ docker push $REPO/ks-apigateway:$TAG
 docker push $REPO/ks-apiserver:$TAG
 docker push $REPO/ks-account:$TAG
 docker push $REPO/ks-controller-manager:$TAG
+docker push $REPO/hypersphere:$TAG
 docker push $REPO/ks-devops:flyway-$TAG

hack/generate_client.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -e
+
+GV="network:v1alpha1 servicemesh:v1alpha2 tenant:v1alpha1 devops:v1alpha1"
+
+rm -rf ./pkg/client
+./hack/generate_group.sh "client,lister,informer" kubesphere.io/kubesphere/pkg/client kubesphere.io/kubesphere/pkg/apis "$GV" --output-base=./  -h "$PWD/hack/boilerplate.go.txt"
+mv kubesphere.io/kubesphere/pkg/client ./pkg/
+rm -rf ./kubesphere.io

hack/generate_group.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# Copyright 2017 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+GOPATH=`go env GOPATH`
+# generate-groups generates everything for a project with external types only, e.g. a project based
+# on CustomResourceDefinitions.
+
+if [ "$#" -lt 4 ] || [ "${1}" == "--help" ]; then
+  cat <<EOF
+Usage: $(basename "$0") <generators> <output-package> <apis-package> <groups-versions> ...
+
+  <generators>        the generators comma separated to run (deepcopy,defaulter,client,lister,informer) or "all".
+  <output-package>    the output package name (e.g. github.com/example/project/pkg/generated).
+  <apis-package>      the external types dir (e.g. github.com/example/api or github.com/example/project/pkg/apis).
+  <groups-versions>   the groups and their versions in the format "groupA:v1,v2 groupB:v1 groupC:v2", relative
+                      to <api-package>.
+  ...                 arbitrary flags passed to all generator binaries.
+
+
+Examples:
+  $(basename "$0") all             github.com/example/project/pkg/client github.com/example/project/pkg/apis "foo:v1 bar:v1alpha1,v1beta1"
+  $(basename "$0") deepcopy,client github.com/example/project/pkg/client github.com/example/project/pkg/apis "foo:v1 bar:v1alpha1,v1beta1"
+EOF
+  exit 0
+fi
+
+GENS="$1"
+OUTPUT_PKG="$2"
+APIS_PKG="$3"
+GROUPS_WITH_VERSIONS="$4"
+shift 4
+
+
+GO111MODULE=on go install -mod=vendor k8s.io/code-generator/cmd/{client-gen,lister-gen,informer-gen}
+
+
+function codegen::join() { local IFS="$1"; shift; echo "$*"; }
+
+# enumerate group versions
+FQ_APIS=() # e.g. k8s.io/api/apps/v1
+for GVs in ${GROUPS_WITH_VERSIONS}; do
+  IFS=: read -r G Vs <<<"${GVs}"
+
+  # enumerate versions
+  for V in ${Vs//,/ }; do
+    FQ_APIS+=("${APIS_PKG}/${G}/${V}")
+  done
+done
+
+if [ "${GENS}" = "all" ] || grep -qw "deepcopy" <<<"${GENS}"; then
+  echo "Generating deepcopy funcs"
+  ${GOPATH}/bin/deepcopy-gen --input-dirs $(codegen::join , "${FQ_APIS[@]}") -O zz_generated.deepcopy --bounding-dirs ${APIS_PKG} "$@"
+fi
+
+if [ "${GENS}" = "all" ] || grep -qw "client" <<<"${GENS}"; then
+  echo "Generating clientset for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}"
+  ${GOPATH}/bin/client-gen --clientset-name ${CLIENTSET_NAME_VERSIONED:-versioned} --input-base "" --input $(codegen::join , "${FQ_APIS[@]}") --output-package ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset} "$@"
+fi
+
+if [ "${GENS}" = "all" ] || grep -qw "lister" <<<"${GENS}"; then
+  echo "Generating listers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/listers"
+  ${GOPATH}/bin/lister-gen --input-dirs $(codegen::join , "${FQ_APIS[@]}") --output-package ${OUTPUT_PKG}/listers "$@"
+fi
+
+if [ "${GENS}" = "all" ] || grep -qw "informer" <<<"${GENS}"; then
+  echo "Generating informers for ${GROUPS_WITH_VERSIONS} at ${OUTPUT_PKG}/informers"
+  ${GOPATH}/bin/informer-gen \
+           --input-dirs $(codegen::join , "${FQ_APIS[@]}") \
+           --versioned-clientset-package ${OUTPUT_PKG}/${CLIENTSET_PKG_NAME:-clientset}/${CLIENTSET_NAME_VERSIONED:-versioned} \
+           --listers-package ${OUTPUT_PKG}/listers \
+           --output-package ${OUTPUT_PKG}/informers \
+           "$@"
+fi

hack/gobuild.sh

@@ -40,6 +40,6 @@ BUILD_GOARCH=${GOARCH:-amd64}
 GOBINARY=${GOBINARY:-go}
 
 # forgoing -i (incremental build) because it will be deprecated by tool chain.
-time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \
+time GOOS=${BUILD_GOOS} CGO_ENABLED=0 GOARCH=${BUILD_GOARCH} ${GOBINARY} build \
         -o ${OUT} \
         ${BUILDPATH}

hack/lib/golang.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# This is a modified version of Kubernetes
+
+
+# Ensure the go tool exists and is a viable version.
+kube::golang::verify_go_version() {
+  if [[ -z "$(command -v go)" ]]; then
+    kube::log::usage_from_stdin <<EOF
+Can't find 'go' in PATH, please fix and retry.
+See http://golang.org/doc/install for installation instructions.
+EOF
+    return 2
+  fi
+
+  local go_version
+  IFS=" " read -ra go_version <<< "$(go version)"
+  local minimum_go_version
+  minimum_go_version=go1.12.1
+  if [[ "${minimum_go_version}" != $(echo -e "${minimum_go_version}\n${go_version[2]}" | sort -s -t. -k 1,1 -k 2,2n -k 3,3n | head -n1) && "${go_version[2]}" != "devel" ]]; then
+    kube::log::usage_from_stdin <<EOF
+Detected go version: ${go_version[*]}.
+Kubernetes requires ${minimum_go_version} or greater.
+Please install ${minimum_go_version} or later.
+EOF
+    return 2
+  fi
+}

hack/lib/init.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+
+# This script is modified version of Kubernetes script
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export GO111MODULE=auto
+
+# The root of the build/dist directory
+KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+
+KUBE_OUTPUT_SUBPATH="${KUBE_OUTPUT_SUBPATH:-_output/local}"
+KUBE_OUTPUT="${KUBE_ROOT}/${KUBE_OUTPUT_SUBPATH}"
+KUBE_OUTPUT_BINPATH="${KUBE_OUTPUT}/bin"
+
+export THIS_PLATFORM_BIN="${KUBE_ROOT}/_output/bin"
+
+source "${KUBE_ROOT}/hack/lib/util.sh"
+source "${KUBE_ROOT}/hack/lib/logging.sh"
+
+kube::log::install_errexit
+
+source "${KUBE_ROOT}/hack/lib/golang.sh"
+
+KUBE_OUTPUT_HOSTBIN="${KUBE_OUTPUT_BINPATH}/$(kube::util::host_platform)"
+export KUBE_OUTPUT_HOSTBIN
+
+
+# This emulates "readlink -f" which is not available on MacOS X.
+# Test:
+# T=/tmp/$$.$RANDOM
+# mkdir $T
+# touch $T/file
+# mkdir $T/dir
+# ln -s $T/file $T/linkfile
+# ln -s $T/dir $T/linkdir
+# function testone() {
+#   X=$(readlink -f $1 2>&1)
+#   Y=$(kube::readlinkdashf $1 2>&1)
+#   if [ "$X" != "$Y" ]; then
+#     echo readlinkdashf $1: expected "$X", got "$Y"
+#   fi
+# }
+# testone /
+# testone /tmp
+# testone $T
+# testone $T/file
+# testone $T/dir
+# testone $T/linkfile
+# testone $T/linkdir
+# testone $T/nonexistant
+# testone $T/linkdir/file
+# testone $T/linkdir/dir
+# testone $T/linkdir/linkfile
+# testone $T/linkdir/linkdir
+function kube::readlinkdashf {
+  # run in a subshell for simpler 'cd'
+  (
+    if [[ -d "${1}" ]]; then # This also catch symlinks to dirs.
+      cd "${1}"
+      pwd -P
+    else
+      cd "$(dirname "${1}")"
+      local f
+      f=$(basename "${1}")
+      if [[ -L "${f}" ]]; then
+        readlink "${f}"
+      else
+        echo "$(pwd -P)/${f}"
+      fi
+    fi
+  )
+}
+
+# This emulates "realpath" which is not available on MacOS X
+# Test:
+# T=/tmp/$$.$RANDOM
+# mkdir $T
+# touch $T/file
+# mkdir $T/dir
+# ln -s $T/file $T/linkfile
+# ln -s $T/dir $T/linkdir
+# function testone() {
+#   X=$(realpath $1 2>&1)
+#   Y=$(kube::realpath $1 2>&1)
+#   if [ "$X" != "$Y" ]; then
+#     echo realpath $1: expected "$X", got "$Y"
+#   fi
+# }
+# testone /
+# testone /tmp
+# testone $T
+# testone $T/file
+# testone $T/dir
+# testone $T/linkfile
+# testone $T/linkdir
+# testone $T/nonexistant
+# testone $T/linkdir/file
+# testone $T/linkdir/dir
+# testone $T/linkdir/linkfile
+# testone $T/linkdir/linkdir
+kube::realpath() {
+  if [[ ! -e "${1}" ]]; then
+    echo "${1}: No such file or directory" >&2
+    return 1
+  fi
+  kube::readlinkdashf "${1}"
+}

hack/lib/logging.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Controls verbosity of the script output and logging.
+KUBE_VERBOSE="${KUBE_VERBOSE:-5}"
+
+# Handler for when we exit automatically on an error.
+# Borrowed from https://gist.github.com/ahendrix/7030300
+kube::log::errexit() {
+  local err="${PIPESTATUS[*]}"
+
+  # If the shell we are in doesn't have errexit set (common in subshells) then
+  # don't dump stacks.
+  set +o | grep -qe "-o errexit" || return
+
+  set +o xtrace
+  local code="${1:-1}"
+  # Print out the stack trace described by $function_stack
+  if [ ${#FUNCNAME[@]} -gt 2 ]
+  then
+    kube::log::error "Call tree:"
+    for ((i=1;i<${#FUNCNAME[@]}-1;i++))
+    do
+      kube::log::error " ${i}: ${BASH_SOURCE[${i}+1]}:${BASH_LINENO[${i}]} ${FUNCNAME[${i}]}(...)"
+    done
+  fi
+  kube::log::error_exit "Error in ${BASH_SOURCE[1]}:${BASH_LINENO[0]}. '${BASH_COMMAND}' exited with status ${err}" "${1:-1}" 1
+}
+
+kube::log::install_errexit() {
+  # trap ERR to provide an error handler whenever a command exits nonzero  this
+  # is a more verbose version of set -o errexit
+  trap 'kube::log::errexit' ERR
+
+  # setting errtrace allows our ERR trap handler to be propagated to functions,
+  # expansions and subshells
+  set -o errtrace
+}
+
+# Print out the stack trace
+#
+# Args:
+#   $1 The number of stack frames to skip when printing.
+kube::log::stack() {
+  local stack_skip=${1:-0}
+  stack_skip=$((stack_skip + 1))
+  if [[ ${#FUNCNAME[@]} -gt ${stack_skip} ]]; then
+    echo "Call stack:" >&2
+    local i
+    for ((i=1 ; i <= ${#FUNCNAME[@]} - stack_skip ; i++))
+    do
+      local frame_no=$((i - 1 + stack_skip))
+      local source_file=${BASH_SOURCE[${frame_no}]}
+      local source_lineno=${BASH_LINENO[$((frame_no - 1))]}
+      local funcname=${FUNCNAME[${frame_no}]}
+      echo "  ${i}: ${source_file}:${source_lineno} ${funcname}(...)" >&2
+    done
+  fi
+}
+
+# Log an error and exit.
+# Args:
+#   $1 Message to log with the error
+#   $2 The error code to return
+#   $3 The number of stack frames to skip when printing.
+kube::log::error_exit() {
+  local message="${1:-}"
+  local code="${2:-1}"
+  local stack_skip="${3:-0}"
+  stack_skip=$((stack_skip + 1))
+
+  if [[ ${KUBE_VERBOSE} -ge 4 ]]; then
+    local source_file=${BASH_SOURCE[${stack_skip}]}
+    local source_line=${BASH_LINENO[$((stack_skip - 1))]}
+    echo "!!! Error in ${source_file}:${source_line}" >&2
+    [[ -z ${1-} ]] || {
+      echo "  ${1}" >&2
+    }
+
+    kube::log::stack ${stack_skip}
+
+    echo "Exiting with status ${code}" >&2
+  fi
+
+  exit "${code}"
+}
+
+# Log an error but keep going.  Don't dump the stack or exit.
+kube::log::error() {
+  timestamp=$(date +"[%m%d %H:%M:%S]")
+  echo "!!! ${timestamp} ${1-}" >&2
+  shift
+  for message; do
+    echo "    ${message}" >&2
+  done
+}
+
+# Print an usage message to stderr.  The arguments are printed directly.
+kube::log::usage() {
+  echo >&2
+  local message
+  for message; do
+    echo "${message}" >&2
+  done
+  echo >&2
+}
+
+kube::log::usage_from_stdin() {
+  local messages=()
+  while read -r line; do
+    messages+=("${line}")
+  done
+
+  kube::log::usage "${messages[@]}"
+}
+
+# Print out some info that isn't a top level status line
+kube::log::info() {
+  local V="${V:-0}"
+  if [[ ${KUBE_VERBOSE} < ${V} ]]; then
+    return
+  fi
+
+  for message; do
+    echo "${message}"
+  done
+}
+
+# Just like kube::log::info, but no \n, so you can make a progress bar
+kube::log::progress() {
+  for message; do
+    echo -e -n "${message}"
+  done
+}
+
+kube::log::info_from_stdin() {
+  local messages=()
+  while read -r line; do
+    messages+=("${line}")
+  done
+
+  kube::log::info "${messages[@]}"
+}
+
+# Print a status line.  Formatted to show up in a stream of output.
+kube::log::status() {
+  local V="${V:-0}"
+  if [[ ${KUBE_VERBOSE} < ${V} ]]; then
+    return
+  fi
+
+  timestamp=$(date +"[%m%d %H:%M:%S]")
+  echo "+++ ${timestamp} ${1}"
+  shift
+  for message; do
+    echo "    ${message}"
+  done
+}

hack/lib/util.sh

@@ -0,0 +1,765 @@
+#!/usr/bin/env bash
+
+# Copyright 2014 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function kube::util::sourced_variable {
+  # Call this function to tell shellcheck that a variable is supposed to
+  # be used from other calling context. This helps quiet an "unused
+  # variable" warning from shellcheck and also document your code.
+  true
+}
+
+kube::util::sortable_date() {
+  date "+%Y%m%d-%H%M%S"
+}
+
+# arguments: target, item1, item2, item3, ...
+# returns 0 if target is in the given items, 1 otherwise.
+kube::util::array_contains() {
+  local search="$1"
+  local element
+  shift
+  for element; do
+    if [[ "${element}" == "${search}" ]]; then
+      return 0
+     fi
+  done
+  return 1
+}
+
+kube::util::wait_for_url() {
+  local url=$1
+  local prefix=${2:-}
+  local wait=${3:-1}
+  local times=${4:-30}
+  local maxtime=${5:-1}
+
+  command -v curl >/dev/null || {
+    kube::log::usage "curl must be installed"
+    exit 1
+  }
+
+  local i
+  for i in $(seq 1 "${times}"); do
+    local out
+    if out=$(curl --max-time "${maxtime}" -gkfs "${url}" 2>/dev/null); then
+      kube::log::status "On try ${i}, ${prefix}: ${out}"
+      return 0
+    fi
+    sleep "${wait}"
+  done
+  kube::log::error "Timed out waiting for ${prefix} to answer at ${url}; tried ${times} waiting ${wait} between each"
+  return 1
+}
+
+# Example:  kube::util::trap_add 'echo "in trap DEBUG"' DEBUG
+# See: http://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal
+kube::util::trap_add() {
+  local trap_add_cmd
+  trap_add_cmd=$1
+  shift
+
+  for trap_add_name in "$@"; do
+    local existing_cmd
+    local new_cmd
+
+    # Grab the currently defined trap commands for this trap
+    existing_cmd=$(trap -p "${trap_add_name}" |  awk -F"'" '{print $2}')
+
+    if [[ -z "${existing_cmd}" ]]; then
+      new_cmd="${trap_add_cmd}"
+    else
+      new_cmd="${trap_add_cmd};${existing_cmd}"
+    fi
+
+    # Assign the test. Disable the shellcheck warning telling that trap
+    # commands should be single quoted to avoid evaluating them at this
+    # point instead evaluating them at run time. The logic of adding new
+    # commands to a single trap requires them to be evaluated right away.
+    # shellcheck disable=SC2064
+    trap "${new_cmd}" "${trap_add_name}"
+  done
+}
+
+# Opposite of kube::util::ensure-temp-dir()
+kube::util::cleanup-temp-dir() {
+  rm -rf "${KUBE_TEMP}"
+}
+
+# Create a temp dir that'll be deleted at the end of this bash session.
+#
+# Vars set:
+#   KUBE_TEMP
+kube::util::ensure-temp-dir() {
+  if [[ -z ${KUBE_TEMP-} ]]; then
+    KUBE_TEMP=$(mktemp -d 2>/dev/null || mktemp -d -t kubernetes.XXXXXX)
+    kube::util::trap_add kube::util::cleanup-temp-dir EXIT
+  fi
+}
+
+kube::util::host_os() {
+  local host_os
+  case "$(uname -s)" in
+    Darwin)
+      host_os=darwin
+      ;;
+    Linux)
+      host_os=linux
+      ;;
+    *)
+      kube::log::error "Unsupported host OS.  Must be Linux or Mac OS X."
+      exit 1
+      ;;
+  esac
+  echo "${host_os}"
+}
+
+kube::util::host_arch() {
+  local host_arch
+  case "$(uname -m)" in
+    x86_64*)
+      host_arch=amd64
+      ;;
+    i?86_64*)
+      host_arch=amd64
+      ;;
+    amd64*)
+      host_arch=amd64
+      ;;
+    aarch64*)
+      host_arch=arm64
+      ;;
+    arm64*)
+      host_arch=arm64
+      ;;
+    arm*)
+      host_arch=arm
+      ;;
+    i?86*)
+      host_arch=x86
+      ;;
+    s390x*)
+      host_arch=s390x
+      ;;
+    ppc64le*)
+      host_arch=ppc64le
+      ;;
+    *)
+      kube::log::error "Unsupported host arch. Must be x86_64, 386, arm, arm64, s390x or ppc64le."
+      exit 1
+      ;;
+  esac
+  echo "${host_arch}"
+}
+
+# This figures out the host platform without relying on golang.  We need this as
+# we don't want a golang install to be a prerequisite to building yet we need
+# this info to figure out where the final binaries are placed.
+kube::util::host_platform() {
+  echo "$(kube::util::host_os)/$(kube::util::host_arch)"
+}
+
+# looks for $1 in well-known output locations for the platform ($2)
+# $KUBE_ROOT must be set
+kube::util::find-binary-for-platform() {
+  local -r lookfor="$1"
+  local -r platform="$2"
+  local locations=(
+    "${KUBE_ROOT}/_output/bin/${lookfor}"
+    "${KUBE_ROOT}/_output/dockerized/bin/${platform}/${lookfor}"
+    "${KUBE_ROOT}/_output/local/bin/${platform}/${lookfor}"
+    "${KUBE_ROOT}/platforms/${platform}/${lookfor}"
+  )
+  # Also search for binary in bazel build tree.
+  # The bazel go rules place some binaries in subtrees like
+  # "bazel-bin/source/path/linux_amd64_pure_stripped/binaryname", so make sure
+  # the platform name is matched in the path.
+  while IFS=$'\n' read -r location; do
+    locations+=("$location");
+  done < <(find "${KUBE_ROOT}/bazel-bin/" -type f -executable \
+    \( -path "*/${platform/\//_}*/${lookfor}" -o -path "*/${lookfor}" \) 2>/dev/null || true)
+
+  # List most recently-updated location.
+  local -r bin=$( (ls -t "${locations[@]}" 2>/dev/null || true) | head -1 )
+  echo -n "${bin}"
+}
+
+# looks for $1 in well-known output locations for the host platform
+# $KUBE_ROOT must be set
+kube::util::find-binary() {
+  kube::util::find-binary-for-platform "$1" "$(kube::util::host_platform)"
+}
+
+# Run all known doc generators (today gendocs and genman for kubectl)
+# $1 is the directory to put those generated documents
+kube::util::gen-docs() {
+  local dest="$1"
+
+  # Find binary
+  gendocs=$(kube::util::find-binary "gendocs")
+  genkubedocs=$(kube::util::find-binary "genkubedocs")
+  genman=$(kube::util::find-binary "genman")
+  genyaml=$(kube::util::find-binary "genyaml")
+  genfeddocs=$(kube::util::find-binary "genfeddocs")
+
+  # TODO: If ${genfeddocs} is not used from anywhere (it isn't used at
+  # least from k/k tree), remove it completely.
+  kube::util::sourced_variable "${genfeddocs}"
+
+  mkdir -p "${dest}/docs/user-guide/kubectl/"
+  "${gendocs}" "${dest}/docs/user-guide/kubectl/"
+  mkdir -p "${dest}/docs/admin/"
+  "${genkubedocs}" "${dest}/docs/admin/" "kube-apiserver"
+  "${genkubedocs}" "${dest}/docs/admin/" "kube-controller-manager"
+  "${genkubedocs}" "${dest}/docs/admin/" "kube-proxy"
+  "${genkubedocs}" "${dest}/docs/admin/" "kube-scheduler"
+  "${genkubedocs}" "${dest}/docs/admin/" "kubelet"
+  "${genkubedocs}" "${dest}/docs/admin/" "kubeadm"
+
+  mkdir -p "${dest}/docs/man/man1/"
+  "${genman}" "${dest}/docs/man/man1/" "kube-apiserver"
+  "${genman}" "${dest}/docs/man/man1/" "kube-controller-manager"
+  "${genman}" "${dest}/docs/man/man1/" "kube-proxy"
+  "${genman}" "${dest}/docs/man/man1/" "kube-scheduler"
+  "${genman}" "${dest}/docs/man/man1/" "kubelet"
+  "${genman}" "${dest}/docs/man/man1/" "kubectl"
+  "${genman}" "${dest}/docs/man/man1/" "kubeadm"
+
+  mkdir -p "${dest}/docs/yaml/kubectl/"
+  "${genyaml}" "${dest}/docs/yaml/kubectl/"
+
+  # create the list of generated files
+  pushd "${dest}" > /dev/null || return 1
+  touch docs/.generated_docs
+  find . -type f | cut -sd / -f 2- | LC_ALL=C sort > docs/.generated_docs
+  popd > /dev/null || return 1
+}
+
+# Removes previously generated docs-- we don't want to check them in. $KUBE_ROOT
+# must be set.
+kube::util::remove-gen-docs() {
+  if [ -e "${KUBE_ROOT}/docs/.generated_docs" ]; then
+    # remove all of the old docs; we don't want to check them in.
+    while read -r file; do
+      rm "${KUBE_ROOT}/${file}" 2>/dev/null || true
+    done <"${KUBE_ROOT}/docs/.generated_docs"
+    # The docs/.generated_docs file lists itself, so we don't need to explicitly
+    # delete it.
+  fi
+}
+
+# Takes a group/version and returns the path to its location on disk, sans
+# "pkg". E.g.:
+# * default behavior: extensions/v1beta1 -> apis/extensions/v1beta1
+# * default behavior for only a group: experimental -> apis/experimental
+# * Special handling for empty group: v1 -> api/v1, unversioned -> api/unversioned
+# * Special handling for groups suffixed with ".k8s.io": foo.k8s.io/v1 -> apis/foo/v1
+# * Very special handling for when both group and version are "": / -> api
+#
+# $KUBE_ROOT must be set.
+kube::util::group-version-to-pkg-path() {
+  local group_version="$1"
+
+  while IFS=$'\n' read -r api; do
+    if [[ "${api}" = "${group_version/.*k8s.io/}" ]]; then
+      echo "vendor/k8s.io/api/${group_version/.*k8s.io/}"
+      return
+    fi
+  done < <(cd "${KUBE_ROOT}/staging/src/k8s.io/api" && find . -name types.go -exec dirname {} \; | sed "s|\./||g" | sort)
+
+  # "v1" is the API GroupVersion
+  if [[ "${group_version}" == "v1" ]]; then
+    echo "vendor/k8s.io/api/core/v1"
+    return
+  fi
+
+  # Special cases first.
+  # TODO(lavalamp): Simplify this by moving pkg/api/v1 and splitting pkg/api,
+  # moving the results to pkg/apis/api.
+  case "${group_version}" in
+    # both group and version are "", this occurs when we generate deep copies for internal objects of the legacy v1 API.
+    __internal)
+      echo "pkg/apis/core"
+      ;;
+    meta/v1)
+      echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1"
+      ;;
+    meta/v1beta1)
+      echo "vendor/k8s.io/apimachinery/pkg/apis/meta/v1beta1"
+      ;;
+    *.k8s.io)
+      echo "pkg/apis/${group_version%.*k8s.io}"
+      ;;
+    *.k8s.io/*)
+      echo "pkg/apis/${group_version/.*k8s.io/}"
+      ;;
+    *)
+      echo "pkg/apis/${group_version%__internal}"
+      ;;
+  esac
+}
+
+# Takes a group/version and returns the swagger-spec file name.
+# default behavior: extensions/v1beta1 -> extensions_v1beta1
+# special case for v1: v1 -> v1
+kube::util::gv-to-swagger-name() {
+  local group_version="$1"
+  case "${group_version}" in
+    v1)
+      echo "v1"
+      ;;
+    *)
+      echo "${group_version%/*}_${group_version#*/}"
+      ;;
+  esac
+}
+
+# Returns the name of the upstream remote repository name for the local git
+# repo, e.g. "upstream" or "origin".
+kube::util::git_upstream_remote_name() {
+  git remote -v | grep fetch |\
+    grep -E 'github.com[/:]kubernetes/kubernetes|k8s.io/kubernetes' |\
+    head -n 1 | awk '{print $1}'
+}
+
+# Exits script if working directory is dirty. If it's run interactively in the terminal
+# the user can commit changes in a second terminal. This script will wait.
+kube::util::ensure_clean_working_dir() {
+  while ! git diff HEAD --exit-code &>/dev/null; do
+    echo -e "\nUnexpected dirty working directory:\n"
+    if tty -s; then
+        git status -s
+    else
+        git diff -a # be more verbose in log files without tty
+        exit 1
+    fi | sed 's/^/  /'
+    echo -e "\nCommit your changes in another terminal and then continue here by pressing enter."
+    read -r
+  done 1>&2
+}
+
+# Find the base commit using:
+# $PULL_BASE_SHA if set (from Prow)
+# current ref from the remote upstream branch
+kube::util::base_ref() {
+  local -r git_branch=$1
+
+  if [[ -n ${PULL_BASE_SHA:-} ]]; then
+    echo "${PULL_BASE_SHA}"
+    return
+  fi
+
+  full_branch="$(kube::util::git_upstream_remote_name)/${git_branch}"
+
+  # make sure the branch is valid, otherwise the check will pass erroneously.
+  if ! git describe "${full_branch}" >/dev/null; then
+    # abort!
+    exit 1
+  fi
+
+  echo "${full_branch}"
+}
+
+# Checks whether there are any files matching pattern $2 changed between the
+# current branch and upstream branch named by $1.
+# Returns 1 (false) if there are no changes
+#         0 (true) if there are changes detected.
+kube::util::has_changes() {
+  local -r git_branch=$1
+  local -r pattern=$2
+  local -r not_pattern=${3:-totallyimpossiblepattern}
+
+  local base_ref
+  base_ref=$(kube::util::base_ref "${git_branch}")
+  echo "Checking for '${pattern}' changes against '${base_ref}'"
+
+  # notice this uses ... to find the first shared ancestor
+  if git diff --name-only "${base_ref}...HEAD" | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then
+    return 0
+  fi
+  # also check for pending changes
+  if git status --porcelain | grep -v -E "${not_pattern}" | grep "${pattern}" > /dev/null; then
+    echo "Detected '${pattern}' uncommitted changes."
+    return 0
+  fi
+  echo "No '${pattern}' changes detected."
+  return 1
+}
+
+kube::util::download_file() {
+  local -r url=$1
+  local -r destination_file=$2
+
+  rm "${destination_file}" 2&> /dev/null || true
+
+  for i in $(seq 5)
+  do
+    if ! curl -fsSL --retry 3 --keepalive-time 2 "${url}" -o "${destination_file}"; then
+      echo "Downloading ${url} failed. $((5-i)) retries left."
+      sleep 1
+    else
+      echo "Downloading ${url} succeed"
+      return 0
+    fi
+  done
+  return 1
+}
+
+# Test whether openssl is installed.
+# Sets:
+#  OPENSSL_BIN: The path to the openssl binary to use
+function kube::util::test_openssl_installed {
+    if ! openssl version >& /dev/null; then
+      echo "Failed to run openssl. Please ensure openssl is installed"
+      exit 1
+    fi
+
+    OPENSSL_BIN=$(command -v openssl)
+}
+
+# creates a client CA, args are sudo, dest-dir, ca-id, purpose
+# purpose is dropped in after "key encipherment", you usually want
+# '"client auth"'
+# '"server auth"'
+# '"client auth","server auth"'
+function kube::util::create_signing_certkey {
+    local sudo=$1
+    local dest_dir=$2
+    local id=$3
+    local purpose=$4
+    # Create client ca
+    ${sudo} /usr/bin/env bash -e <<EOF
+    rm -f "${dest_dir}/${id}-ca.crt" "${dest_dir}/${id}-ca.key"
+    ${OPENSSL_BIN} req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${dest_dir}/${id}-ca.key" -out "${dest_dir}/${id}-ca.crt" -subj "/C=xx/ST=x/L=x/O=x/OU=x/CN=ca/emailAddress=x/"
+    echo '{"signing":{"default":{"expiry":"43800h","usages":["signing","key encipherment",${purpose}]}}}' > "${dest_dir}/${id}-ca-config.json"
+EOF
+}
+
+# signs a client certificate: args are sudo, dest-dir, CA, filename (roughly), username, groups...
+function kube::util::create_client_certkey {
+    local sudo=$1
+    local dest_dir=$2
+    local ca=$3
+    local id=$4
+    local cn=${5:-$4}
+    local groups=""
+    local SEP=""
+    shift 5
+    while [ -n "${1:-}" ]; do
+        groups+="${SEP}{\"O\":\"$1\"}"
+        SEP=","
+        shift 1
+    done
+    ${sudo} /usr/bin/env bash -e <<EOF
+    cd ${dest_dir}
+    echo '{"CN":"${cn}","names":[${groups}],"hosts":[""],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare client-${id}
+    mv "client-${id}-key.pem" "client-${id}.key"
+    mv "client-${id}.pem" "client-${id}.crt"
+    rm -f "client-${id}.csr"
+EOF
+}
+
+# signs a serving certificate: args are sudo, dest-dir, ca, filename (roughly), subject, hosts...
+function kube::util::create_serving_certkey {
+    local sudo=$1
+    local dest_dir=$2
+    local ca=$3
+    local id=$4
+    local cn=${5:-$4}
+    local hosts=""
+    local SEP=""
+    shift 5
+    while [ -n "${1:-}" ]; do
+        hosts+="${SEP}\"$1\""
+        SEP=","
+        shift 1
+    done
+    ${sudo} /usr/bin/env bash -e <<EOF
+    cd ${dest_dir}
+    echo '{"CN":"${cn}","hosts":[${hosts}],"key":{"algo":"rsa","size":2048}}' | ${CFSSL_BIN} gencert -ca=${ca}.crt -ca-key=${ca}.key -config=${ca}-config.json - | ${CFSSLJSON_BIN} -bare serving-${id}
+    mv "serving-${id}-key.pem" "serving-${id}.key"
+    mv "serving-${id}.pem" "serving-${id}.crt"
+    rm -f "serving-${id}.csr"
+EOF
+}
+
+# creates a self-contained kubeconfig: args are sudo, dest-dir, ca file, host, port, client id, token(optional)
+function kube::util::write_client_kubeconfig {
+    local sudo=$1
+    local dest_dir=$2
+    local ca_file=$3
+    local api_host=$4
+    local api_port=$5
+    local client_id=$6
+    local token=${7:-}
+    cat <<EOF | ${sudo} tee "${dest_dir}"/"${client_id}".kubeconfig > /dev/null
+apiVersion: v1
+kind: Config
+clusters:
+  - cluster:
+      certificate-authority: ${ca_file}
+      server: https://${api_host}:${api_port}/
+    name: local-up-cluster
+users:
+  - user:
+      token: ${token}
+      client-certificate: ${dest_dir}/client-${client_id}.crt
+      client-key: ${dest_dir}/client-${client_id}.key
+    name: local-up-cluster
+contexts:
+  - context:
+      cluster: local-up-cluster
+      user: local-up-cluster
+    name: local-up-cluster
+current-context: local-up-cluster
+EOF
+
+    # flatten the kubeconfig files to make them self contained
+    username=$(whoami)
+    ${sudo} /usr/bin/env bash -e <<EOF
+    $(kube::util::find-binary kubectl) --kubeconfig="${dest_dir}/${client_id}.kubeconfig" config view --minify --flatten > "/tmp/${client_id}.kubeconfig"
+    mv -f "/tmp/${client_id}.kubeconfig" "${dest_dir}/${client_id}.kubeconfig"
+    chown ${username} "${dest_dir}/${client_id}.kubeconfig"
+EOF
+}
+
+# list_staging_repos outputs a sorted list of repos in staging/src/k8s.io
+# each entry will just be the $repo portion of staging/src/k8s.io/$repo/...
+# $KUBE_ROOT must be set.
+function kube::util::list_staging_repos() {
+  (
+    cd "${KUBE_ROOT}/staging/src/k8s.io" && \
+    find . -mindepth 1 -maxdepth 1 -type d | cut -c 3- | sort
+  )
+}
+
+
+# Determines if docker can be run, failures may simply require that the user be added to the docker group.
+function kube::util::ensure_docker_daemon_connectivity {
+  IFS=" " read -ra DOCKER <<< "${DOCKER_OPTS}"
+  # Expand ${DOCKER[@]} only if it's not unset. This is to work around
+  # Bash 3 issue with unbound variable.
+  DOCKER=(docker ${DOCKER[@]:+"${DOCKER[@]}"})
+  if ! "${DOCKER[@]}" info > /dev/null 2>&1 ; then
+    cat <<'EOF' >&2
+Can't connect to 'docker' daemon.  please fix and retry.
+
+Possible causes:
+  - Docker Daemon not started
+    - Linux: confirm via your init system
+    - macOS w/ docker-machine: run `docker-machine ls` and `docker-machine start <name>`
+    - macOS w/ Docker for Mac: Check the menu bar and start the Docker application
+  - DOCKER_HOST hasn't been set or is set incorrectly
+    - Linux: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
+    - macOS w/ docker-machine: run `eval "$(docker-machine env <name>)"`
+    - macOS w/ Docker for Mac: domain socket is used, DOCKER_* should be unset. In Bash run `unset ${!DOCKER_*}`
+  - Other things to check:
+    - Linux: User isn't in 'docker' group.  Add and relogin.
+      - Something like 'sudo usermod -a -G docker ${USER}'
+      - RHEL7 bug and workaround: https://bugzilla.redhat.com/show_bug.cgi?id=1119282#c8
+EOF
+    return 1
+  fi
+}
+
+# Wait for background jobs to finish. Return with
+# an error status if any of the jobs failed.
+kube::util::wait-for-jobs() {
+  local fail=0
+  local job
+  for job in $(jobs -p); do
+    wait "${job}" || fail=$((fail + 1))
+  done
+  return ${fail}
+}
+
+# kube::util::join <delim> <list...>
+# Concatenates the list elements with the delimiter passed as first parameter
+#
+# Ex: kube::util::join , a b c
+#  -> a,b,c
+function kube::util::join {
+  local IFS="$1"
+  shift
+  echo "$*"
+}
+
+# Downloads cfssl/cfssljson into $1 directory if they do not already exist in PATH
+#
+# Assumed vars:
+#   $1 (cfssl directory) (optional)
+#
+# Sets:
+#  CFSSL_BIN: The path of the installed cfssl binary
+#  CFSSLJSON_BIN: The path of the installed cfssljson binary
+#
+function kube::util::ensure-cfssl {
+  if command -v cfssl &>/dev/null && command -v cfssljson &>/dev/null; then
+    CFSSL_BIN=$(command -v cfssl)
+    CFSSLJSON_BIN=$(command -v cfssljson)
+    return 0
+  fi
+
+  host_arch=$(kube::util::host_arch)
+
+  if [[ "${host_arch}" != "amd64" ]]; then
+    echo "Cannot download cfssl on non-amd64 hosts and cfssl does not appear to be installed."
+    echo "Please install cfssl and cfssljson and verify they are in \$PATH."
+    echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
+    exit 1
+  fi
+
+  # Create a temp dir for cfssl if no directory was given
+  local cfssldir=${1:-}
+  if [[ -z "${cfssldir}" ]]; then
+    kube::util::ensure-temp-dir
+    cfssldir="${KUBE_TEMP}/cfssl"
+  fi
+
+  mkdir -p "${cfssldir}"
+  pushd "${cfssldir}" > /dev/null || return 1
+
+    echo "Unable to successfully run 'cfssl' from ${PATH}; downloading instead..."
+    kernel=$(uname -s)
+    case "${kernel}" in
+      Linux)
+        curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
+        curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
+        ;;
+      Darwin)
+        curl --retry 10 -L -o cfssl https://pkg.cfssl.org/R1.2/cfssl_darwin-amd64
+        curl --retry 10 -L -o cfssljson https://pkg.cfssl.org/R1.2/cfssljson_darwin-amd64
+        ;;
+      *)
+        echo "Unknown, unsupported platform: ${kernel}." >&2
+        echo "Supported platforms: Linux, Darwin." >&2
+        exit 2
+    esac
+
+    chmod +x cfssl || true
+    chmod +x cfssljson || true
+
+    CFSSL_BIN="${cfssldir}/cfssl"
+    CFSSLJSON_BIN="${cfssldir}/cfssljson"
+    if [[ ! -x ${CFSSL_BIN} || ! -x ${CFSSLJSON_BIN} ]]; then
+      echo "Failed to download 'cfssl'. Please install cfssl and cfssljson and verify they are in \$PATH."
+      echo "Hint: export PATH=\$PATH:\$GOPATH/bin; go get -u github.com/cloudflare/cfssl/cmd/..."
+      exit 1
+    fi
+  popd > /dev/null || return 1
+}
+
+# kube::util::ensure_dockerized
+# Confirms that the script is being run inside a kube-build image
+#
+function kube::util::ensure_dockerized {
+  if [[ -f /kube-build-image ]]; then
+    return 0
+  else
+    echo "ERROR: This script is designed to be run inside a kube-build container"
+    exit 1
+  fi
+}
+
+# kube::util::ensure-gnu-sed
+# Determines which sed binary is gnu-sed on linux/darwin
+#
+# Sets:
+#  SED: The name of the gnu-sed binary
+#
+function kube::util::ensure-gnu-sed {
+  if LANG=C sed --help 2>&1 | grep -q GNU; then
+    SED="sed"
+  elif command -v gsed &>/dev/null; then
+    SED="gsed"
+  else
+    kube::log::error "Failed to find GNU sed as sed or gsed. If you are on Mac: brew install gnu-sed." >&2
+    return 1
+  fi
+  kube::util::sourced_variable "${SED}"
+}
+
+# kube::util::check-file-in-alphabetical-order <file>
+# Check that the file is in alphabetical order
+#
+function kube::util::check-file-in-alphabetical-order {
+  local failure_file="$1"
+  if ! diff -u "${failure_file}" <(LC_ALL=C sort "${failure_file}"); then
+    {
+      echo
+      echo "${failure_file} is not in alphabetical order. Please sort it:"
+      echo
+      echo "  LC_ALL=C sort -o ${failure_file} ${failure_file}"
+      echo
+    } >&2
+    false
+  fi
+}
+
+# kube::util::require-jq
+# Checks whether jq is installed.
+function kube::util::require-jq {
+  if ! command -v jq &>/dev/null; then
+    echo "jq not found. Please install." 1>&2
+    return 1
+  fi
+}
+
+# outputs md5 hash of $1, works on macOS and Linux
+function kube::util::md5() {
+  if which md5 >/dev/null 2>&1; then
+    md5 -q "$1"
+  else
+    md5sum "$1" | awk '{ print $1 }'
+  fi
+}
+
+# kube::util::read-array
+# Reads in stdin and adds it line by line to the array provided. This can be
+# used instead of "mapfile -t", and is bash 3 compatible.
+#
+# Assumed vars:
+#   $1 (name of array to create/modify)
+#
+# Example usage:
+# kube::util::read-array files < <(ls -1)
+#
+function kube::util::read-array {
+  local i=0
+  unset -v "$1"
+  while IFS= read -r "$1[i++]"; do :; done
+  eval "[[ \${$1[--i]} ]]" || unset "$1[i]" # ensures last element isn't empty
+}
+
+# Some useful colors.
+if [[ -z "${color_start-}" ]]; then
+  declare -r color_start="\033["
+  declare -r color_red="${color_start}0;31m"
+  declare -r color_yellow="${color_start}0;33m"
+  declare -r color_green="${color_start}0;32m"
+  declare -r color_blue="${color_start}1;34m"
+  declare -r color_cyan="${color_start}1;36m"
+  declare -r color_norm="${color_start}0m"
+
+  kube::util::sourced_variable "${color_start}"
+  kube::util::sourced_variable "${color_red}"
+  kube::util::sourced_variable "${color_yellow}"
+  kube::util::sourced_variable "${color_green}"
+  kube::util::sourced_variable "${color_blue}"
+  kube::util::sourced_variable "${color_cyan}"
+  kube::util::sourced_variable "${color_norm}"
+fi
+
+# ex: ts=2 sw=2 et filetype=sh

hack/lint-dependencies.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Explicitly opt into go modules, even though we're inside a GOPATH directory
+export GO111MODULE=on
+# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+export GOFLAGS=
+# Detect problematic GOPROXY settings that prevent lookup of dependencies
+if [[ "${GOPROXY:-}" == "off" ]]; then
+  kube::log::error "Cannot run with \$GOPROXY=off"
+  exit 1
+fi
+
+kube::golang::verify_go_version
+kube::util::require-jq
+
+case "${1:-}" in
+"--all")
+  echo "Checking all dependencies"
+  filter=''
+  ;;
+"-a")
+  echo "Checking all dependencies"
+  filter=''
+  ;;
+"")
+  # by default, skip checking golang.org/x/... dependencies... we pin to levels that match our go version for those
+  echo "Skipping golang.org/x/... dependencies, pass --all to include"
+  filter='select(.Path | startswith("golang.org/x/") | not) |'
+  ;;
+*)
+  kube::log::error "Unrecognized arg: ${1}"
+  exit 1
+  ;;
+esac
+
+outdated=$(go list -m -json all | jq -r "
+  select(.Replace.Version != null) |
+  select(.Version != .Replace.Version) |
+  ${filter}
+  select(.Path) |
+  \"\(.Path)
+    pinned:    \(.Replace.Version)
+    preferred: \(.Version)
+    hack/pin-dependency.sh \(.Path) \(.Version)\"
+")
+if [[ -n "${outdated}" ]]; then
+  echo "These modules are pinned to versions different than the minimal preferred version."
+  echo "That means that without replace directives, a different version would be selected,"
+  echo "which breaks consumers of our published modules."
+  echo "1. Use hack/pin-dependency.sh to switch to the preferred version for each module"
+  echo "2. Run hack/update-vendor.sh to rebuild the vendor directory"
+  echo "3. Run hack/lint-dependencies.sh to verify no additional changes are required"
+  echo ""
+  echo "${outdated}"
+fi
+
+unused=$(comm -23 \
+  <(go mod edit -json | jq -r '.Replace[] | select(.New.Version != null) | .Old.Path' | sort) \
+  <(go list -m -json all | jq -r .Path | sort))
+if [[ -n "${unused}" ]]; then
+  echo ""
+  echo "Use the given commands to remove pinned module versions that aren't actually used:"
+  echo "${unused}" | xargs -L 1 echo 'GO111MODULE=on go mod edit -dropreplace'
+fi
+
+if [[ -n "${unused}${outdated}" ]]; then
+  exit 1
+fi
+
+echo "All pinned versions of checked dependencies match their preferred version."
+exit 0

hack/pin-dependency.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Usage:
+#   hack/pin-dependency.sh $MODULE $SHA-OR-TAG
+#
+# Example:
+#   hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7
+
+# Explicitly opt into go modules, even though we're inside a GOPATH directory
+export GO111MODULE=on
+# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+export GOFLAGS=
+# Detect problematic GOPROXY settings that prevent lookup of dependencies
+if [[ "${GOPROXY:-}" == "off" ]]; then
+  kube::log::error "Cannot run with \$GOPROXY=off"
+  exit 1
+fi
+
+kube::golang::verify_go_version
+kube::util::require-jq
+
+dep="${1:-}"
+sha="${2:-}"
+if [[ -z "${dep}" || -z "${sha}" ]]; then
+  echo "Usage:"
+  echo "  hack/pin-dependency.sh \$MODULE \$SHA-OR-TAG"
+  echo ""
+  echo "Example:"
+  echo "  hack/pin-dependency.sh github.com/docker/docker 501cb131a7b7"
+  echo ""
+  exit 1
+fi
+
+_tmp="${KUBE_ROOT}/_tmp"
+cleanup() {
+  rm -rf "${_tmp}"
+}
+trap "cleanup" EXIT SIGINT
+cleanup
+mkdir -p "${_tmp}"
+
+# Add the require directive
+echo "Running: go get ${dep}@${sha}"
+go get -m -d "${dep}@${sha}"
+
+# Find the resolved version
+rev=$(go mod edit -json | jq -r ".Require[] | select(.Path == \"${dep}\") | .Version")
+
+# No entry in go.mod, we must be using the natural version indirectly
+if [[ -z "${rev}" ]]; then
+  # backup the go.mod file, since go list modifies it
+  cp go.mod "${_tmp}/go.mod.bak"
+  # find the revision
+  rev=$(go list -m -json "${dep}" | jq -r .Version)
+  # restore the go.mod file
+  mv "${_tmp}/go.mod.bak" go.mod
+fi
+
+# No entry found
+if [[ -z "${rev}" ]]; then
+  echo "Could not resolve ${sha}"
+  exit 1
+fi
+
+echo "Resolved to ${dep}@${rev}"
+
+# Add the replace directive
+echo "Running: go mod edit -replace ${dep}=${dep}@${rev}"
+go mod edit -replace "${dep}=${dep}@${rev}"
+
+echo ""
+echo "Run hack/update-vendor.sh to rebuild the vendor directory"

hack/update-vendor-licenses.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+# Copyright 2015 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Update the Godeps/LICENSES document.
+# Generates a table of Godep dependencies and their license.
+#
+# Usage:
+#    $0 [--create-missing] [/path/to/licenses]
+#
+#    --create-missing will write the files that only exist upstream, locally.
+#    This option is mostly used for testing as we cannot check-in any of the
+#    additionally created files into the vendor auto-generated tree.
+#
+#    Run every time a license file is added/modified within /vendor to
+#    update /Godeps/LICENSES
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+export LANG=C
+export LC_ALL=C
+
+###############################################################################
+# Process package content
+#
+# @param package  The incoming package name
+# @param type     The type of content (LICENSE, COPYRIGHT or COPYING)
+#
+process_content () {
+  local package=$1
+  local type=$2
+
+  local package_root
+  local ensure_pattern
+  local dir_root
+  local find_maxdepth
+  local find_names
+  local -a local_files=()
+
+  # Necessary to expand {}
+  case ${type} in
+      LICENSE) find_names=(-iname 'licen[sc]e*')
+               find_maxdepth=1
+               # Sadly inconsistent in the wild, but mostly license files
+               # containing copyrights, but no readme/notice files containing
+               # licenses (except to "see license file")
+               ensure_pattern="license|copyright"
+               ;;
+    # We search READMEs for copyrights and this includes notice files as well
+    # Look in as many places as we find files matching
+    COPYRIGHT) find_names=(-iname 'notice*' -o -iname 'readme*')
+               find_maxdepth=3
+               ensure_pattern="copyright"
+               ;;
+      COPYING) find_names=(-iname 'copying*')
+               find_maxdepth=1
+               ensure_pattern="license|copyright"
+               ;;
+  esac
+
+  # Start search at package root
+  case ${package} in
+    github.com/*|golang.org/*|bitbucket.org/*|gonum.org/*)
+     package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2"/"$3 }')
+     ;;
+    go4.org/*)
+     package_root=$(echo "${package}" |awk -F/ '{ print $1 }')
+     ;;
+    gopkg.in/*)
+     # Root of gopkg.in package always ends with '.v(number)' and my contain
+     # more than two path elements. For example:
+     # - gopkg.in/yaml.v2
+     # - gopkg.in/inf.v0
+     # - gopkg.in/square/go-jose.v2
+     package_root=$(echo "${package}" |grep -oh '.*\.v[0-9]')
+     ;;
+    */*)
+     package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2 }')
+     ;;
+    *)
+     package_root="${package}"
+     ;;
+  esac
+
+  # Find files - only root and package level
+  local_files=()
+  IFS=" " read -r -a local_files <<< "$(
+    for dir_root in ${package} ${package_root}; do
+      [[ -d ${DEPS_DIR}/${dir_root} ]] || continue
+
+      # One (set) of these is fine
+      find "${DEPS_DIR}/${dir_root}" \
+          -xdev -follow -maxdepth ${find_maxdepth} \
+          -type f "${find_names[@]}"
+    done | sort -u)"
+
+  local index
+  local f
+  index="${package}-${type}"
+  if [[ -z "${CONTENT[${index}]-}" ]]; then
+    for f in "${local_files[@]-}"; do
+      if [[ -z "$f" ]]; then
+        # Set the default value and then check it to prevent
+        # accessing potentially empty array
+        continue
+      fi
+      # Find some copyright info in any file and break
+      if grep -E -i -wq "${ensure_pattern}" "${f}"; then
+        CONTENT[${index}]="${f}"
+        break
+      fi
+    done
+  fi
+}
+
+
+#############################################################################
+# MAIN
+#############################################################################
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+export GO111MODULE=on
+
+# Check bash version
+if (( BASH_VERSINFO[0] < 4 )); then
+  echo
+  echo "ERROR: Bash v4+ required."
+  # Extra help for OSX
+  if [[ "$(uname -s)" == "Darwin" ]]; then
+    echo
+    echo "Ensure you are up to date on the following packages:"
+    echo "$ brew install md5sha1sum bash jq"
+  fi
+  echo
+  exit 9
+fi
+
+# This variable can be injected, as in the verify script.
+LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}"
+cd "${LICENSE_ROOT}"
+
+VENDOR_LICENSE_FILE="Godeps/LICENSES"
+TMP_LICENSE_FILE="/tmp/Godeps.LICENSES.$$"
+DEPS_DIR="vendor"
+declare -Ag CONTENT
+
+# Put the K8S LICENSE on top
+(
+echo "================================================================================"
+echo "= Kubernetes licensed under: ="
+echo
+cat "${LICENSE_ROOT}/LICENSE"
+echo
+echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")"
+echo "================================================================================"
+) > ${TMP_LICENSE_FILE}
+
+# Loop through every vendored package
+for PACKAGE in $(go list -m -json all | jq -r .Path | sort -f); do
+  if [[ -e "staging/src/${PACKAGE}" ]]; then
+    echo "$PACKAGE is a staging package, skipping" > /dev/stderr
+    continue
+  fi
+  if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then
+    echo "$PACKAGE doesn't exist in vendor, skipping" > /dev/stderr
+    continue
+  fi
+
+  process_content "${PACKAGE}" LICENSE
+  process_content "${PACKAGE}" COPYRIGHT
+  process_content "${PACKAGE}" COPYING
+
+  # display content
+  echo
+  echo "================================================================================"
+  echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
+  echo
+
+  file=""
+  if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
+      file="${CONTENT[${PACKAGE}-LICENSE]-}"
+  elif [[ -n "${CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
+      file="${CONTENT[${PACKAGE}-COPYRIGHT]-}"
+  elif [[ -n "${CONTENT[${PACKAGE}-COPYING]-}" ]]; then
+      file="${CONTENT[${PACKAGE}-COPYING]-}"
+  fi
+  if [[ -z "${file}" ]]; then
+      cat > /dev/stderr << __EOF__
+No license could be found for ${PACKAGE} - aborting.
+
+Options:
+1. Check if the upstream repository has a newer version with LICENSE, COPYRIGHT and/or
+   COPYING files.
+2. Contact the author of the package to ensure there is a LICENSE, COPYRIGHT and/or
+   COPYING file present.
+3. Do not use this package in Kubernetes.
+__EOF__
+      exit 9
+  fi
+  cat "${file}"
+
+  echo
+  echo "= ${file} $(kube::util::md5 "${file}")"
+  echo "================================================================================"
+  echo
+done >> ${TMP_LICENSE_FILE}
+
+cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE}

hack/update-vendor.sh

@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+# This script is a modified version of Kubernetes
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
+source "${KUBE_ROOT}/hack/lib/init.sh"
+
+# Explicitly opt into go modules, even though we're inside a GOPATH directory
+export GO111MODULE=on
+# Explicitly clear GOFLAGS, since GOFLAGS=-mod=vendor breaks dependency resolution while rebuilding vendor
+export GOFLAGS=
+# Ensure sort order doesn't depend on locale
+export LANG=C
+export LC_ALL=C
+# Detect problematic GOPROXY settings that prevent lookup of dependencies
+if [[ "${GOPROXY:-}" == "off" ]]; then
+  kube::log::error "Cannot run hack/update-vendor.sh with \$GOPROXY=off"
+  exit 1
+fi
+
+kube::golang::verify_go_version
+kube::util::require-jq
+
+TMP_DIR="${TMP_DIR:-$(mktemp -d /tmp/update-vendor.XXXX)}"
+LOG_FILE="${LOG_FILE:-${TMP_DIR}/update-vendor.log}"
+kube::log::status "logfile at ${LOG_FILE}"
+
+if [ -z "${BASH_XTRACEFD:-}" ]; then
+  exec 19> "${LOG_FILE}"
+  export BASH_XTRACEFD="19"
+  set -x
+fi
+
+# ensure_require_replace_directives_for_all_dependencies:
+# - ensures all existing 'require' directives have an associated 'replace' directive pinning a version
+# - adds explicit 'require' directives for all transitive dependencies
+# - adds explicit 'replace' directives for all require directives (existing 'replace' directives take precedence)
+function ensure_require_replace_directives_for_all_dependencies() {
+  local local_tmp_dir
+  local_tmp_dir=$(mktemp -d "${TMP_DIR}/pin_replace.XXXX")
+
+  # collect 'require' directives that actually specify a version
+  local require_filter='(.Version != null) and (.Version != "v0.0.0") and (.Version != "v0.0.0-00010101000000-000000000000")'
+  # collect 'replace' directives that unconditionally pin versions (old=new@version)
+  local replace_filter='(.Old.Version == null) and (.New.Version != null)'
+
+  # Capture local require/replace directives before running any go commands that can modify the go.mod file
+  local require_json="${local_tmp_dir}/require.json"
+  local replace_json="${local_tmp_dir}/replace.json"
+  go mod edit -json | jq -r ".Require // [] | sort | .[] | select(${require_filter})" > "${require_json}"
+  go mod edit -json | jq -r ".Replace // [] | sort | .[] | select(${replace_filter})" > "${replace_json}"
+
+  # 1a. Ensure replace directives have an explicit require directive
+  jq -r '"-require \(.Old.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
+  # 1b. Ensure require directives have a corresponding replace directive pinning a version
+  jq -r '"-replace \(.Path)=\(.Path)@\(.Version)"' < "${require_json}" | xargs -L 100 go mod edit -fmt
+  jq -r '"-replace \(.Old.Path)=\(.New.Path)@\(.New.Version)"' < "${replace_json}" | xargs -L 100 go mod edit -fmt
+
+  # 3. Add explicit require directives for indirect dependencies
+  go list -m -json all | jq -r 'select(.Main != true) | select(.Indirect == true) | "-require \(.Path)@\(.Version)"'          | xargs -L 100 go mod edit -fmt
+
+  # 4. Add explicit replace directives pinning dependencies that aren't pinned yet
+  go list -m -json all | jq -r 'select(.Main != true) | select(.Replace == null)  | "-replace \(.Path)=\(.Path)@\(.Version)"' | xargs -L 100 go mod edit -fmt
+}
+
+function group_replace_directives() {
+  local local_tmp_dir
+  local_tmp_dir=$(mktemp -d "${TMP_DIR}/group_replace.XXXX")
+  local go_mod_replace="${local_tmp_dir}/go.mod.replace.tmp"
+  local go_mod_noreplace="${local_tmp_dir}/go.mod.noreplace.tmp"
+  # separate replace and non-replace directives
+  awk "
+     # print lines between 'replace (' ... ')' lines
+     /^replace [(]/      { inreplace=1; next                   }
+     inreplace && /^[)]/ { inreplace=0; next                   }
+     inreplace           { print > \"${go_mod_replace}\"; next }
+
+     # print ungrouped replace directives with the replace directive trimmed
+     /^replace [^(]/ { sub(/^replace /,\"\"); print > \"${go_mod_replace}\"; next }
+
+     # otherwise print to the noreplace file
+     { print > \"${go_mod_noreplace}\" }
+  " < go.mod
+  {
+    cat "${go_mod_noreplace}";
+    echo "replace (";
+    cat "${go_mod_replace}";
+    echo ")";
+  } > go.mod
+
+  go mod edit -fmt
+}
+
+function add_generated_comments() {
+  local local_tmp_dir
+  local_tmp_dir=$(mktemp -d "${TMP_DIR}/add_generated_comments.XXXX")
+  local go_mod_nocomments="${local_tmp_dir}/go.mod.nocomments.tmp"
+
+  # drop comments before the module directive
+  awk "
+     BEGIN           { dropcomments=1 }
+     /^module /      { dropcomments=0 }
+     dropcomments && /^\/\// { next }
+     { print }
+  " < go.mod > "${go_mod_nocomments}"
+
+  # Add the specified comments
+  local comments="${1}"
+  {
+    echo "${comments}"
+    echo ""
+    cat "${go_mod_nocomments}"
+   } > go.mod
+
+  # Format
+  go mod edit -fmt
+}
+
+
+# Phase 1: ensure go.mod files for staging modules and main module
+if [[ ! -f go.mod ]]; then
+  kube::log::status "go.mod: initialize kubesphere.io/kubesphere"
+  go mod init "kubesphere.io/kubesphere"
+  rm -f Godeps/Godeps.json # remove after initializing
+fi
+
+# Phase 2: ensure staging repo require/replace directives
+
+kube::log::status "go.mod: update references"
+# Prune
+go mod edit -json | jq -r '.Require[]? | select(.Version == "v0.0.0")                 | "-droprequire \(.Path)"'     | xargs -L 100 go mod edit -fmt
+go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("./staging/")) | "-dropreplace \(.Old.Path)"' | xargs -L 100 go mod edit -fmt
+
+# Phase 3: capture required (minimum) versions from all modules, and replaced (pinned) versions from the root module
+
+# pin referenced versions
+ensure_require_replace_directives_for_all_dependencies
+# resolves/expands references in the root go.mod (if needed)
+go mod tidy >>"${LOG_FILE}" 2>&1
+# pin expanded versions
+ensure_require_replace_directives_for_all_dependencies
+# group replace directives
+group_replace_directives
+
+kube::log::status "go.mod: tidying"
+echo "=== tidying root" >> "${LOG_FILE}"
+go mod tidy >>"${LOG_FILE}" 2>&1
+
+
+# Phase 6: add generated comments to go.mod files
+kube::log::status "go.mod: adding generated comments"
+add_generated_comments "
+// This is a generated file. Do not edit directly.
+// Run hack/pin-dependency.sh to change pinned dependency versions.
+// Run hack/update-vendor.sh to update go.mod files and the vendor directory.
+"
+
+# Phase 7: rebuild vendor directory
+
+kube::log::status "vendor: running 'go mod vendor'"
+go mod vendor >>"${LOG_FILE}" 2>&1
+
+# sort recorded packages for a given vendored dependency in modules.txt.
+# `go mod vendor` outputs in imported order, which means slight go changes (or different platforms) can result in a differently ordered modules.txt.
+# scan                 | prefix comment lines with the module name       | sort field 1  | strip leading text on comment lines
+awk '{if($1=="#") print $2 " " $0; else print}' < vendor/modules.txt | sort -k1,1 -s | sed 's/.*#/#/' > "${TMP_DIR}/modules.txt.tmp"
+mv "${TMP_DIR}/modules.txt.tmp" vendor/modules.txt
+
+#kube::log::status "vendor: updating LICENSES file"
+#hack/update-vendor-licenses.sh >>"${LOG_FILE}" 2>&1

install/ingress-controller/ingress-controller.yaml

@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: ks-router
@@ -22,15 +22,23 @@ spec:
       serviceAccountName: kubesphere-router-serviceaccount
       containers:
         - name: nginx-ingress-controller
-          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.16.2
+          image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1
           args:
             - /nginx-ingress-controller
             - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
-            - --configmap=$(POD_NAMESPACE)/nginx-configuration
             - --annotations-prefix=nginx.ingress.kubernetes.io
             - --force-namespace-isolation
             - --update-status
             - --update-status-on-shutdown
+          securityContext:
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+            # www-data -> 33
+            runAsUser: 33
           env:
             - name: POD_NAME
               valueFrom:
@@ -54,7 +62,7 @@ spec:
             initialDelaySeconds: 10
             periodSeconds: 10
             successThreshold: 1
-            timeoutSeconds: 1
+            timeoutSeconds: 10
           readinessProbe:
             failureThreshold: 3
             httpGet:
@@ -63,7 +71,6 @@ spec:
               scheme: HTTP
             periodSeconds: 10
             successThreshold: 1
-            timeoutSeconds: 1
-          securityContext:
-            runAsNonRoot: false
+            timeoutSeconds: 10
+
 

kustomize/network/OWNERS

@@ -0,0 +1,11 @@
+approvers:
+    - magicsong
+    - zheng1
+
+reviewers:
+    - magicsong
+    - zheng1
+
+labels:
+    - area/deploy
+    - area/networking

kustomize/network/calico-etcd/kustomization.yaml

@@ -0,0 +1,23 @@
+bases:
+- ../crds
+
+resources:
+  - network.yaml
+  - rbac/role.yaml
+  - rbac/role_binding.yaml
+
+generatorOptions:
+   disableNameSuffixHash: true
+
+secretGenerator:
+  - name: calico-etcd-secrets
+    files:
+    - etcd-ca=etcd/ca
+    - etcd-key=etcd/key
+    - etcd-cert=etcd/crt
+    type: Opaque
+
+patchesStrategicMerge:
+  - patch_image_name.yaml
+
+namespace: network-test-f22e8ea9

kustomize/network/calico-etcd/network.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name:  network-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: network-manager
+  namespace: network-system
+  labels:
+    control-plane: network-manager
+spec:
+  selector:
+    matchLabels:
+      control-plane: network-manager
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        control-plane: network-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      hostNetwork: true
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+        - key: "node-role.kubernetes.io/master"
+          effect: NoSchedule
+      containers:
+      - command:
+        - /ks-network
+        args:
+          - -v=4
+          - np-provider=calico
+        image: network:latest
+        imagePullPolicy: Always
+        name: manager
+        resources:
+          limits:
+            cpu: 100m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 20Mi
+        volumeMounts:
+        - mountPath: /calicocerts
+          name: etcd-certs
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: etcd-certs
+        secret:
+          secretName: calico-etcd-secrets
+          defaultMode: 0400

kustomize/network/calico-etcd/patch_image_name.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: network-manager
+  namespace: network-system
+spec:
+  template:
+    spec:
+      containers:
+      # Change the value of image field below to your controller image URL
+      - image: magicsong/ks-network:f22e8ea9
+        name: manager

kustomize/network/calico-etcd/patch_role_binding.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: manager-rolebinding
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: network-test-f22e8ea9

kustomize/network/calico-etcd/role.yaml

@@ -0,0 +1,33 @@
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: network-manager
+rules:
+- apiGroups:
+  - network.kubesphere.io
+  resources:
+  - namespacenetworkpolicies
+  - workspacenetworkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - tenant.kubesphere.io
+  resources:
+  - workspaces
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

kustomize/network/calico-etcd/role_binding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: manager-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: net-manager-role
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: network-system

kustomize/network/calico-k8s/kustomization.yaml

@@ -0,0 +1,11 @@
+bases:
+- ../crds
+
+resources:
+- network.yaml
+- role.yaml
+
+patchesStrategicMerge:
+  - patch_image_name.yaml
+
+namespace: network-test-f22e8ea9

kustomize/network/calico-k8s/network.yaml

@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name:  network-system
+
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: network-manager
+  namespace: network-system
+  labels:
+    control-plane: network-manager
+spec:
+  selector:
+    matchLabels:
+      control-plane: network-manager
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        control-plane: network-manager
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+        - key: "CriticalAddonsOnly"
+          operator: "Exists"
+        - key: "node-role.kubernetes.io/master"
+          effect: NoSchedule
+      serviceAccountName: network-manager
+      containers:
+      - command:
+        - /ks-network
+        args:
+          - -v=4
+          - np-provider=calico
+          - datastore-type=k8s
+        image: network:latest
+        imagePullPolicy: Always
+        name: manager
+        resources:
+          limits:
+            cpu: 100m
+            memory: 30Mi
+          requests:
+            cpu: 100m
+            memory: 20Mi
+      terminationGracePeriodSeconds: 10
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: net-role-binding
+  namespace: network-system
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: network-manager
+subjects:
+- kind: ServiceAccount
+  name: network-manager
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: network-manager

kustomize/network/calico-k8s/patch_image_name.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: network-manager
+  namespace: network-system
+spec:
+  template:
+    spec:
+      containers:
+      # Change the value of image field below to your controller image URL
+      - image: magicsong/ks-network:f22e8ea9
+        name: manager

kustomize/network/calico-k8s/patch_role_binding.yaml

@@ -0,0 +1,8 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: net-role-binding
+subjects:
+- kind: ServiceAccount
+  name: network-manager
+  namespace: network-test-f22e8ea9

kustomize/network/calico-k8s/role.yaml

@@ -0,0 +1,54 @@
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  creationTimestamp: null
+  name: network-manager
+rules:
+- apiGroups:
+  - crd.projectcalico.org
+  resources:
+  - clusterinformations
+  - felixconfigurations
+  - globalfelixconfigs
+  - globalnetworkpolicies
+  - globalnetworksets
+  - hostendpoints
+  - ipamblocks
+  - ippools
+  - networkpolicies
+  - networksets
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - network.kubesphere.io
+  resources:
+  - namespacenetworkpolicies
+  - workspacenetworkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - tenant.kubesphere.io
+  resources:
+  - workspaces
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

kustomize/network/crds/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+  - wsnp.yaml
+  - nsnp.yaml

kustomize/network/crds/nsnp.yaml

@@ -0,0 +1,711 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: namespacenetworkpolicies.network.kubesphere.io
+spec:
+  group: network.kubesphere.io
+  names:
+    categories:
+    - networking
+    kind: NamespaceNetworkPolicy
+    plural: namespacenetworkpolicies
+    shortNames:
+    - nsnp
+  scope: Namespaced
+  validation:
+    openAPIV3Schema:
+      description: NamespaceNetworkPolicy is the Schema for the namespacenetworkpolicies
+        API
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: NamespaceNetworkPolicySpec defines the desired state of NamespaceNetworkPolicy
+          properties:
+            egress:
+              description: The ordered set of egress rules.  Each rule contains a
+                set of packet match criteria and a corresponding action to apply.
+              items:
+                description: "A Rule encapsulates a set of match criteria and an action.
+                  \ Both selector-based security Policy and security Profiles reference
+                  rules - separated out as a list of rules for both ingress and egress
+                  packet matching. \n Each positive match criteria has a negated version,
+                  prefixed with ”Not”. All the match criteria within a rule must be
+                  satisfied for a packet to match. A single rule can contain the positive
+                  and negative version of a match and both must be satisfied for the
+                  rule to match."
+                properties:
+                  action:
+                    type: string
+                  destination:
+                    description: Destination contains the match criteria that apply
+                      to destination entity.
+                    properties:
+                      namespaceSelector:
+                        description: "NamespaceSelector is an optional field that
+                          contains a selector expression. Only traffic that originates
+                          from (or terminates at) endpoints within the selected namespaces
+                          will be matched. When both NamespaceSelector and Selector
+                          are defined on the same rule, then only workload endpoints
+                          that are matched by both selectors will be selected by the
+                          rule. \n For NetworkPolicy, an empty NamespaceSelector implies
+                          that the Selector is limited to selecting only workload
+                          endpoints in the same namespace as the NetworkPolicy. \n
+                          For GlobalNetworkPolicy, an empty NamespaceSelector implies
+                          the Selector applies to workload endpoints across all namespaces."
+                        type: string
+                      nets:
+                        description: Nets is an optional field that restricts the
+                          rule to only apply to traffic that originates from (or terminates
+                          at) IP addresses in any of the given subnets.
+                        items:
+                          type: string
+                        type: array
+                      notNets:
+                        description: NotNets is the negated version of the Nets field.
+                        items:
+                          type: string
+                        type: array
+                      notPorts:
+                        items:
+                          type: object
+                          x-kubernetes-int-or-string: true
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                        description: NotPorts is the negated version of the Ports
+                          field. Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to "TCP" or "UDP".
+                        type: array
+                      notSelector:
+                        description: NotSelector is the negated version of the Selector
+                          field.  See Selector field for subtleties with negated selectors.
+                        type: string
+                      ports:
+                        description: "Ports is an optional field that restricts the
+                          rule to only apply to traffic that has a source (destination)
+                          port that matches one of these ranges/values. This value
+                          is a list of integers or strings that represent ranges of
+                          ports. \n Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to \"TCP\" or \"UDP\"."
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          type: object
+                        type: array
+                      selector:
+                        description: "Selector is an optional field that contains
+                          a selector expression (see Policy for sample syntax).  Only
+                          traffic that originates from (terminates at) endpoints matching
+                          the selector will be matched. \n Note that: in addition
+                          to the negated version of the Selector (see NotSelector
+                          below), the selector expression syntax itself supports negation.
+                          \ The two types of negation are subtly different. One negates
+                          the set of matched endpoints, the other negates the whole
+                          match: \n \tSelector = \"!has(my_label)\" matches packets
+                          that are from other Calico-controlled \tendpoints that do
+                          not have the label “my_label”. \n \tNotSelector = \"has(my_label)\"
+                          matches packets that are not from Calico-controlled \tendpoints
+                          that do have the label “my_label”. \n The effect is that
+                          the latter will accept packets from non-Calico sources whereas
+                          the former is limited to packets from Calico-controlled
+                          endpoints."
+                        type: string
+                      serviceAccounts:
+                        description: ServiceAccounts is an optional field that restricts
+                          the rule to only apply to traffic that originates from (or
+                          terminates at) a pod running as a matching service account.
+                        properties:
+                          names:
+                            description: Names is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              whose name is in the list.
+                            items:
+                              type: string
+                            type: array
+                          selector:
+                            description: Selector is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              that matches the given label selector. If both Names
+                              and Selector are specified then they are AND'ed.
+                            type: string
+                        type: object
+                    type: object
+                  http:
+                    description: HTTP contains match criteria that apply to HTTP requests.
+                    properties:
+                      methods:
+                        description: Methods is an optional field that restricts the
+                          rule to apply only to HTTP requests that use one of the
+                          listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods
+                          are OR'd together.
+                        items:
+                          type: string
+                        type: array
+                      paths:
+                        description: 'Paths is an optional field that restricts the
+                          rule to apply to HTTP requests that use one of the listed
+                          HTTP Paths. Multiple paths are OR''d together. e.g: - exact:
+                          /foo - prefix: /bar NOTE: Each entry may ONLY specify either
+                          a `exact` or a `prefix` match. The validator will check
+                          for it.'
+                        items:
+                          description: 'HTTPPath specifies an HTTP path to match.
+                            It may be either of the form: exact: <path>: which matches
+                            the path exactly or prefix: <path-prefix>: which matches
+                            the path prefix'
+                          properties:
+                            exact:
+                              type: string
+                            prefix:
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                  icmp:
+                    description: ICMP is an optional field that restricts the rule
+                      to apply to a specific type and code of ICMP traffic.  This
+                      should only be specified if the Protocol field is set to "ICMP"
+                      or "ICMPv6".
+                    properties:
+                      code:
+                        description: Match on a specific ICMP code.  If specified,
+                          the Type value must also be specified. This is a technical
+                          limitation imposed by the kernel’s iptables firewall, which
+                          Calico uses to enforce the rule.
+                        type: integer
+                      type:
+                        description: Match on a specific ICMP type.  For example a
+                          value of 8 refers to ICMP Echo Request (i.e. pings).
+                        type: integer
+                    type: object
+                  ipVersion:
+                    description: IPVersion is an optional field that restricts the
+                      rule to only match a specific IP version.
+                    type: integer
+                  notICMP:
+                    description: NotICMP is the negated version of the ICMP field.
+                    properties:
+                      code:
+                        description: Match on a specific ICMP code.  If specified,
+                          the Type value must also be specified. This is a technical
+                          limitation imposed by the kernel’s iptables firewall, which
+                          Calico uses to enforce the rule.
+                        type: integer
+                      type:
+                        description: Match on a specific ICMP type.  For example a
+                          value of 8 refers to ICMP Echo Request (i.e. pings).
+                        type: integer
+                    type: object
+                  notProtocol:
+                    description: NotProtocol is the negated version of the Protocol
+                      field.
+                    type: string
+                  protocol:
+                    description: "Protocol is an optional field that restricts the
+                      rule to only apply to traffic of a specific IP protocol. Required
+                      if any of the EntityRules contain Ports (because ports only
+                      apply to certain protocols). \n Must be one of these string
+                      values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\"
+                      or an integer in the range 1-255."
+                    type: string
+                  source:
+                    description: Source contains the match criteria that apply to
+                      source entity.
+                    properties:
+                      namespaceSelector:
+                        description: "NamespaceSelector is an optional field that
+                          contains a selector expression. Only traffic that originates
+                          from (or terminates at) endpoints within the selected namespaces
+                          will be matched. When both NamespaceSelector and Selector
+                          are defined on the same rule, then only workload endpoints
+                          that are matched by both selectors will be selected by the
+                          rule. \n For NetworkPolicy, an empty NamespaceSelector implies
+                          that the Selector is limited to selecting only workload
+                          endpoints in the same namespace as the NetworkPolicy. \n
+                          For GlobalNetworkPolicy, an empty NamespaceSelector implies
+                          the Selector applies to workload endpoints across all namespaces."
+                        type: string
+                      nets:
+                        description: Nets is an optional field that restricts the
+                          rule to only apply to traffic that originates from (or terminates
+                          at) IP addresses in any of the given subnets.
+                        items:
+                          type: string
+                        type: array
+                      notNets:
+                        description: NotNets is the negated version of the Nets field.
+                        items:
+                          type: string
+                        type: array
+                      notPorts:
+                        description: NotPorts is the negated version of the Ports
+                          field. Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to "TCP" or "UDP".
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          type: object
+                        type: array
+                      notSelector:
+                        description: NotSelector is the negated version of the Selector
+                          field.  See Selector field for subtleties with negated selectors.
+                        type: string
+                      ports:
+                        description: "Ports is an optional field that restricts the
+                          rule to only apply to traffic that has a source (destination)
+                          port that matches one of these ranges/values. This value
+                          is a list of integers or strings that represent ranges of
+                          ports. \n Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to \"TCP\" or \"UDP\"."
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          type: object
+                        type: array
+                      selector:
+                        description: "Selector is an optional field that contains
+                          a selector expression (see Policy for sample syntax).  Only
+                          traffic that originates from (terminates at) endpoints matching
+                          the selector will be matched. \n Note that: in addition
+                          to the negated version of the Selector (see NotSelector
+                          below), the selector expression syntax itself supports negation.
+                          \ The two types of negation are subtly different. One negates
+                          the set of matched endpoints, the other negates the whole
+                          match: \n \tSelector = \"!has(my_label)\" matches packets
+                          that are from other Calico-controlled \tendpoints that do
+                          not have the label “my_label”. \n \tNotSelector = \"has(my_label)\"
+                          matches packets that are not from Calico-controlled \tendpoints
+                          that do have the label “my_label”. \n The effect is that
+                          the latter will accept packets from non-Calico sources whereas
+                          the former is limited to packets from Calico-controlled
+                          endpoints."
+                        type: string
+                      serviceAccounts:
+                        description: ServiceAccounts is an optional field that restricts
+                          the rule to only apply to traffic that originates from (or
+                          terminates at) a pod running as a matching service account.
+                        properties:
+                          names:
+                            description: Names is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              whose name is in the list.
+                            items:
+                              type: string
+                            type: array
+                          selector:
+                            description: Selector is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              that matches the given label selector. If both Names
+                              and Selector are specified then they are AND'ed.
+                            type: string
+                        type: object
+                    type: object
+                required:
+                - action
+                type: object
+              type: array
+            ingress:
+              description: The ordered set of ingress rules.  Each rule contains a
+                set of packet match criteria and a corresponding action to apply.
+              items:
+                description: "A Rule encapsulates a set of match criteria and an action.
+                  \ Both selector-based security Policy and security Profiles reference
+                  rules - separated out as a list of rules for both ingress and egress
+                  packet matching. \n Each positive match criteria has a negated version,
+                  prefixed with ”Not”. All the match criteria within a rule must be
+                  satisfied for a packet to match. A single rule can contain the positive
+                  and negative version of a match and both must be satisfied for the
+                  rule to match."
+                properties:
+                  action:
+                    type: string
+                  destination:
+                    description: Destination contains the match criteria that apply
+                      to destination entity.
+                    properties:
+                      namespaceSelector:
+                        description: "NamespaceSelector is an optional field that
+                          contains a selector expression. Only traffic that originates
+                          from (or terminates at) endpoints within the selected namespaces
+                          will be matched. When both NamespaceSelector and Selector
+                          are defined on the same rule, then only workload endpoints
+                          that are matched by both selectors will be selected by the
+                          rule. \n For NetworkPolicy, an empty NamespaceSelector implies
+                          that the Selector is limited to selecting only workload
+                          endpoints in the same namespace as the NetworkPolicy. \n
+                          For GlobalNetworkPolicy, an empty NamespaceSelector implies
+                          the Selector applies to workload endpoints across all namespaces."
+                        type: string
+                      nets:
+                        description: Nets is an optional field that restricts the
+                          rule to only apply to traffic that originates from (or terminates
+                          at) IP addresses in any of the given subnets.
+                        items:
+                          type: string
+                        type: array
+                      notNets:
+                        description: NotNets is the negated version of the Nets field.
+                        items:
+                          type: string
+                        type: array
+                      notPorts:
+                        description: NotPorts is the negated version of the Ports
+                          field. Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to "TCP" or "UDP".
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          anyOf:
+                          - type: integer
+                          - type: string
+                        type: array
+                      notSelector:
+                        description: NotSelector is the negated version of the Selector
+                          field.  See Selector field for subtleties with negated selectors.
+                        type: string
+                      ports:
+                        description: "Ports is an optional field that restricts the
+                          rule to only apply to traffic that has a source (destination)
+                          port that matches one of these ranges/values. This value
+                          is a list of integers or strings that represent ranges of
+                          ports. \n Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to \"TCP\" or \"UDP\"."
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          anyOf:
+                          - type: integer
+                          - type: string
+                        type: array
+                      selector:
+                        description: "Selector is an optional field that contains
+                          a selector expression (see Policy for sample syntax).  Only
+                          traffic that originates from (terminates at) endpoints matching
+                          the selector will be matched. \n Note that: in addition
+                          to the negated version of the Selector (see NotSelector
+                          below), the selector expression syntax itself supports negation.
+                          \ The two types of negation are subtly different. One negates
+                          the set of matched endpoints, the other negates the whole
+                          match: \n \tSelector = \"!has(my_label)\" matches packets
+                          that are from other Calico-controlled \tendpoints that do
+                          not have the label “my_label”. \n \tNotSelector = \"has(my_label)\"
+                          matches packets that are not from Calico-controlled \tendpoints
+                          that do have the label “my_label”. \n The effect is that
+                          the latter will accept packets from non-Calico sources whereas
+                          the former is limited to packets from Calico-controlled
+                          endpoints."
+                        type: string
+                      serviceAccounts:
+                        description: ServiceAccounts is an optional field that restricts
+                          the rule to only apply to traffic that originates from (or
+                          terminates at) a pod running as a matching service account.
+                        properties:
+                          names:
+                            description: Names is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              whose name is in the list.
+                            items:
+                              type: string
+                            type: array
+                          selector:
+                            description: Selector is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              that matches the given label selector. If both Names
+                              and Selector are specified then they are AND'ed.
+                            type: string
+                        type: object
+                    type: object
+                  http:
+                    description: HTTP contains match criteria that apply to HTTP requests.
+                    properties:
+                      methods:
+                        description: Methods is an optional field that restricts the
+                          rule to apply only to HTTP requests that use one of the
+                          listed HTTP Methods (e.g. GET, PUT, etc.) Multiple methods
+                          are OR'd together.
+                        items:
+                          type: string
+                        type: array
+                      paths:
+                        description: 'Paths is an optional field that restricts the
+                          rule to apply to HTTP requests that use one of the listed
+                          HTTP Paths. Multiple paths are OR''d together. e.g: - exact:
+                          /foo - prefix: /bar NOTE: Each entry may ONLY specify either
+                          a `exact` or a `prefix` match. The validator will check
+                          for it.'
+                        items:
+                          description: 'HTTPPath specifies an HTTP path to match.
+                            It may be either of the form: exact: <path>: which matches
+                            the path exactly or prefix: <path-prefix>: which matches
+                            the path prefix'
+                          properties:
+                            exact:
+                              type: string
+                            prefix:
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                  icmp:
+                    description: ICMP is an optional field that restricts the rule
+                      to apply to a specific type and code of ICMP traffic.  This
+                      should only be specified if the Protocol field is set to "ICMP"
+                      or "ICMPv6".
+                    properties:
+                      code:
+                        description: Match on a specific ICMP code.  If specified,
+                          the Type value must also be specified. This is a technical
+                          limitation imposed by the kernel’s iptables firewall, which
+                          Calico uses to enforce the rule.
+                        type: integer
+                      type:
+                        description: Match on a specific ICMP type.  For example a
+                          value of 8 refers to ICMP Echo Request (i.e. pings).
+                        type: integer
+                    type: object
+                  ipVersion:
+                    description: IPVersion is an optional field that restricts the
+                      rule to only match a specific IP version.
+                    type: integer
+                  notICMP:
+                    description: NotICMP is the negated version of the ICMP field.
+                    properties:
+                      code:
+                        description: Match on a specific ICMP code.  If specified,
+                          the Type value must also be specified. This is a technical
+                          limitation imposed by the kernel’s iptables firewall, which
+                          Calico uses to enforce the rule.
+                        type: integer
+                      type:
+                        description: Match on a specific ICMP type.  For example a
+                          value of 8 refers to ICMP Echo Request (i.e. pings).
+                        type: integer
+                    type: object
+                  notProtocol:
+                    description: NotProtocol is the negated version of the Protocol
+                      field.
+                    type: string
+                  protocol:
+                    description: "Protocol is an optional field that restricts the
+                      rule to only apply to traffic of a specific IP protocol. Required
+                      if any of the EntityRules contain Ports (because ports only
+                      apply to certain protocols). \n Must be one of these string
+                      values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", \"UDPLite\"
+                      or an integer in the range 1-255."
+                    type: string
+                  source:
+                    description: Source contains the match criteria that apply to
+                      source entity.
+                    properties:
+                      namespaceSelector:
+                        description: "NamespaceSelector is an optional field that
+                          contains a selector expression. Only traffic that originates
+                          from (or terminates at) endpoints within the selected namespaces
+                          will be matched. When both NamespaceSelector and Selector
+                          are defined on the same rule, then only workload endpoints
+                          that are matched by both selectors will be selected by the
+                          rule. \n For NetworkPolicy, an empty NamespaceSelector implies
+                          that the Selector is limited to selecting only workload
+                          endpoints in the same namespace as the NetworkPolicy. \n
+                          For GlobalNetworkPolicy, an empty NamespaceSelector implies
+                          the Selector applies to workload endpoints across all namespaces."
+                        type: string
+                      nets:
+                        description: Nets is an optional field that restricts the
+                          rule to only apply to traffic that originates from (or terminates
+                          at) IP addresses in any of the given subnets.
+                        items:
+                          type: string
+                        type: array
+                      notNets:
+                        description: NotNets is the negated version of the Nets field.
+                        items:
+                          type: string
+                        type: array
+                      notPorts:
+                        description: NotPorts is the negated version of the Ports
+                          field. Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to "TCP" or "UDP".
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          type: object
+                        type: array
+                      notSelector:
+                        description: NotSelector is the negated version of the Selector
+                          field.  See Selector field for subtleties with negated selectors.
+                        type: string
+                      ports:
+                        description: "Ports is an optional field that restricts the
+                          rule to only apply to traffic that has a source (destination)
+                          port that matches one of these ranges/values. This value
+                          is a list of integers or strings that represent ranges of
+                          ports. \n Since only some protocols have ports, if any ports
+                          are specified it requires the Protocol match in the Rule
+                          to be set to \"TCP\" or \"UDP\"."
+                        items:
+                          description: "Port represents either a range of numeric
+                            ports or a named port. \n     - For a named port, set
+                            the PortName, leaving MinPort and MaxPort as 0.     -
+                            For a port range, set MinPort and MaxPort to the (inclusive)
+                            port numbers.  Set       PortName to \"\".     - For a
+                            single port, set MinPort = MaxPort and PortName = \"\"."
+                          x-kubernetes-int-or-string: true
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          type: object
+                        type: array
+                      selector:
+                        description: "Selector is an optional field that contains
+                          a selector expression (see Policy for sample syntax).  Only
+                          traffic that originates from (terminates at) endpoints matching
+                          the selector will be matched. \n Note that: in addition
+                          to the negated version of the Selector (see NotSelector
+                          below), the selector expression syntax itself supports negation.
+                          \ The two types of negation are subtly different. One negates
+                          the set of matched endpoints, the other negates the whole
+                          match: \n \tSelector = \"!has(my_label)\" matches packets
+                          that are from other Calico-controlled \tendpoints that do
+                          not have the label “my_label”. \n \tNotSelector = \"has(my_label)\"
+                          matches packets that are not from Calico-controlled \tendpoints
+                          that do have the label “my_label”. \n The effect is that
+                          the latter will accept packets from non-Calico sources whereas
+                          the former is limited to packets from Calico-controlled
+                          endpoints."
+                        type: string
+                      serviceAccounts:
+                        description: ServiceAccounts is an optional field that restricts
+                          the rule to only apply to traffic that originates from (or
+                          terminates at) a pod running as a matching service account.
+                        properties:
+                          names:
+                            description: Names is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              whose name is in the list.
+                            items:
+                              type: string
+                            type: array
+                          selector:
+                            description: Selector is an optional field that restricts
+                              the rule to only apply to traffic that originates from
+                              (or terminates at) a pod running as a service account
+                              that matches the given label selector. If both Names
+                              and Selector are specified then they are AND'ed.
+                            type: string
+                        type: object
+                    type: object
+                required:
+                - action
+                type: object
+              type: array
+            order:
+              description: Order is an optional field that specifies the order in
+                which the policy is applied. Policies with higher "order" are applied
+                after those with lower order.  If the order is omitted, it may be
+                considered to be "infinite" - i.e. the policy will be applied last.  Policies
+                with identical order will be applied in alphanumerical order based
+                on the Policy "Name".
+              type: integer
+            selector:
+              description: "The selector is an expression used to pick pick out the
+                endpoints that the policy should be applied to. \n Selector expressions
+                follow this syntax: \n \tlabel == \"string_literal\"  ->  comparison,
+                e.g. my_label == \"foo bar\" \tlabel != \"string_literal\"   ->  not
+                equal; also matches if label is not present \tlabel in { \"a\", \"b\",
+                \"c\", ... }  ->  true if the value of label X is one of \"a\", \"b\",
+                \"c\" \tlabel not in { \"a\", \"b\", \"c\", ... }  ->  true if the
+                value of label X is not one of \"a\", \"b\", \"c\" \thas(label_name)
+                \ -> True if that label is present \t! expr -> negation of expr \texpr
+                && expr  -> Short-circuit and \texpr || expr  -> Short-circuit or
+                \t( expr ) -> parens for grouping \tall() or the empty selector ->
+                matches all endpoints. \n Label names are allowed to contain alphanumerics,
+                -, _ and /. String literals are more permissive but they do not support
+                escape characters. \n Examples (with made-up labels): \n \ttype ==
+                \"webserver\" && deployment == \"prod\" \ttype in {\"frontend\", \"backend\"}
+                \tdeployment != \"dev\" \t! has(label_name)"
+              type: string
+            types:
+              description: "Types indicates whether this policy applies to ingress,
+                or to egress, or to both.  When not explicitly specified (and so the
+                value on creation is empty or nil), Calico defaults Types according
+                to what Ingress and Egress are present in the policy.  The default
+                is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including
+                the case where there are   also no Ingress rules) \n - [ PolicyTypeEgress
+                ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress,
+                PolicyTypeEgress ], if there are both Ingress and Egress rules. \n
+                When the policy is read back again, Types will always be one of these
+                values, never empty or nil."
+              items:
+                type: string
+              type: array
+          required:
+          - selector
+          type: object
+      type: object
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

kustomize/network/crds/wsnp.yaml

@@ -0,0 +1,523 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  creationTimestamp: null
+  name: workspacenetworkpolicies.network.kubesphere.io
+spec:
+  group: network.kubesphere.io
+  names:
+    categories:
+    - networking
+    kind: WorkspaceNetworkPolicy
+    plural: workspacenetworkpolicies
+    shortNames:
+    - wsnp
+  scope: Cluster
+  validation:
+    openAPIV3Schema:
+      description: WorkspaceNetworkPolicy is a set of network policies applied to
+        the scope to workspace
+      properties:
+        apiVersion:
+          description: 'APIVersion defines the versioned schema of this representation
+            of an object. Servers should convert recognized schemas to the latest
+            internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
+          type: string
+        kind:
+          description: 'Kind is a string value representing the REST resource this
+            object represents. Servers may infer this from the endpoint the client
+            submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
+          type: string
+        metadata:
+          type: object
+        spec:
+          description: WorkspaceNetworkPolicySpec defines the desired state of WorkspaceNetworkPolicy
+          properties:
+            egress:
+              description: List of egress rules to be applied to the selected pods.
+                Outgoing traffic is allowed if there are no NetworkPolicies selecting
+                the pod (and cluster policy otherwise allows the traffic), OR if the
+                traffic matches at least one egress rule across all of the NetworkPolicy
+                objects whose podSelector matches the pod. If this field is empty
+                then this NetworkPolicy limits all outgoing traffic (and serves solely
+                to ensure that the pods it selects are isolated by default). This
+                field is beta-level in 1.8
+              items:
+                description: WorkspaceNetworkPolicyEgressRule describes a particular
+                  set of traffic that is allowed out of pods matched by a WorkspaceNetworkPolicySpec's
+                  podSelector. The traffic must match both ports and to.
+                properties:
+                  from:
+                    description: List of sources which should be able to access the
+                      pods selected for this rule. Items in this list are combined
+                      using a logical OR operation. If this field is empty or missing,
+                      this rule matches all sources (traffic not restricted by source).
+                      If this field is present and contains at least on item, this
+                      rule allows traffic only if the traffic matches at least one
+                      item in the from list.
+                    items:
+                      description: WorkspaceNetworkPolicyPeer describes a peer to
+                        allow traffic from. Only certain combinations of fields are
+                        allowed. It is same as 'NetworkPolicyPeer' in k8s but with
+                        an additional field 'WorkspaceSelector'
+                      properties:
+                        ipBlock:
+                          description: IPBlock defines policy on a particular IPBlock.
+                            If this field is set then neither of the other fields
+                            can be.
+                          properties:
+                            cidr:
+                              description: CIDR is a string representing the IP Block
+                                Valid examples are "192.168.1.1/24"
+                              type: string
+                            except:
+                              description: Except is a slice of CIDRs that should
+                                not be included within an IP Block Valid examples
+                                are "192.168.1.1/24" Except values will be rejected
+                                if they are outside the CIDR range
+                              items:
+                                type: string
+                              type: array
+                          required:
+                          - cidr
+                          type: object
+                        namespaceSelector:
+                          description: "Selects Namespaces using cluster-scoped labels.
+                            This field follows standard label selector semantics;
+                            if present but empty, it selects all namespaces. \n If
+                            PodSelector is also set, then the NetworkPolicyPeer as
+                            a whole selects the Pods matching PodSelector in the Namespaces
+                            selected by NamespaceSelector. Otherwise it selects all
+                            Pods in the Namespaces selected by NamespaceSelector."
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                        podSelector:
+                          description: "This is a label selector which selects Pods.
+                            This field follows standard label selector semantics;
+                            if present but empty, it selects all pods. \n If NamespaceSelector
+                            is also set, then the NetworkPolicyPeer as a whole selects
+                            the Pods matching PodSelector in the Namespaces selected
+                            by NamespaceSelector. Otherwise it selects the Pods matching
+                            PodSelector in the policy's own Namespace."
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                        workspaceSelector:
+                          description: A label selector is a label query over a set
+                            of resources. The result of matchLabels and matchExpressions
+                            are ANDed. An empty label selector matches all objects.
+                            A null label selector matches no objects.
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                      type: object
+                    type: array
+                  ports:
+                    description: List of ports which should be made accessible on
+                      the pods selected for this rule. Each item in this list is combined
+                      using a logical OR. If this field is empty or missing, this
+                      rule matches all ports (traffic not restricted by port). If
+                      this field is present and contains at least one item, then this
+                      rule allows traffic only if the traffic matches at least one
+                      port in the list.
+                    items:
+                      description: NetworkPolicyPort describes a port to allow traffic
+                        on
+                      properties:
+                        port:
+                          anyOf:
+                          - type: string
+                          - type: integer
+                          description: The port on the given protocol. This can either
+                            be a numerical or named port on a pod. If this field is
+                            not provided, this matches all port names and numbers.
+                        protocol:
+                          description: The protocol (TCP, UDP, or SCTP) which traffic
+                            must match. If not specified, this field defaults to TCP.
+                          type: string
+                      type: object
+                    type: array
+                type: object
+              type: array
+            ingress:
+              description: List of ingress rules to be applied to the selected pods.
+                Traffic is allowed to a pod if there are no NetworkPolicies selecting
+                the pod (and cluster policy otherwise allows the traffic), OR if the
+                traffic source is the pod's local node, OR if the traffic matches
+                at least one ingress rule across all of the NetworkPolicy objects
+                whose podSelector matches the pod. If this field is empty then this
+                NetworkPolicy does not allow any traffic (and serves solely to ensure
+                that the pods it selects are isolated by default)
+              items:
+                description: WorkspaceNetworkPolicyIngressRule describes a particular
+                  set of traffic that is allowed to the pods matched by a WorkspaceNetworkPolicySpec's
+                  podSelector. The traffic must match both ports and from.
+                properties:
+                  from:
+                    description: List of sources which should be able to access the
+                      pods selected for this rule. Items in this list are combined
+                      using a logical OR operation. If this field is empty or missing,
+                      this rule matches all sources (traffic not restricted by source).
+                      If this field is present and contains at least on item, this
+                      rule allows traffic only if the traffic matches at least one
+                      item in the from list.
+                    items:
+                      description: WorkspaceNetworkPolicyPeer describes a peer to
+                        allow traffic from. Only certain combinations of fields are
+                        allowed. It is same as 'NetworkPolicyPeer' in k8s but with
+                        an additional field 'WorkspaceSelector'
+                      properties:
+                        ipBlock:
+                          description: IPBlock defines policy on a particular IPBlock.
+                            If this field is set then neither of the other fields
+                            can be.
+                          properties:
+                            cidr:
+                              description: CIDR is a string representing the IP Block
+                                Valid examples are "192.168.1.1/24"
+                              type: string
+                            except:
+                              description: Except is a slice of CIDRs that should
+                                not be included within an IP Block Valid examples
+                                are "192.168.1.1/24" Except values will be rejected
+                                if they are outside the CIDR range
+                              items:
+                                type: string
+                              type: array
+                          required:
+                          - cidr
+                          type: object
+                        namespaceSelector:
+                          description: "Selects Namespaces using cluster-scoped labels.
+                            This field follows standard label selector semantics;
+                            if present but empty, it selects all namespaces. \n If
+                            PodSelector is also set, then the NetworkPolicyPeer as
+                            a whole selects the Pods matching PodSelector in the Namespaces
+                            selected by NamespaceSelector. Otherwise it selects all
+                            Pods in the Namespaces selected by NamespaceSelector."
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                        podSelector:
+                          description: "This is a label selector which selects Pods.
+                            This field follows standard label selector semantics;
+                            if present but empty, it selects all pods. \n If NamespaceSelector
+                            is also set, then the NetworkPolicyPeer as a whole selects
+                            the Pods matching PodSelector in the Namespaces selected
+                            by NamespaceSelector. Otherwise it selects the Pods matching
+                            PodSelector in the policy's own Namespace."
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                        workspaceSelector:
+                          description: A label selector is a label query over a set
+                            of resources. The result of matchLabels and matchExpressions
+                            are ANDed. An empty label selector matches all objects.
+                            A null label selector matches no objects.
+                          properties:
+                            matchExpressions:
+                              description: matchExpressions is a list of label selector
+                                requirements. The requirements are ANDed.
+                              items:
+                                description: A label selector requirement is a selector
+                                  that contains values, a key, and an operator that
+                                  relates the key and values.
+                                properties:
+                                  key:
+                                    description: key is the label key that the selector
+                                      applies to.
+                                    type: string
+                                  operator:
+                                    description: operator represents a key's relationship
+                                      to a set of values. Valid operators are In,
+                                      NotIn, Exists and DoesNotExist.
+                                    type: string
+                                  values:
+                                    description: values is an array of string values.
+                                      If the operator is In or NotIn, the values array
+                                      must be non-empty. If the operator is Exists
+                                      or DoesNotExist, the values array must be empty.
+                                      This array is replaced during a strategic merge
+                                      patch.
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              description: matchLabels is a map of {key,value} pairs.
+                                A single {key,value} in the matchLabels map is equivalent
+                                to an element of matchExpressions, whose key field
+                                is "key", the operator is "In", and the values array
+                                contains only "value". The requirements are ANDed.
+                              type: object
+                          type: object
+                      type: object
+                    type: array
+                  ports:
+                    description: List of ports which should be made accessible on
+                      the pods selected for this rule. Each item in this list is combined
+                      using a logical OR. If this field is empty or missing, this
+                      rule matches all ports (traffic not restricted by port). If
+                      this field is present and contains at least one item, then this
+                      rule allows traffic only if the traffic matches at least one
+                      port in the list.
+                    items:
+                      description: NetworkPolicyPort describes a port to allow traffic
+                        on
+                      properties:
+                        port:
+                          anyOf:
+                          - type: string
+                          - type: integer
+                          description: The port on the given protocol. This can either
+                            be a numerical or named port on a pod. If this field is
+                            not provided, this matches all port names and numbers.
+                        protocol:
+                          description: The protocol (TCP, UDP, or SCTP) which traffic
+                            must match. If not specified, this field defaults to TCP.
+                          type: string
+                      type: object
+                    type: array
+                type: object
+              type: array
+            policyTypes:
+              description: List of rule types that the WorkspaceNetworkPolicy relates
+                to. Valid options are Ingress, Egress, or Ingress,Egress. If this
+                field is not specified, it will default based on the existence of
+                Ingress or Egress rules; policies that contain an Egress section are
+                assumed to affect Egress, and all policies (whether or not they contain
+                an Ingress section) are assumed to affect Ingress. If you want to
+                write an egress-only policy, you must explicitly specify policyTypes
+                [ "Egress" ]. Likewise, if you want to write a policy that specifies
+                that no egress is allowed, you must specify a policyTypes value that
+                include "Egress" (since such a policy would not include an Egress
+                section and would otherwise default to just [ "Ingress" ]).
+              items:
+                description: Policy Type string describes the NetworkPolicy type This
+                  type is beta-level in 1.8
+                type: string
+              type: array
+            workspace:
+              description: Workspace specify the name of ws to apply this workspace
+                network policy
+              type: string
+          type: object
+        status:
+          description: WorkspaceNetworkPolicyStatus defines the observed state of
+            WorkspaceNetworkPolicy
+          type: object
+      type: object
+  version: v1alpha1
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

kustomize/network/rbac/role.yaml

@@ -0,0 +1,30 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: net-manager-role
+rules:
+- apiGroups:
+  - network.kubesphere.io
+  resources:
+  - namespacenetworkpolicies
+  - workspacenetworkpolicies
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - tenant.kubesphere.io
+  resources:
+  - workspaces
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch

pkg/api/devops/v1alpha2/types.go

@@ -0,0 +1,20 @@
+package v1alpha2
+
+import "time"
+
+type PageableDevOpsProject struct {
+	Items      []*DevOpsProject `json:"items"`
+	TotalCount int              `json:"total_count"`
+}
+
+type DevOpsProject struct {
+	ProjectId   string    `json:"project_id" db:"project_id" description:"ProjectId must be unique within a workspace, it is generated by kubesphere."`
+	Name        string    `json:"name" description:"DevOps Projects's Name"`
+	Description string    `json:"description,omitempty" description:"DevOps Projects's Description, used to describe the DevOps Project"`
+	Creator     string    `json:"creator" description:"Creator's username"`
+	CreateTime  time.Time `json:"create_time" description:"DevOps Project's Creation time"`
+	Status      string    `json:"status" description:"DevOps project's status. e.g. active"`
+	Visibility  string    `json:"visibility,omitempty" description:"Deprecated Field"`
+	Extra       string    `json:"extra,omitempty" description:"Internal Use"`
+	Workspace   string    `json:"workspace" description:"The workspace to which the devops project belongs"`
+}

pkg/api/logging/v1alpha2/types.go

@@ -0,0 +1,229 @@
+package v1alpha2
+
+import (
+	"encoding/json"
+	"time"
+)
+
+const (
+	OperationQuery int = iota
+	OperationStatistics
+	OperationHistogram
+	OperationExport
+)
+
+// elasticsearch client config
+type Config struct {
+	Host         string
+	Port         string
+	Index        string
+	VersionMajor string
+}
+
+type QueryParameters struct {
+	// when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace
+	NamespaceNotFound bool
+	// a map of namespace with creation time
+	NamespaceWithCreationTime map[string]string
+
+	// filter for literally matching
+	// query for fuzzy matching
+	WorkloadFilter  []string
+	WorkloadQuery   []string
+	PodFilter       []string
+	PodQuery        []string
+	ContainerFilter []string
+	ContainerQuery  []string
+	LogQuery        []string
+
+	Operation     int
+	Interval      string
+	StartTime     string
+	EndTime       string
+	Sort          string
+	From          int64
+	Size          int64
+	ScrollTimeout time.Duration
+}
+
+// elasticsearch request body
+type Request struct {
+	From      int64       `json:"from"`
+	Size      int64       `json:"size"`
+	Sorts     []Sort      `json:"sort,omitempty"`
+	MainQuery BoolQuery   `json:"query"`
+	Aggs      interface{} `json:"aggs,omitempty"`
+}
+
+type Sort struct {
+	Order Order `json:"time"`
+}
+
+type Order struct {
+	Order string `json:"order"`
+}
+
+type BoolQuery struct {
+	Bool interface{} `json:"bool"`
+}
+
+// user filter instead of must
+// filter ignores scoring
+type BoolFilter struct {
+	Filter []interface{} `json:"filter"`
+}
+
+type BoolShould struct {
+	Should             []interface{} `json:"should"`
+	MinimumShouldMatch int64         `json:"minimum_should_match"`
+}
+
+type RangeQuery struct {
+	RangeSpec RangeSpec `json:"range"`
+}
+
+type RangeSpec struct {
+	TimeRange TimeRange `json:"time"`
+}
+
+type TimeRange struct {
+	Gte string `json:"gte,omitempty"`
+	Lte string `json:"lte,omitempty"`
+}
+
+type MatchPhrase struct {
+	MatchPhrase map[string]string `json:"match_phrase"`
+}
+
+type MatchPhrasePrefix struct {
+	MatchPhrasePrefix interface{} `json:"match_phrase_prefix"`
+}
+
+type RegexpQuery struct {
+	Regexp interface{} `json:"regexp"`
+}
+
+// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting
+type StatisticsAggs struct {
+	ContainerAgg ContainerAgg `json:"containers"`
+}
+
+type ContainerAgg struct {
+	Cardinality AggField `json:"cardinality"`
+}
+
+type AggField struct {
+	Field string `json:"field"`
+}
+
+type HistogramAggs struct {
+	HistogramAgg HistogramAgg `json:"histogram"`
+}
+
+type HistogramAgg struct {
+	DateHistogram DateHistogram `json:"date_histogram"`
+}
+
+type DateHistogram struct {
+	Field    string `json:"field"`
+	Interval string `json:"interval"`
+}
+
+// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html
+// Response body from the elasticsearch engine
+type Response struct {
+	ScrollId     string          `json:"_scroll_id"`
+	Shards       Shards          `json:"_shards"`
+	Hits         Hits            `json:"hits"`
+	Aggregations json.RawMessage `json:"aggregations"`
+}
+
+type Shards struct {
+	Total      int64 `json:"total"`
+	Successful int64 `json:"successful"`
+	Skipped    int64 `json:"skipped"`
+	Failed     int64 `json:"failed"`
+}
+
+type Hits struct {
+	// As of ElasticSearch v7.x, hits.total is changed
+	Total interface{} `json:"total"`
+	Hits  []Hit       `json:"hits"`
+}
+
+type Hit struct {
+	Source Source  `json:"_source"`
+	Sort   []int64 `json:"sort"`
+}
+
+type Source struct {
+	Log        string     `json:"log"`
+	Time       string     `json:"time"`
+	Kubernetes Kubernetes `json:"kubernetes"`
+}
+
+type Kubernetes struct {
+	Namespace string `json:"namespace_name"`
+	Pod       string `json:"pod_name"`
+	Container string `json:"container_name"`
+	Host      string `json:"host"`
+}
+
+type LogRecord struct {
+	Time      string `json:"time,omitempty" description:"log timestamp"`
+	Log       string `json:"log,omitempty" description:"log message"`
+	Namespace string `json:"namespace,omitempty" description:"namespace"`
+	Pod       string `json:"pod,omitempty" description:"pod name"`
+	Container string `json:"container,omitempty" description:"container name"`
+	Host      string `json:"host,omitempty" description:"node id"`
+}
+
+type ReadResult struct {
+	ScrollID string      `json:"_scroll_id,omitempty"`
+	Total    int64       `json:"total" description:"total number of matched results"`
+	Records  []LogRecord `json:"records,omitempty" description:"actual array of results"`
+}
+
+// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs
+type StatisticsResponseAggregations struct {
+	ContainerCount ContainerCount `json:"containers"`
+}
+
+type ContainerCount struct {
+	Value int64 `json:"value"`
+}
+
+type HistogramAggregations struct {
+	HistogramAggregation HistogramAggregation `json:"histogram"`
+}
+
+type HistogramAggregation struct {
+	Histograms []HistogramStatistics `json:"buckets"`
+}
+
+type HistogramStatistics struct {
+	Time  int64 `json:"key"`
+	Count int64 `json:"doc_count"`
+}
+
+type HistogramRecord struct {
+	Time  int64 `json:"time" description:"timestamp"`
+	Count int64 `json:"count" description:"total number of logs at intervals"`
+}
+
+type StatisticsResult struct {
+	Containers int64 `json:"containers" description:"total number of containers"`
+	Logs       int64 `json:"logs" description:"total number of logs"`
+}
+
+type HistogramResult struct {
+	Total      int64             `json:"total" description:"total number of logs"`
+	Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"`
+}
+
+// Wrap elasticsearch response
+type QueryResult struct {
+	Read       *ReadResult       `json:"query,omitempty" description:"query results"`
+	Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"`
+	Histogram  *HistogramResult  `json:"histogram,omitempty" description:"histogram results"`
+}

pkg/api/monitoring/v1alpha2/types.go

@@ -0,0 +1,23 @@
+package v1alpha2
+
+// Prometheus query api response
+type APIResponse struct {
+	Status    string      `json:"status" description:"result status, one of error, success"`
+	Data      QueryResult `json:"data" description:"actual metric result"`
+	ErrorType string      `json:"errorType,omitempty"`
+	Error     string      `json:"error,omitempty"`
+	Warnings  []string    `json:"warnings,omitempty"`
+}
+
+// QueryResult includes result data from a query.
+type QueryResult struct {
+	ResultType string       `json:"resultType" description:"result type, one of matrix, vector"`
+	Result     []QueryValue `json:"result" description:"metric data including labels, time series and values"`
+}
+
+// Time Series
+type QueryValue struct {
+	Metric map[string]string `json:"metric,omitempty" description:"time series labels"`
+	Value  []interface{}     `json:"value,omitempty" description:"time series, values of vector type"`
+	Values [][]interface{}   `json:"values,omitempty" description:"time series, values of matrix type"`
+}

pkg/apigateway/caddy-plugin/authenticate/authenticate.go

@@ -23,24 +23,31 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal"
+	"kubesphere.io/kubesphere/pkg/simple/client/redis"
 	"log"
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/dgrijalva/jwt-go"
 	"github.com/mholt/caddy/caddyhttp/httpserver"
 )
 
 type Auth struct {
-	Rule Rule
+	Rule *Rule
 	Next httpserver.Handler
 }
 
 type Rule struct {
-	Secret       []byte
-	Path         string
-	ExceptedPath []string
+	Secret           []byte
+	Path             string
+	RedisOptions     *redis.RedisOptions
+	TokenIdleTimeout time.Duration
+	RedisClient      *redis.RedisClient
+	ExclusionRules   []internal.ExclusionRule
 }
 
 type User struct {
@@ -55,8 +62,8 @@ var requestInfoFactory = request.RequestInfoFactory{
 	GrouplessAPIPrefixes: sets.NewString("api")}
 
 func (h Auth) ServeHTTP(resp http.ResponseWriter, req *http.Request) (int, error) {
-	for _, path := range h.Rule.ExceptedPath {
-		if httpserver.Path(req.URL.Path).Matches(path) {
+	for _, rule := range h.Rule.ExclusionRules {
+		if httpserver.Path(req.URL.Path).Matches(rule.Path) && (rule.Method == internal.AllMethod || req.Method == rule.Method) {
 			return h.Next.ServeHTTP(resp, req)
 		}
 	}
@@ -87,7 +94,7 @@ func (h Auth) ServeHTTP(resp http.ResponseWriter, req *http.Request) (int, error
 
 func (h Auth) InjectContext(req *http.Request, token *jwt.Token) (*http.Request, error) {
 
-	payLoad, ok := token.Claims.(jwt.MapClaims)
+	payload, ok := token.Claims.(jwt.MapClaims)
 
 	if !ok {
 		return nil, errors.New("invalid payload")
@@ -101,14 +108,14 @@ func (h Auth) InjectContext(req *http.Request, token *jwt.Token) (*http.Request,
 
 	usr := &user.DefaultInfo{}
 
-	username, ok := payLoad["username"].(string)
+	username, ok := payload["username"].(string)
 
 	if ok && username != "" {
 		req.Header.Set("X-Token-Username", username)
 		usr.Name = username
 	}
 
-	uid := payLoad["uid"]
+	uid := payload["uid"]
 
 	if uid != nil {
 		switch uid.(type) {
@@ -123,7 +130,7 @@ func (h Auth) InjectContext(req *http.Request, token *jwt.Token) (*http.Request,
 		}
 	}
 
-	groups, ok := payLoad["groups"].([]string)
+	groups, ok := payload["groups"].([]string)
 	if ok && len(groups) > 0 {
 		req.Header.Set("X-Token-Groups", strings.Join(groups, ","))
 		usr.Groups = groups
@@ -160,10 +167,46 @@ func (h Auth) Validate(uToken string) (*jwt.Token, error) {
 	token, err := jwt.Parse(uToken, h.ProvideKey)
 
 	if err != nil {
+		klog.Errorln(err)
 		return nil, err
 	}
 
-	return token, nil
+	payload, ok := token.Claims.(jwt.MapClaims)
+
+	if !ok {
+		err := fmt.Errorf("invalid payload")
+		klog.Errorln(err)
+		return nil, err
+	}
+
+	username, ok := payload["username"].(string)
+
+	if !ok {
+		err := fmt.Errorf("invalid payload")
+		klog.Errorln(err)
+		return nil, err
+	}
+
+	if _, ok = payload["exp"]; ok {
+		// allow static token has expiration time
+		return token, nil
+	}
+
+	tokenKey := fmt.Sprintf("kubesphere:users:%s:token:%s", username, uToken)
+
+	exist, err := h.Rule.RedisClient.Redis().Exists(tokenKey).Result()
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	if exist == 1 {
+		// reset expiration time if token exist
+		h.Rule.RedisClient.Redis().Expire(tokenKey, h.Rule.TokenIdleTimeout)
+		return token, nil
+	} else {
+		return nil, errors.New("illegal token")
+	}
 }
 
 func (h Auth) HandleUnauthorized(w http.ResponseWriter, err error) int {

pkg/apigateway/caddy-plugin/authenticate/auto_load.go

@@ -19,19 +19,15 @@ package authenticate
 
 import (
 	"fmt"
-	"strings"
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal"
+	"kubesphere.io/kubesphere/pkg/simple/client/redis"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
+	"time"
 
 	"github.com/mholt/caddy"
 	"github.com/mholt/caddy/caddyhttp/httpserver"
 )
 
-func init() {
-	caddy.RegisterPlugin("authenticate", caddy.Plugin{
-		ServerType: "http",
-		Action:     Setup,
-	})
-}
-
 func Setup(c *caddy.Controller) error {
 
 	rule, err := parse(c)
@@ -41,20 +37,30 @@ func Setup(c *caddy.Controller) error {
 	}
 
 	c.OnStartup(func() error {
+		rule.RedisClient, err = redis.NewRedisClient(rule.RedisOptions, nil)
+		// ensure redis is connected  when startup
+		if err != nil {
+			return err
+		}
 		fmt.Println("Authenticate middleware is initiated")
 		return nil
 	})
 
+	c.OnShutdown(func() error {
+		return rule.RedisClient.Redis().Close()
+	})
+
 	httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler {
 		return &Auth{Next: next, Rule: rule}
 	})
 
 	return nil
 }
-func parse(c *caddy.Controller) (Rule, error) {
 
-	rule := Rule{ExceptedPath: make([]string, 0)}
+func parse(c *caddy.Controller) (*Rule, error) {
 
+	rule := &Rule{}
+	rule.ExclusionRules = make([]internal.ExclusionRule, 0)
 	if c.Next() {
 		args := c.RemainingArgs()
 		switch len(args) {
@@ -63,47 +69,83 @@ func parse(c *caddy.Controller) (Rule, error) {
 				switch c.Val() {
 				case "path":
 					if !c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
 					}
 
 					rule.Path = c.Val()
 
 					if c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
+					}
+				case "token-idle-timeout":
+					if !c.NextArg() {
+						return nil, c.ArgErr()
+					}
+
+					if timeout, err := time.ParseDuration(c.Val()); err != nil {
+						return nil, c.ArgErr()
+					} else {
+						rule.TokenIdleTimeout = timeout
+					}
+
+					if c.NextArg() {
+						return nil, c.ArgErr()
+					}
+				case "redis-url":
+					if !c.NextArg() {
+						return nil, c.ArgErr()
+					}
+
+					options := &redis.RedisOptions{RedisURL: c.Val()}
+
+					if err := options.Validate(); len(err) > 0 {
+						return nil, c.ArgErr()
+					} else {
+						rule.RedisOptions = options
+					}
+
+					if c.NextArg() {
+						return nil, c.ArgErr()
 					}
 				case "secret":
 					if !c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
 					}
 
 					rule.Secret = []byte(c.Val())
 
 					if c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
 					}
 				case "except":
+
 					if !c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
 					}
 
-					rule.ExceptedPath = strings.Split(c.Val(), ",")
+					method := c.Val()
 
-					for i := 0; i < len(rule.ExceptedPath); i++ {
-						rule.ExceptedPath[i] = strings.TrimSpace(rule.ExceptedPath[i])
+					if !sliceutil.HasString(internal.HttpMethods, method) {
+						return nil, c.ArgErr()
 					}
 
-					if c.NextArg() {
-						return rule, c.ArgErr()
+					for c.NextArg() {
+						path := c.Val()
+						rule.ExclusionRules = append(rule.ExclusionRules, internal.ExclusionRule{Method: method, Path: path})
 					}
 				}
 			}
 		default:
-			return rule, c.ArgErr()
+			return nil, c.ArgErr()
 		}
 	}
 
 	if c.Next() {
-		return rule, c.ArgErr()
+		return nil, c.ArgErr()
+	}
+
+	if rule.RedisOptions == nil {
+		return nil, c.Err("redis-url must be specified")
 	}
 
 	return rule, nil

pkg/apigateway/caddy-plugin/authentication/authentication.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/request"
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal"
 	"kubesphere.io/kubesphere/pkg/utils/k8sutil"
 	"log"
 	"net/http"
@@ -38,21 +39,21 @@ import (
 )
 
 type Authentication struct {
-	Rule Rule
+	Rule *Rule
 	Next httpserver.Handler
 }
 
 type Rule struct {
-	Path         string
-	ExceptedPath []string
+	Path           string
+	ExclusionRules []internal.ExclusionRule
 }
 
 func (c Authentication) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
 
 	if httpserver.Path(r.URL.Path).Matches(c.Rule.Path) {
 
-		for _, path := range c.Rule.ExceptedPath {
-			if httpserver.Path(r.URL.Path).Matches(path) {
+		for _, rule := range c.Rule.ExclusionRules {
+			if httpserver.Path(r.URL.Path).Matches(rule.Path) && (rule.Method == internal.AllMethod || r.Method == rule.Method) {
 				return c.Next.ServeHTTP(w, r)
 			}
 		}

pkg/apigateway/caddy-plugin/authentication/auto_load.go

@@ -19,21 +19,14 @@ package authentication
 
 import (
 	"fmt"
-	"strings"
-
 	"github.com/mholt/caddy"
 	"github.com/mholt/caddy/caddyhttp/httpserver"
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal"
+	"kubesphere.io/kubesphere/pkg/utils/sliceutil"
 
 	"kubesphere.io/kubesphere/pkg/informers"
 )
 
-func init() {
-	caddy.RegisterPlugin("authentication", caddy.Plugin{
-		ServerType: "http",
-		Action:     Setup,
-	})
-}
-
 // Setup is called by Caddy to parse the config block
 func Setup(c *caddy.Controller) error {
 
@@ -66,10 +59,10 @@ func Setup(c *caddy.Controller) error {
 	return nil
 }
 
-func parse(c *caddy.Controller) (Rule, error) {
-
-	rule := Rule{ExceptedPath: make([]string, 0)}
+func parse(c *caddy.Controller) (*Rule, error) {
 
+	rule := &Rule{}
+	rule.ExclusionRules = make([]internal.ExclusionRule, 0)
 	if c.Next() {
 		args := c.RemainingArgs()
 		switch len(args) {
@@ -90,17 +83,18 @@ func parse(c *caddy.Controller) (Rule, error) {
 					break
 				case "except":
 					if !c.NextArg() {
-						return rule, c.ArgErr()
+						return nil, c.ArgErr()
 					}
 
-					rule.ExceptedPath = strings.Split(c.Val(), ",")
+					method := c.Val()
 
-					for i := 0; i < len(rule.ExceptedPath); i++ {
-						rule.ExceptedPath[i] = strings.TrimSpace(rule.ExceptedPath[i])
+					if !sliceutil.HasString(internal.HttpMethods, method) {
+						return nil, c.ArgErr()
 					}
 
-					if c.NextArg() {
-						return rule, c.ArgErr()
+					for c.NextArg() {
+						path := c.Val()
+						rule.ExclusionRules = append(rule.ExclusionRules, internal.ExclusionRule{Method: method, Path: path})
 					}
 					break
 				}

pkg/apigateway/caddy-plugin/internal/exclusion_rule.go

@@ -0,0 +1,32 @@
+/*
+ *
+ * Copyright 2019 The KubeSphere Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package internal
+
+import "net/http"
+
+const AllMethod = "*"
+
+var HttpMethods = []string{AllMethod, http.MethodPost, http.MethodDelete,
+	http.MethodPatch, http.MethodPut, http.MethodGet, http.MethodOptions, http.MethodConnect}
+
+// Path exclusion rule
+type ExclusionRule struct {
+	Method string
+	Path   string
+}

pkg/apigateway/caddy-plugin/swagger/auto_load.go

@@ -25,13 +25,6 @@ import (
 	"github.com/mholt/caddy/caddyhttp/httpserver"
 )
 
-func init() {
-	caddy.RegisterPlugin("swagger", caddy.Plugin{
-		ServerType: "http",
-		Action:     Setup,
-	})
-}
-
 func Setup(c *caddy.Controller) error {
 
 	handler, err := parse(c)

pkg/apigateway/plugins.go

@@ -0,0 +1,26 @@
+package apigateway
+
+import (
+	"github.com/mholt/caddy"
+
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authenticate"
+	"kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authentication"
+	swagger "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/swagger"
+)
+
+func RegisterPlugins() {
+	caddy.RegisterPlugin("swagger", caddy.Plugin{
+		ServerType: "http",
+		Action:     swagger.Setup,
+	})
+
+	caddy.RegisterPlugin("authenticate", caddy.Plugin{
+		ServerType: "http",
+		Action:     authenticate.Setup,
+	})
+
+	caddy.RegisterPlugin("authentication", caddy.Plugin{
+		ServerType: "http",
+		Action:     authentication.Setup,
+	})
+}

pkg/apis/addtoscheme_devops_v1alpha1.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2019 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apis
+
+import (
+	"kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+)
+
+func init() {
+	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
+	AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme)
+}

pkg/apis/addtoscheme_network_v1alpha1.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2019 The KubeSphere authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apis
+
+import (
+	api "kubesphere.io/kubesphere/pkg/apis/network/v1alpha1"
+)
+
+func init() {
+	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
+	AddToSchemes = append(AddToSchemes, api.SchemeBuilder.AddToScheme)
+}

pkg/apis/addtoscheme_servicemesh_v1alpha2.go

@@ -20,7 +20,7 @@ import (
 	"github.com/knative/pkg/apis/istio/v1alpha3"
 	"kubesphere.io/kubesphere/pkg/apis/servicemesh/v1alpha2"
 
-	"github.com/kubernetes-sigs/application/pkg/apis/app/v1beta1"
+	appv1beta1 "sigs.k8s.io/application/pkg/apis/app/v1beta1"
 )
 
 func init() {
@@ -31,5 +31,5 @@ func init() {
 	AddToSchemes = append(AddToSchemes, v1alpha3.SchemeBuilder.AddToScheme)
 
 	// Register application scheme
-	AddToSchemes = append(AddToSchemes, v1beta1.SchemeBuilder.AddToScheme)
+	AddToSchemes = append(AddToSchemes, appv1beta1.SchemeBuilder.AddToScheme)
 }

pkg/apis/devops/OWNERS

@@ -0,0 +1,11 @@
+approvers:
+    - runzexia
+    - soulseen
+
+reviewers:
+    - runzexia
+    - soulseen
+
+labels:
+    - area/api
+    - area/devops

pkg/apis/devops/crdinstall/install.go

@@ -0,0 +1,30 @@
+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package install
+
+import (
+	k8sruntime "k8s.io/apimachinery/pkg/runtime"
+	urlruntime "k8s.io/apimachinery/pkg/util/runtime"
+	devopsv1alpha1 "kubesphere.io/kubesphere/pkg/apis/devops/v1alpha1"
+)
+
+func Install(scheme *k8sruntime.Scheme) {
+	urlruntime.Must(devopsv1alpha1.AddToScheme(scheme))
+	urlruntime.Must(scheme.SetVersionPriority(devopsv1alpha1.SchemeGroupVersion))
+}