update vendor

Signed-off-by: Roland.Ma <rolandma@yunify.com>
This commit is contained in:
Roland.Ma
2021-08-11 07:10:14 +00:00
parent a18f72b565
commit ea8f47c73a
2901 changed files with 269317 additions and 43103 deletions

View File

@@ -19,6 +19,7 @@ package client
import (
"context"
"errors"
"fmt"
"io"
"math/rand"
"net"
@@ -26,7 +27,7 @@ import (
"time"
"google.golang.org/grpc"
"k8s.io/klog"
"k8s.io/klog/v2"
"sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
)
@@ -49,11 +50,23 @@ type grpcTunnel struct {
conns map[int64]*conn
pendingDialLock sync.RWMutex
connsLock sync.RWMutex
// The tunnel will be closed if the caller fails to read via conn.Read()
// more than readTimeoutSeconds after a packet has been received.
readTimeoutSeconds int
}
// CreateGrpcTunnel creates a Tunnel to dial to a remote server through a
type clientConn interface {
Close() error
}
var _ clientConn = &grpc.ClientConn{}
// CreateSingleUseGrpcTunnel creates a Tunnel to dial to a remote server through a
// gRPC based proxy service.
func CreateGrpcTunnel(address string, opts ...grpc.DialOption) (Tunnel, error) {
// Currently, a single tunnel supports a single connection, and the tunnel is closed when the connection is terminated
// The Dial() method of the returned tunnel should only be called once
func CreateSingleUseGrpcTunnel(address string, opts ...grpc.DialOption) (Tunnel, error) {
c, err := grpc.Dial(address, opts...)
if err != nil {
return nil, err
@@ -67,28 +80,31 @@ func CreateGrpcTunnel(address string, opts ...grpc.DialOption) (Tunnel, error) {
}
tunnel := &grpcTunnel{
stream: stream,
pendingDial: make(map[int64]chan<- dialResult),
conns: make(map[int64]*conn),
stream: stream,
pendingDial: make(map[int64]chan<- dialResult),
conns: make(map[int64]*conn),
readTimeoutSeconds: 10,
}
go tunnel.serve()
go tunnel.serve(c)
return tunnel, nil
}
func (t *grpcTunnel) serve() {
func (t *grpcTunnel) serve(c clientConn) {
defer c.Close()
for {
pkt, err := t.stream.Recv()
if err == io.EOF {
return
}
if err != nil || pkt == nil {
klog.Warningf("stream read error: %v", err)
klog.ErrorS(err, "stream read failure")
return
}
klog.V(6).Infof("[tracing] recv packet, type: %s", pkt.Type)
klog.V(5).InfoS("[tracing] recv packet", "type", pkt.Type)
switch pkt.Type {
case client.PacketType_DIAL_RSP:
@@ -98,13 +114,26 @@ func (t *grpcTunnel) serve() {
t.pendingDialLock.RUnlock()
if !ok {
klog.Warning("DialResp not recognized; dropped")
klog.V(1).Infoln("DialResp not recognized; dropped")
} else {
ch <- dialResult{
result := dialResult{
err: resp.Error,
connid: resp.ConnectID,
}
select {
case ch <- result:
default:
klog.ErrorS(fmt.Errorf("blocked pending channel"), "Received second dial response for connection request", "connectionID", resp.ConnectID, "dialID", resp.Random)
// On multiple dial responses, avoid leaking serve goroutine.
return
}
}
if resp.Error != "" {
// On dial error, avoid leaking serve goroutine.
return
}
case client.PacketType_DATA:
resp := pkt.GetData()
// TODO: flow control
@@ -113,9 +142,16 @@ func (t *grpcTunnel) serve() {
t.connsLock.RUnlock()
if ok {
conn.readCh <- resp.Data
timer := time.NewTimer((time.Duration)(t.readTimeoutSeconds) * time.Second)
select {
case conn.readCh <- resp.Data:
timer.Stop()
case <-timer.C:
klog.ErrorS(fmt.Errorf("timeout"), "readTimeout has been reached, the grpc connection to the proxy server will be closed", "connectionID", conn.connID, "readTimeoutSeconds", t.readTimeoutSeconds)
return
}
} else {
klog.Warningf("connection id %d not recognized", resp.ConnectID)
klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID)
}
case client.PacketType_CLOSE_RSP:
resp := pkt.GetCloseResponse()
@@ -130,9 +166,9 @@ func (t *grpcTunnel) serve() {
t.connsLock.Lock()
delete(t.conns, resp.ConnectID)
t.connsLock.Unlock()
} else {
klog.Warningf("connection id %d not recognized", resp.ConnectID)
return
}
klog.V(1).InfoS("connection not recognized", "connectionID", resp.ConnectID)
}
}
}
@@ -144,8 +180,8 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) {
return nil, errors.New("protocol not supported")
}
random := rand.Int63()
resCh := make(chan dialResult)
random := rand.Int63() /* #nosec G404 */
resCh := make(chan dialResult, 1)
t.pendingDialLock.Lock()
t.pendingDial[random] = resCh
t.pendingDialLock.Unlock()
@@ -165,14 +201,14 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) {
},
},
}
klog.V(6).Infof("[tracing] send packet, type: %s", req.Type)
klog.V(5).InfoS("[tracing] send packet", "type", req.Type)
err := t.stream.Send(req)
if err != nil {
return nil, err
}
klog.Info("DIAL_REQ sent to proxy server")
klog.V(5).Infoln("DIAL_REQ sent to proxy server")
c := &conn{stream: t.stream}
@@ -183,7 +219,7 @@ func (t *grpcTunnel) Dial(protocol, address string) (net.Conn, error) {
}
c.connID = res.connid
c.readCh = make(chan []byte, 10)
c.closeCh = make(chan string)
c.closeCh = make(chan string, 1)
t.connsLock.Lock()
t.conns[res.connid] = c
t.connsLock.Unlock()

View File

@@ -22,7 +22,7 @@ import (
"net"
"time"
"k8s.io/klog"
"k8s.io/klog/v2"
"sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
)
@@ -54,7 +54,7 @@ func (c *conn) Write(data []byte) (n int, err error) {
},
}
klog.V(6).Infof("[tracing] send req, type: %s", req.Type)
klog.V(5).InfoS("[tracing] send req", "type", req.Type)
err = c.stream.Send(req)
if err != nil {
@@ -112,7 +112,7 @@ func (c *conn) SetWriteDeadline(t time.Time) error {
// Close closes the connection. It also sends CLOSE_REQ packet over
// proxy service to notify remote to drop the connection.
func (c *conn) Close() error {
klog.Info("conn.Close()")
klog.V(4).Infoln("closing connection")
req := &client.Packet{
Type: client.PacketType_CLOSE_REQ,
Payload: &client.Packet_CloseRequest{
@@ -122,7 +122,7 @@ func (c *conn) Close() error {
},
}
klog.V(6).Infof("[tracing] send req, type: %s", req.Type)
klog.V(5).InfoS("[tracing] send req", "type", req.Type)
if err := c.stream.Send(req); err != nil {
return err

View File

@@ -1,36 +1,130 @@
run:
deadline: 5m
linters-settings:
lll:
line-length: 170
dupl:
threshold: 400
issues:
# don't skip warning about doc comments
exclude-use-default: false
# restore some of the defaults
# (fill in the rest as needed)
exclude-rules:
- linters: [errcheck]
text: "Error return value of .((os\\.)?std(out|err)\\..*|.*Close|.*Flush|os\\.Remove(All)?|.*printf?|os\\.(Un)?Setenv). is not checked"
linters:
disable-all: true
enable:
- misspell
- structcheck
- golint
- govet
- asciicheck
- bodyclose
- deadcode
- depguard
- dogsled
- errcheck
- varcheck
- exportloopref
- goconst
- unparam
- ineffassign
- nakedret
- interfacer
- gocritic
- gocyclo
- lll
- dupl
- godot
- gofmt
- goimports
- golint
- goprintffuncname
- gosec
- gosimple
- govet
- ifshort
- importas
- ineffassign
- misspell
- nakedret
- nilerr
- nolintlint
- prealloc
- revive
- rowserrcheck
- staticcheck
- structcheck
- stylecheck
- typecheck
- unconvert
- unparam
- varcheck
- whitespace
linters-settings:
ifshort:
# Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax.
max-decl-chars: 50
importas:
no-unaliased: true
alias:
# Kubernetes
- pkg: k8s.io/api/core/v1
alias: corev1
- pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1
alias: apiextensionsv1
- pkg: k8s.io/apimachinery/pkg/apis/meta/v1
alias: metav1
- pkg: k8s.io/apimachinery/pkg/api/errors
alias: apierrors
- pkg: k8s.io/apimachinery/pkg/util/errors
alias: kerrors
# Controller Runtime
- pkg: sigs.k8s.io/controller-runtime
alias: ctrl
staticcheck:
go: "1.16"
stylecheck:
go: "1.16"
issues:
max-same-issues: 0
max-issues-per-linter: 0
# We are disabling default golangci exclusions because we want to help reviewers to focus on reviewing the most relevant
# changes in PRs and avoid nitpicking.
exclude-use-default: false
# List of regexps of issue texts to exclude, empty list by default.
exclude:
# The following are being worked on to remove their exclusion. This list should be reduced or go away all together over time.
# If it is decided they will not be addressed they should be moved above this comment.
- Subprocess launch(ed with variable|ing should be audited)
- (G204|G104|G307)
- "ST1000: at least one file in a package should have a package comment"
exclude-rules:
- linters:
- gosec
text: "G108: Profiling endpoint is automatically exposed on /debug/pprof"
- linters:
- revive
text: "exported: exported method .*\\.(Reconcile|SetupWithManager|SetupWebhookWithManager) should have comment or be unexported"
- linters:
- errcheck
text: Error return value of .((os\.)?std(out|err)\..*|.*Close|.*Flush|os\.Remove(All)?|.*print(f|ln)?|os\.(Un)?Setenv). is not checked
# With Go 1.16, the new embed directive can be used with an un-named import,
# revive (previously, golint) only allows these to be imported in a main.go, which wouldn't work for us.
# This directive allows the embed package to be imported with an underscore everywhere.
- linters:
- revive
source: _ "embed"
# Exclude some packages or code to require comments, for example test code, or fake clients.
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
source: (func|type).*Fake.*
- linters:
- revive
text: exported (method|function|type|const) (.+) should have comment or be unexported
path: fake_\.go
# Disable unparam "always receives" which might not be really
# useful when building libraries.
- linters:
- unparam
text: always receives
# Dot imports for gomega or ginkgo are allowed
# within test files.
- path: _test\.go
text: should not use dot imports
- path: _test\.go
text: cyclomatic complexity
- path: _test\.go
text: "G107: Potential HTTP request made with variable url"
# Append should be able to assign to a different var/slice.
- linters:
- gocritic
text: "appendAssign: append result not assigned to the same slice"
- linters:
- gocritic
text: "singleCaseSwitch: should rewrite switch statement to if statement"
run:
timeout: 10m
skip-files:
- "zz_generated.*\\.go$"
- ".*conversion.*\\.go$"
allow-parallel-runners: true

View File

@@ -39,6 +39,7 @@ TOOLS_DIR := hack/tools
TOOLS_BIN_DIR := $(TOOLS_DIR)/bin
GOLANGCI_LINT := $(abspath $(TOOLS_BIN_DIR)/golangci-lint)
GO_APIDIFF := $(TOOLS_BIN_DIR)/go-apidiff
CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen
# The help will print out all targets with their descriptions organized bellow their categories. The categories are represented by `##@` and the target descriptions by `##`.
# The awk commands is responsible to read the entire set of makefiles included in this invocation, looking for lines of the file as xyz: ## something, and then pretty-format the target and help. Then, if there's a line with ##@ something, that gets pretty-printed as a category.
@@ -53,26 +54,40 @@ help: ## Display this help
## --------------------------------------
.PHONY: test
test: ## Run the script check-everything.sh which will check all.
test: test-tools ## Run the script check-everything.sh which will check all.
TRACE=1 ./hack/check-everything.sh
.PHONY: test-tools
test-tools: ## tests the tools codebase (setup-envtest)
cd tools/setup-envtest && go test ./...
## --------------------------------------
## Binaries
## --------------------------------------
$(GOLANGCI_LINT): $(TOOLS_DIR)/go.mod # Build golangci-lint from tools folder.
cd $(TOOLS_DIR) && go build -tags=tools -o bin/golangci-lint github.com/golangci/golangci-lint/cmd/golangci-lint
$(GO_APIDIFF): $(TOOLS_DIR)/go.mod # Build go-apidiff from tools folder.
cd $(TOOLS_DIR) && go build -tags=tools -o bin/go-apidiff github.com/joelanford/go-apidiff
$(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder.
cd $(TOOLS_DIR) && go build -tags=tools -o bin/controller-gen sigs.k8s.io/controller-tools/cmd/controller-gen
$(GOLANGCI_LINT): .github/workflows/golangci-lint.yml # Download golanci-lint using hack script into tools folder.
hack/ensure-golangci-lint.sh \
-b $(TOOLS_BIN_DIR) \
$(shell cat .github/workflows/golangci-lint.yml | grep version | sed 's/.*version: //')
## --------------------------------------
## Linting
## --------------------------------------
.PHONY: lint
lint: $(GOLANGCI_LINT) ## Lint codebase.
$(GOLANGCI_LINT) run -v
lint: $(GOLANGCI_LINT) ## Lint codebase
$(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
cd tools/setup-envtest; $(GOLANGCI_LINT) run -v $(GOLANGCI_LINT_EXTRA_ARGS)
.PHONY: lint-fix
lint-fix: $(GOLANGCI_LINT) ## Lint the codebase and run auto-fixers if supported by the linter.
GOLANGCI_LINT_EXTRA_ARGS=--fix $(MAKE) lint
## --------------------------------------
## Generate
@@ -83,6 +98,10 @@ modules: ## Runs go mod to ensure modules are up to date.
go mod tidy
cd $(TOOLS_DIR); go mod tidy
.PHONY: generate
generate: $(CONTROLLER_GEN) ## Runs controller-gen for internal types for config file
$(CONTROLLER_GEN) object paths="./pkg/config/v1alpha1/...;./examples/configfile/custom/v1alpha1/..."
## --------------------------------------
## Cleanup / Verification
## --------------------------------------
@@ -98,5 +117,5 @@ clean-bin: ## Remove all generated binaries.
.PHONY: verify-modules
verify-modules: modules
@if !(git diff --quiet HEAD -- go.sum go.mod); then \
echo "go module files are out of date"; exit 1; \
echo "go module files are out of date, please run 'make modules'"; exit 1; \
fi

View File

@@ -4,7 +4,6 @@ aliases:
# active folks who can be contacted to perform admin-related
# tasks on the repo, or otherwise approve any PRS.
controller-runtime-admins:
- directxman12
- droot
- mengqiy
- pwittrock
@@ -36,4 +35,5 @@ aliases:
# folks who may have context on ancient history,
# but are no longer directly involved
# controller-runtime-emeritus-maintainers:
controller-runtime-emeritus-maintainers:
- directxman12

View File

@@ -11,10 +11,10 @@ see how it can be used.
Documentation:
- [Package overview](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg)
- [Basic controller using builder](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/builder#example-Builder)
- [Creating a manager](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/manager#example-New)
- [Creating a controller](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/controller#example-New)
- [Package overview](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg)
- [Basic controller using builder](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/builder#example-Builder)
- [Creating a manager](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/manager#example-New)
- [Creating a controller](https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/controller#example-New)
- [Examples](https://github.com/kubernetes-sigs/controller-runtime/blob/master/examples)
- [Designs](https://github.com/kubernetes-sigs/controller-runtime/blob/master/designs)
@@ -63,4 +63,3 @@ Before starting any work, please either comment on an existing issue, or file a
## Code of conduct
Participation in the Kubernetes community is governed by the [Kubernetes Code of Conduct](code-of-conduct.md).

View File

@@ -10,6 +10,5 @@
# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
# INSTRUCTIONS AT https://kubernetes.io/security/
directxman12
pwittrock
droot

View File

@@ -90,6 +90,9 @@ It's acceptable to log call `log.Error` with a nil error object. This
conveys that an error occurred in some capacity, but that no actual
`error` object was involved.
Errors returned by the `Reconcile` implementation of the `Reconciler` interface are commonly logged as a `Reconciler error`.
It's a developer choice to create an additional error log in the `Reconcile` implementation so a more specific file name and line for the error are returned.
## Logging messages
- Don't put variable content in your messages -- use key-value pairs for

View File

@@ -1,271 +1,30 @@
# Versioning and Branching in controller-runtime
*NB*: this also applies to controller-tools.
We follow the [common KubeBuilder versioning guidelines][guidelines], and
use the corresponding tooling.
## TL;DR:
For the purposes of the aforementioned guidelines, controller-runtime
counts as a "library project", but otherwise follows the guidelines
exactly.
### Users
[guidelines]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md
- We follow [Semantic Versioning (semver)](https://semver.org)
- Use releases with your dependency management to ensure that you get
compatible code
- The master branch contains all the latest code, some of which may break
compatibility (so "normal" `go get` is not recommended)
## Compatiblity and Release Support
### Contributors
For release branches, we generally tend to support backporting one (1)
major release (`release-{X-1}` or `release-0.{Y-1}`), but may go back
further if the need arises and is very pressing (e.g. security updates).
- All code PR must be labeled with :bug: (patch fixes), :sparkles:
(backwards-compatible features), or :warning: (breaking changes)
### Dependency Support
- Breaking changes will find their way into the next major release, other
changes will go into an semi-immediate patch or minor release
Note the [guidelines on dependency versions][dep-versions]. Particularly:
- Please *try* to avoid breaking changes when you can. They make users
face difficult decisions ("when do I go through the pain of
upgrading?"), and make life hard for maintainers and contributors
(dealing with differences on stable branches).
- We **DO** guarantee Kubernetes REST API compability -- if a given
version of controller-runtime stops working with what should be
a supported version of Kubernetes, this is almost certainly a bug.
### Mantainers
- We **DO NOT** guarantee any particular compability matrix between
kubernetes library dependencies (client-go, apimachinery, etc); Such
compability is infeasible due to the way those libraries are versioned.
Don't be lazy, read the rest of this doc :-)
## Overview
controller-runtime (and friends) follow [Semantic
Versioning](https://semver.org). I'd recommend reading the aforementioned
link if you're not familiar, but essentially, for any given release X.Y.Z:
- an X (*major*) release indicates a set of backwards-compatible code.
Changing X means there's a breaking change.
- a Y (*minor*) release indicates a minimum feature set. Changing Y means
the addition of a backwards-compatible feature.
- a Z (*patch*) release indicates minimum set of bugfixes. Changing
Z means a backwards-compatible change that doesn't add functionality.
*NB*: If the major release is `0`, any minor release may contain breaking
changes.
These guarantees extend to all code exposed in public APIs of
controller-runtime. This includes code both in controller-runtime itself,
*plus types from dependencies in public APIs*. Types and functions not in
public APIs are not considered part of the guarantee.
In order to easily maintain the guarantees, we have a couple of processes
that we follow.
## Branches
controller-runtime contains two types of branches: the *master* branch and
*release-X* branches.
The *master* branch is where development happens. All the latest and
greatest code, including breaking changes, happens on master.
The *release-X* branches contain stable, backwards compatible code. Every
major (X) release, a new such branch is created. It is from these
branches that minor and patch releases are tagged. If some cases, it may
be necessary open PRs for bugfixes directly against stable branches, but
this should generally not be the case.
The maintainers are responsible for updating the contents of this branch;
generally, this is done just before a release using release tooling that
filters and checks for changes tagged as breaking (see below).
### Tooling
* [release-notes.sh](hack/release/release-notes.sh): generate release notes
for a range of commits, and check for next version type (***TODO***)
* [verify-emoji.sh](hack/release/verify-emoji.sh): check that
your PR and/or commit messages have the right versioning icon
(***TODO***).
## PR Process
Every PR should be annotated with an icon indicating whether it's
a:
- Breaking change: :warning: (`:warning:`)
- Non-breaking feature: :sparkles: (`:sparkles:`)
- Patch fix: :bug: (`:bug:`)
- Docs: :book: (`:book:`)
- Infra/Tests/Other: :seedling: (`:seedling:`)
- No release note: :ghost: (`:ghost:`)
Use :ghost: (no release note) only for the PRs that change or revert unreleased
changes, which don't deserve a release note. Please don't abuse it.
You can also use the equivalent emoji directly, since GitHub doesn't
render the `:xyz:` aliases in PR titles.
Individual commits should not be tagged separately, but will generally be
assumed to match the PR. For instance, if you have a bugfix in with
a breaking change, it's generally encouraged to submit the bugfix
separately, but if you must put them in one PR, mark the commit
separately.
### Commands and Workflow
controller-runtime follows the standard Kubernetes workflow: any PR needs
`lgtm` and `approved` labels, PRs authors must have signed the CNCF CLA,
and PRs must pass the tests before being merged. See [the contributor
docs](https://github.com/kubernetes/community/blob/master/contributors/guide/pull-requests.md#the-testing-and-merge-workflow)
for more info.
We use the same priority and kind labels as Kubernetes. See the labels
tab in GitHub for the full list.
The standard Kubernetes comment commands should work in
controller-runtime. See [Prow](https://prow.k8s.io/command-help) for
a command reference.
## Release Process
Minor and patch releases are generally done immediately after a feature or
bugfix is landed, or sometimes a series of features tied together.
Minor releases will only be tagged on the *most recent* major release
branch, except in exceptional circumstances. Patches will be backported
to maintained stable versions, as needed.
Major releases are done shortly after a breaking change is merged -- once
a breaking change is merged, the next release *must* be a major revision.
We don't intend to have a lot of these, so we may put off merging breaking
PRs until a later date.
### Exact Steps
Follow the release-specific steps below, then follow the general steps
after that.
#### Minor and patch releases
1. Update the release-X branch with the latest set of changes by calling
`git rebase master` from the release branch.
#### Major releases
1. Create a new release branch named `release-X` (where `X` is the new
version) off of master.
#### General
2. Generate release notes using the release note tooling.
3. Add a release for controller-runtime on GitHub, using those release
notes, with a title of `vX.Y.Z`.
4. Do a similar process for
[controller-tools](https://github.com/kubernetes-sigs/controller-tools)
5. Announce the release in `#kubebuilder` on Slack with a pinned message.
6. Potentially update
[kubebuilder](https://github.com/kubernetes-sigs/kubebuilder) as well.
### Breaking Changes
Try to avoid breaking changes. They make life difficult for users, who
have to rewrite their code when they eventually upgrade, and for
maintainers/contributors, who have to deal with differences between master
and stable branches.
That being said, we'll occasionally want to make breaking changes. They'll
be merged onto master, and will then trigger a major release (see [Release
Process](#release-process)). Because breaking changes induce a major
revision, the maintainers may delay a particular breaking change until
a later date when they are ready to make a major revision with a few
breaking changes.
If you're going to make a breaking change, please make sure to explain in
detail why it's helpful. Is it necessary to cleanly resolve an issue?
Does it improve API ergonomics?
Maintainers should treat breaking changes with caution, and evaluate
potential non-breaking solutions (see below).
Note that API breakage in public APIs due to dependencies will trigger
a major revision, so you may occasionally need to have a major release
anyway, due to changes in libraries like `k8s.io/client-go` or
`k8s.io/apimachinery`.
*NB*: Pre-1.0 releases treat breaking changes a bit more lightly. We'll
still consider carefully, but the pre-1.0 timeframe is useful for
converging on a ergonomic API.
#### Avoiding breaking changes
##### Solutions to avoid
- **Confusingly duplicate methods, functions, or variables.**
For instance, suppose we have an interface method `List(ctx
context.Context, options *ListOptions, obj runtime.Object) error`, and
we decide to switch it so that options come at the end, parametrically.
Adding a new interface method `ListParametric(ctx context.Context, obj
runtime.Object, options... ListOption)` is probably not the right
solution:
- Users will intuitively see `List`, and use that in new projects, even
if it's marked as deprecated.
- Users who don't notice the deprecation may be confused as to the
difference between `List` and `ListParametric`.
- It's not immediately obvious in isolation (e.g. in surrounding code)
why the method is called `ListParametric`, and may cause confusion
when reading code that makes use of that method.
In this case, it may be better to make the breaking change, and then
eventually do a major release.
## Why don't we...
### Use "next"-style branches
Development branches:
- don't win us much in terms of maintenance in the case of breaking
changes (we still have to merge/manage multiple branches for development
and stable)
- can be confusing to contributors, who often expect master to have the
latest changes.
### Never break compatibility
Never doing a new major release could be an admirable goal, but gradually
leads to API cruft.
Since one of the goals of controller-runtime is to be a friendly and
intuitive API, we want to avoid too much API cruft over time, and
occasional breaking changes in major releases help accomplish that goal.
Furthermore, our dependency on Kubernetes libraries makes this difficult
(see below)
### Always assume we've broken compatibility
*a.k.a. k8s.io/client-go style*
While this makes life easier (a bit) for maintainers, it's problematic for
users. While breaking changes arrive sooner, upgrading becomes very
painful.
Furthermore, we still have to maintain stable branches for bugfixes, so
the maintenance burden isn't lessened by a ton.
### Extend compatibility guarantees to all dependencies
This is very difficult with the number of Kubernetes dependencies we have.
Kubernetes dependencies tend to either break compatibility every major
release (e.g. k8s.io/client-go, which loosely follows semver), or at
a whim (many other Kubernetes libraries).
If we limit to the few objects we expose, we can better inform users about
how *controller-runtime itself* has changed in a given release. Then,
users can make informed decisions about how to proceed with any direct
uses of Kubernetes dependencies their controller-runtime-based application
may have.
[dep-versions]: https://sigs.k8s.io/kubebuilder-release-tools/VERSIONING.md#kubernetes-version-compatibility

View File

@@ -21,6 +21,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/builder"
"sigs.k8s.io/controller-runtime/pkg/client/config"
cfg "sigs.k8s.io/controller-runtime/pkg/config"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -44,7 +45,7 @@ type Result = reconcile.Result
// A Manager is required to create Controllers.
type Manager = manager.Manager
// Options are the arguments for creating a new Manager
// Options are the arguments for creating a new Manager.
type Options = manager.Options
// SchemeBuilder builds a new Scheme for mapping go types to Kubernetes GroupVersionKinds.
@@ -54,7 +55,7 @@ type SchemeBuilder = scheme.Builder
type GroupVersion = schema.GroupVersion
// GroupResource specifies a Group and a Resource, but does not force a version. This is useful for identifying
// concepts during lookup stages without having partially valid types
// concepts during lookup stages without having partially valid types.
type GroupResource = schema.GroupResource
// TypeMeta describes an individual object in an API response or request
@@ -88,13 +89,18 @@ var (
//
// * In-cluster config if running in cluster
//
// * $HOME/.kube/config if exists
// * $HOME/.kube/config if exists.
GetConfig = config.GetConfig
// NewControllerManagedBy returns a new controller builder that will be started by the provided Manager
// ConfigFile returns the cfg.File function for deferred config file loading,
// this is passed into Options{}.From() to populate the Options fields for
// the manager.
ConfigFile = cfg.File
// NewControllerManagedBy returns a new controller builder that will be started by the provided Manager.
NewControllerManagedBy = builder.ControllerManagedBy
// NewWebhookManagedBy returns a new webhook builder that will be started by the provided Manager
// NewWebhookManagedBy returns a new webhook builder that will be started by the provided Manager.
NewWebhookManagedBy = builder.WebhookManagedBy
// NewManager returns a new Manager for creating Controllers.
@@ -125,10 +131,19 @@ var (
// get any actual logging.
Log = log.Log
// LoggerFromContext returns a logger with predefined values from a context.Context.
// LoggerFrom returns a logger with predefined values from a context.Context.
// The logger, when used with controllers, can be expected to contain basic information about the object
// that's being reconciled like:
// - `reconciler group` and `reconciler kind` coming from the For(...) object passed in when building a controller.
// - `name` and `namespace` injected from the reconciliation request.
//
// This is meant to be used with the context supplied in a struct that satisfies the Reconciler interface.
LoggerFromContext = log.FromContext
LoggerFrom = log.FromContext
// LoggerInto takes a context and sets the logger as one of its keys.
//
// This is meant to be used in reconcilers to enrich the logger within a context with additional values.
LoggerInto = log.IntoContext
// SetLogger sets a concrete logging implementation for all deferred Loggers.
SetLogger = log.SetLogger

View File

@@ -28,7 +28,7 @@ limitations under the License.
// The main entrypoint for controller-runtime is this root package, which
// contains all of the common types needed to get started building controllers:
// import (
// controllers "sigs.k8s.io/controller-runtime"
// ctrl "sigs.k8s.io/controller-runtime"
// )
//
// The examples in this package walk through a basic controller setup. The

View File

@@ -1,33 +1,30 @@
module sigs.k8s.io/controller-runtime
go 1.13
go 1.16
require (
github.com/evanphx/json-patch v4.9.0+incompatible
github.com/evanphx/json-patch v4.11.0+incompatible
github.com/fsnotify/fsnotify v1.4.9
github.com/go-logr/logr v0.1.0
github.com/go-logr/zapr v0.1.0
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
github.com/googleapis/gnostic v0.3.1 // indirect
github.com/go-logr/logr v0.4.0
github.com/go-logr/zapr v0.4.0
github.com/googleapis/gnostic v0.5.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/imdario/mergo v0.3.9 // indirect
github.com/json-iterator/go v1.1.10 // indirect
github.com/onsi/ginkgo v1.12.1
github.com/onsi/gomega v1.10.1
github.com/prometheus/client_golang v1.0.0
github.com/imdario/mergo v0.3.12 // indirect
github.com/onsi/ginkgo v1.16.4
github.com/onsi/gomega v1.13.0
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/client_model v0.2.0
github.com/prometheus/procfs v0.0.11 // indirect
github.com/spf13/pflag v1.0.5
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/zap v1.10.0
golang.org/x/text v0.3.3 // indirect
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4
gomodules.xyz/jsonpatch/v2 v2.0.1
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
k8s.io/api v0.18.6
k8s.io/apiextensions-apiserver v0.18.6
k8s.io/apimachinery v0.18.6
k8s.io/client-go v0.18.6
k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
go.uber.org/goleak v1.1.10
go.uber.org/zap v1.17.0
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6
gomodules.xyz/jsonpatch/v2 v2.2.0
google.golang.org/appengine v1.6.7 // indirect
k8s.io/api v0.21.2
k8s.io/apiextensions-apiserver v0.21.2
k8s.io/apimachinery v0.21.2
k8s.io/client-go v0.21.2
k8s.io/component-base v0.21.2
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b
sigs.k8s.io/yaml v1.2.0
)

View File

@@ -1,230 +1,276 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.38.0 h1:ROfEUZz+Gh5pa62DJWXSaonyu3StP6EA6lPEXPI6mCo=
cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To=
cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4=
cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M=
cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0=
cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw=
cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
github.com/Azure/go-autorest/autorest/adal v0.5.0 h1:q2gDruN08/guU9vAjuPWff0+QIrpH6ediguzdAzXAUU=
github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
github.com/Azure/go-autorest/autorest/date v0.1.0 h1:YGrhWfrgtFs84+h0o46rJrlmsZtyZRg470CqAXTZaGM=
github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY=
github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M=
github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0 h1:w3NnFcKR5241cfmQU5ZZAsf0xcpId6mWOupTvJlUX2U=
github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QLSV/BsnenAOcDXdX4cMv4wP0B/5QbPg=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.5.0+incompatible h1:ouOWdg56aJriqS0huScTkVXPC5IcNrDCXZ6OoTAWu7M=
github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses=
github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ=
github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs=
github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54=
github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI=
github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik=
github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk=
github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU=
github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0=
github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94=
github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M=
github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc=
github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM=
github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk=
github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg=
github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU=
github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs=
github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk=
github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA=
github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64=
github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4=
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI=
github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU=
github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY=
github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg=
github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4=
github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=
github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk=
github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU=
github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -234,209 +280,399 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb
github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI=
github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U=
go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo=
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 h1:AeiKBIuRw3UomYXSbLy0Mc2dDLfdtbT/IVn4keq83P0=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0=
golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs=
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0=
gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c=
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA=
google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -446,41 +682,53 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
k8s.io/api v0.18.6 h1:osqrAXbOQjkKIWDTjrqxWQ3w0GkKb1KA1XkUGHHYpeE=
k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI=
k8s.io/apiextensions-apiserver v0.18.6 h1:vDlk7cyFsDyfwn2rNAO2DbmUbvXy5yT5GE3rrqOzaMo=
k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M=
k8s.io/apimachinery v0.18.6 h1:RtFHnfGNfd1N0LeSrKCUznz5xtUP1elRGvHJbL3Ntag=
k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko=
k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg=
k8s.io/client-go v0.18.6 h1:I+oWqJbibLSGsZj8Xs8F0aWVXJVIoUHWaaJV3kUN/Zw=
k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q=
k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
k8s.io/klog/v2 v2.0.0 h1:Foj74zO6RbjjP4hBEKjnYtjjAhGg4jNynUdYF6fJrok=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
k8s.io/api v0.21.2 h1:vz7DqmRsXTCSa6pNxXwQ1IYeAZgdIsua+DZU+o+SX3Y=
k8s.io/api v0.21.2/go.mod h1:Lv6UGJZ1rlMI1qusN8ruAp9PUBFyBwpEHAdG24vIsiU=
k8s.io/apiextensions-apiserver v0.21.2 h1:+exKMRep4pDrphEafRvpEi79wTnCFMqKf8LBtlA3yrE=
k8s.io/apiextensions-apiserver v0.21.2/go.mod h1:+Axoz5/l3AYpGLlhJDfcVQzCerVYq3K3CvDMvw6X1RA=
k8s.io/apimachinery v0.21.2 h1:vezUc/BHqWlQDnZ+XkrpXSmnANSLbpnlpwo0Lhk0gpc=
k8s.io/apimachinery v0.21.2/go.mod h1:CdTY8fU/BlvAbJ2z/8kBwimGki5Zp8/fbVuLY8gJumM=
k8s.io/apiserver v0.21.2/go.mod h1:lN4yBoGyiNT7SC1dmNk0ue6a5Wi6O3SWOIw91TsucQw=
k8s.io/client-go v0.21.2 h1:Q1j4L/iMN4pTw6Y4DWppBoUxgKO8LbffEMVEV00MUp0=
k8s.io/client-go v0.21.2/go.mod h1:HdJ9iknWpbl3vMGtib6T2PyI/VYxiZfq936WNVHBRrA=
k8s.io/code-generator v0.21.2/go.mod h1:8mXJDCB7HcRo1xiEQstcguZkbxZaqeUOrO9SsicWs3U=
k8s.io/component-base v0.21.2 h1:EsnmFFoJ86cEywC0DoIkAUiEV6fjgauNugiw1lmIjs4=
k8s.io/component-base v0.21.2/go.mod h1:9lvmIThzdlrJj5Hp8Z/TOgIkdfsNARQ1pT+3PByuiuc=
k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6 h1:Oh3Mzx5pJ+yIumsAD0MOECPVeXsVot0UkiaCGVyfGQY=
k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E=
k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 h1:v8ud2Up6QK1lNOKFgiIVrZdMg7MpmSnvtrOieolJKoE=
k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0=
sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0=
k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b h1:MSqsVQ3pZvPGTqCjptfimO2WjG7A9un2zcpiHkA6M/s=
k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.19/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8=
sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

View File

@@ -22,9 +22,9 @@ import (
"github.com/go-logr/logr"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -34,18 +34,18 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
)
// Supporting mocking out functions for testing
// Supporting mocking out functions for testing.
var newController = controller.New
var getGvk = apiutil.GVKForObject
// project represents other forms that the we can use to
// send/receive a given resource (metadata-only, unstructured, etc)
// send/receive a given resource (metadata-only, unstructured, etc).
type objectProjection int
const (
// projectAsNormal doesn't change the object from the form given
// projectAsNormal doesn't change the object from the form given.
projectAsNormal objectProjection = iota
// projectAsMetadata turns this into an metadata-only watch
// projectAsMetadata turns this into an metadata-only watch.
projectAsMetadata
)
@@ -56,40 +56,33 @@ type Builder struct {
watchesInput []WatchesInput
mgr manager.Manager
globalPredicates []predicate.Predicate
config *rest.Config
ctrl controller.Controller
ctrlOptions controller.Options
log logr.Logger
name string
}
// ControllerManagedBy returns a new controller builder that will be started by the provided Manager
// ControllerManagedBy returns a new controller builder that will be started by the provided Manager.
func ControllerManagedBy(m manager.Manager) *Builder {
return &Builder{mgr: m}
}
// ForType defines the type of Object being *reconciled*, and configures the ControllerManagedBy to respond to create / delete /
// update events by *reconciling the object*.
// This is the equivalent of calling
// Watches(&source.Kind{Type: apiType}, &handler.EnqueueRequestForObject{})
//
// Deprecated: Use For
func (blder *Builder) ForType(apiType runtime.Object) *Builder {
return blder.For(apiType)
}
// ForInput represents the information set by For method.
type ForInput struct {
object runtime.Object
object client.Object
predicates []predicate.Predicate
objectProjection objectProjection
err error
}
// For defines the type of Object being *reconciled*, and configures the ControllerManagedBy to respond to create / delete /
// update events by *reconciling the object*.
// This is the equivalent of calling
// Watches(&source.Kind{Type: apiType}, &handler.EnqueueRequestForObject{})
func (blder *Builder) For(object runtime.Object, opts ...ForOption) *Builder {
// Watches(&source.Kind{Type: apiType}, &handler.EnqueueRequestForObject{}).
func (blder *Builder) For(object client.Object, opts ...ForOption) *Builder {
if blder.forInput.object != nil {
blder.forInput.err = fmt.Errorf("For(...) should only be called once, could not assign multiple objects for reconciliation")
return blder
}
input := ForInput{object: object}
for _, opt := range opts {
opt.ApplyToFor(&input)
@@ -101,15 +94,15 @@ func (blder *Builder) For(object runtime.Object, opts ...ForOption) *Builder {
// OwnsInput represents the information set by Owns method.
type OwnsInput struct {
object runtime.Object
object client.Object
predicates []predicate.Predicate
objectProjection objectProjection
}
// Owns defines types of Objects being *generated* by the ControllerManagedBy, and configures the ControllerManagedBy to respond to
// create / delete / update events by *reconciling the owner object*. This is the equivalent of calling
// Watches(&source.Kind{Type: <ForType-forInput>}, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true})
func (blder *Builder) Owns(object runtime.Object, opts ...OwnsOption) *Builder {
// Watches(&source.Kind{Type: <ForType-forInput>}, &handler.EnqueueRequestForOwner{OwnerType: apiType, IsController: true}).
func (blder *Builder) Owns(object client.Object, opts ...OwnsOption) *Builder {
input := OwnsInput{object: object}
for _, opt := range opts {
opt.ApplyToOwns(&input)
@@ -140,14 +133,6 @@ func (blder *Builder) Watches(src source.Source, eventhandler handler.EventHandl
return blder
}
// WithConfig sets the Config to use for configuring clients. Defaults to the in-cluster config or to ~/.kube/config.
//
// Deprecated: Use ControllerManagedBy(Manager) and this isn't needed.
func (blder *Builder) WithConfig(config *rest.Config) *Builder {
blder.config = config
return blder
}
// WithEventFilter sets the event filters, to filter which create/update/delete/generic events eventually
// trigger reconciliations. For example, filtering on whether the resource version has changed.
// Given predicate is added for all watched objects.
@@ -163,6 +148,12 @@ func (blder *Builder) WithOptions(options controller.Options) *Builder {
return blder
}
// WithLogger overrides the controller options's logger used.
func (blder *Builder) WithLogger(log logr.Logger) *Builder {
blder.ctrlOptions.Log = log
return blder
}
// Named sets the name of the controller to the given name. The name shows up
// in metrics, among other things, and thus should be a prometheus compatible name
// (underscores and alphanumeric characters only).
@@ -173,19 +164,13 @@ func (blder *Builder) Named(name string) *Builder {
return blder
}
// WithLogger overrides the controller options's logger used.
func (blder *Builder) WithLogger(log logr.Logger) *Builder {
blder.log = log
return blder
}
// Complete builds the Application ControllerManagedBy.
// Complete builds the Application Controller.
func (blder *Builder) Complete(r reconcile.Reconciler) error {
_, err := blder.Build(r)
return err
}
// Build builds the Application ControllerManagedBy and returns the Controller it created.
// Build builds the Application Controller and returns the Controller it created.
func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, error) {
if r == nil {
return nil, fmt.Errorf("must provide a non-nil Reconciler")
@@ -193,9 +178,13 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro
if blder.mgr == nil {
return nil, fmt.Errorf("must provide a non-nil Manager")
}
// Set the Config
blder.loadRestConfig()
if blder.forInput.err != nil {
return nil, blder.forInput.err
}
// Checking the reconcile type exist or not
if blder.forInput.object == nil {
return nil, fmt.Errorf("must provide an object for reconciliation")
}
// Set the ControllerManagedBy
if err := blder.doController(r); err != nil {
@@ -210,7 +199,7 @@ func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, erro
return blder.ctrl, nil
}
func (blder *Builder) project(obj runtime.Object, proj objectProjection) (runtime.Object, error) {
func (blder *Builder) project(obj client.Object, proj objectProjection) (client.Object, error) {
switch proj {
case projectAsNormal:
return obj, nil
@@ -279,12 +268,6 @@ func (blder *Builder) doWatch() error {
return nil
}
func (blder *Builder) loadRestConfig() {
if blder.config == nil {
blder.config = blder.mgr.GetConfig()
}
}
func (blder *Builder) getControllerName(gvk schema.GroupVersionKind) string {
if blder.name != "" {
return blder.name
@@ -293,6 +276,8 @@ func (blder *Builder) getControllerName(gvk schema.GroupVersionKind) string {
}
func (blder *Builder) doController(r reconcile.Reconciler) error {
globalOpts := blder.mgr.GetControllerOptions()
ctrlOptions := blder.ctrlOptions
if ctrlOptions.Reconciler == nil {
ctrlOptions.Reconciler = r
@@ -305,11 +290,25 @@ func (blder *Builder) doController(r reconcile.Reconciler) error {
return err
}
// Setup concurrency.
if ctrlOptions.MaxConcurrentReconciles == 0 {
groupKind := gvk.GroupKind().String()
if concurrency, ok := globalOpts.GroupKindConcurrency[groupKind]; ok && concurrency > 0 {
ctrlOptions.MaxConcurrentReconciles = concurrency
}
}
// Setup cache sync timeout.
if ctrlOptions.CacheSyncTimeout == 0 && globalOpts.CacheSyncTimeout != nil {
ctrlOptions.CacheSyncTimeout = *globalOpts.CacheSyncTimeout
}
// Setup the logger.
if ctrlOptions.Log == nil {
ctrlOptions.Log = blder.mgr.GetLogger()
}
ctrlOptions.Log = ctrlOptions.Log.WithValues("reconcilerGroup", gvk.Group, "reconcilerKind", gvk.Kind)
ctrlOptions.Log = ctrlOptions.Log.WithValues("reconciler group", gvk.Group, "reconciler kind", gvk.Kind)
// Build the controller and return.
blder.ctrl, err = newController(blder.getControllerName(gvk), blder.mgr, ctrlOptions)

View File

@@ -38,7 +38,7 @@ type WebhookBuilder struct {
config *rest.Config
}
// WebhookManagedBy allows inform its manager.Manager
// WebhookManagedBy allows inform its manager.Manager.
func WebhookManagedBy(m manager.Manager) *WebhookBuilder {
return &WebhookBuilder{mgr: m}
}
@@ -86,7 +86,7 @@ func (blder *WebhookBuilder) registerWebhooks() error {
return nil
}
// registerDefaultingWebhook registers a defaulting webhook if th
// registerDefaultingWebhook registers a defaulting webhook if th.
func (blder *WebhookBuilder) registerDefaultingWebhook() {
defaulter, isDefaulter := blder.apiType.(admission.Defaulter)
if !isDefaulter {
@@ -157,11 +157,11 @@ func (blder *WebhookBuilder) isAlreadyHandled(path string) bool {
}
func generateMutatePath(gvk schema.GroupVersionKind) string {
return "/mutate-" + strings.Replace(gvk.Group, ".", "-", -1) + "-" +
return "/mutate-" + strings.ReplaceAll(gvk.Group, ".", "-") + "-" +
gvk.Version + "-" + strings.ToLower(gvk.Kind)
}
func generateValidatePath(gvk schema.GroupVersionKind) string {
return "/validate-" + strings.Replace(gvk.Group, ".", "-", -1) + "-" +
return "/validate-" + strings.ReplaceAll(gvk.Group, ".", "-") + "-" +
gvk.Version + "-" + strings.ToLower(gvk.Kind)
}

View File

@@ -52,24 +52,24 @@ type Cache interface {
type Informers interface {
// GetInformer fetches or constructs an informer for the given object that corresponds to a single
// API kind and resource.
GetInformer(ctx context.Context, obj runtime.Object) (Informer, error)
GetInformer(ctx context.Context, obj client.Object) (Informer, error)
// GetInformerForKind is similar to GetInformer, except that it takes a group-version-kind, instead
// of the underlying object.
GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error)
// Start runs all the informers known to this cache until the given channel is closed.
// Start runs all the informers known to this cache until the context is closed.
// It blocks.
Start(stopCh <-chan struct{}) error
Start(ctx context.Context) error
// WaitForCacheSync waits for all the caches to sync. Returns false if it could not sync a cache.
WaitForCacheSync(stop <-chan struct{}) bool
WaitForCacheSync(ctx context.Context) bool
// Informers knows how to add indices to the caches (informers) that it manages.
client.FieldIndexer
}
// Informer - informer allows you interact with the underlying informer
// Informer - informer allows you interact with the underlying informer.
type Informer interface {
// AddEventHandler adds an event handler to the shared informer using the shared informer's resync
// period. Events to a single handler are delivered sequentially, but there is no coordination
@@ -82,11 +82,14 @@ type Informer interface {
// AddIndexers adds more indexers to this store. If you call this after you already have data
// in the store, the results are undefined.
AddIndexers(indexers toolscache.Indexers) error
//HasSynced return true if the informers underlying store has synced
// HasSynced return true if the informers underlying store has synced.
HasSynced() bool
}
// Options are the optional arguments for creating a new InformersMap object
// SelectorsByObject associate a client.Object's GVK to a field/label selector.
type SelectorsByObject map[client.Object]internal.Selector
// Options are the optional arguments for creating a new InformersMap object.
type Options struct {
// Scheme is the scheme to use for mapping objects to GroupVersionKinds
Scheme *runtime.Scheme
@@ -103,6 +106,13 @@ type Options struct {
// Namespace restricts the cache's ListWatch to the desired namespace
// Default watches all namespaces
Namespace string
// SelectorsByObject restricts the cache's ListWatch to the desired
// fields per GVK at the specified object, the map's value must implement
// Selector [1] using for example a Set [2]
// [1] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Selector
// [2] https://pkg.go.dev/k8s.io/apimachinery/pkg/fields#Set
SelectorsByObject SelectorsByObject
}
var defaultResyncTime = 10 * time.Hour
@@ -113,10 +123,38 @@ func New(config *rest.Config, opts Options) (Cache, error) {
if err != nil {
return nil, err
}
im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace)
selectorsByGVK, err := convertToSelectorsByGVK(opts.SelectorsByObject, opts.Scheme)
if err != nil {
return nil, err
}
im := internal.NewInformersMap(config, opts.Scheme, opts.Mapper, *opts.Resync, opts.Namespace, selectorsByGVK)
return &informerCache{InformersMap: im}, nil
}
// BuilderWithOptions returns a Cache constructor that will build the a cache
// honoring the options argument, this is useful to specify options like
// SelectorsByObject
// WARNING: if SelectorsByObject is specified. filtered out resources are not
// returned.
func BuilderWithOptions(options Options) NewCacheFunc {
return func(config *rest.Config, opts Options) (Cache, error) {
if opts.Scheme == nil {
opts.Scheme = options.Scheme
}
if opts.Mapper == nil {
opts.Mapper = options.Mapper
}
if opts.Resync == nil {
opts.Resync = options.Resync
}
if opts.Namespace == "" {
opts.Namespace = options.Namespace
}
opts.SelectorsByObject = options.SelectorsByObject
return New(config, opts)
}
}
func defaultOpts(config *rest.Config, opts Options) (Options, error) {
// Use the default Kubernetes Scheme if unset
if opts.Scheme == nil {
@@ -139,3 +177,15 @@ func defaultOpts(config *rest.Config, opts Options) (Options, error) {
}
return opts, nil
}
func convertToSelectorsByGVK(selectorsByObject SelectorsByObject, scheme *runtime.Scheme) (internal.SelectorsByGVK, error) {
selectorsByGVK := internal.SelectorsByGVK{}
for object, selector := range selectorsByObject {
gvk, err := apiutil.GVKForObject(object, scheme)
if err != nil {
return nil, err
}
selectorsByGVK[gvk] = selector
}
return selectorsByGVK, nil
}

View File

@@ -50,8 +50,8 @@ type informerCache struct {
*internal.InformersMap
}
// Get implements Reader
func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out runtime.Object) error {
// Get implements Reader.
func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out client.Object) error {
gvk, err := apiutil.GVKForObject(out, ip.Scheme)
if err != nil {
return err
@@ -68,9 +68,8 @@ func (ip *informerCache) Get(ctx context.Context, key client.ObjectKey, out runt
return cache.Reader.Get(ctx, key, out)
}
// List implements Reader
func (ip *informerCache) List(ctx context.Context, out runtime.Object, opts ...client.ListOption) error {
// List implements Reader.
func (ip *informerCache) List(ctx context.Context, out client.ObjectList, opts ...client.ListOption) error {
gvk, cacheTypeObj, err := ip.objectTypeForListObject(out)
if err != nil {
return err
@@ -91,7 +90,7 @@ func (ip *informerCache) List(ctx context.Context, out runtime.Object, opts ...c
// objectTypeForListObject tries to find the runtime.Object and associated GVK
// for a single object corresponding to the passed-in list type. We need them
// because they are used as cache map key.
func (ip *informerCache) objectTypeForListObject(list runtime.Object) (*schema.GroupVersionKind, runtime.Object, error) {
func (ip *informerCache) objectTypeForListObject(list client.ObjectList) (*schema.GroupVersionKind, runtime.Object, error) {
gvk, err := apiutil.GVKForObject(list, ip.Scheme)
if err != nil {
return nil, nil, err
@@ -130,7 +129,7 @@ func (ip *informerCache) objectTypeForListObject(list runtime.Object) (*schema.G
return &gvk, cacheTypeObj, nil
}
// GetInformerForKind returns the informer for the GroupVersionKind
// GetInformerForKind returns the informer for the GroupVersionKind.
func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
// Map the gvk to an object
obj, err := ip.Scheme.New(gvk)
@@ -145,8 +144,8 @@ func (ip *informerCache) GetInformerForKind(ctx context.Context, gvk schema.Grou
return i.Informer, err
}
// GetInformer returns the informer for the obj
func (ip *informerCache) GetInformer(ctx context.Context, obj runtime.Object) (Informer, error) {
// GetInformer returns the informer for the obj.
func (ip *informerCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) {
gvk, err := apiutil.GVKForObject(obj, ip.Scheme)
if err != nil {
return nil, err
@@ -160,7 +159,7 @@ func (ip *informerCache) GetInformer(ctx context.Context, obj runtime.Object) (I
}
// NeedLeaderElection implements the LeaderElectionRunnable interface
// to indicate that this can be started without requiring the leader lock
// to indicate that this can be started without requiring the leader lock.
func (ip *informerCache) NeedLeaderElection() bool {
return false
}
@@ -170,7 +169,7 @@ func (ip *informerCache) NeedLeaderElection() bool {
// to List. For one-to-one compatibility with "normal" field selectors, only return one value.
// The values may be anything. They will automatically be prefixed with the namespace of the
// given object, if present. The objects passed are guaranteed to be objects of the correct type.
func (ip *informerCache) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error {
func (ip *informerCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
informer, err := ip.GetInformer(ctx, obj)
if err != nil {
return err
@@ -181,7 +180,7 @@ func (ip *informerCache) IndexField(ctx context.Context, obj runtime.Object, fie
func indexByField(indexer Informer, field string, extractor client.IndexerFunc) error {
indexFunc := func(objRaw interface{}) ([]string, error) {
// TODO(directxman12): check if this is the correct type?
obj, isObj := objRaw.(runtime.Object)
obj, isObj := objRaw.(client.Object)
if !isObj {
return nil, fmt.Errorf("object of type %T is not an Object", objRaw)
}

View File

@@ -21,7 +21,7 @@ import (
"fmt"
"reflect"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
@@ -29,23 +29,30 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/selection"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
)
// CacheReader is a client.Reader
// CacheReader is a client.Reader.
var _ client.Reader = &CacheReader{}
// CacheReader wraps a cache.Index to implement the client.CacheReader interface for a single type
// CacheReader wraps a cache.Index to implement the client.CacheReader interface for a single type.
type CacheReader struct {
// indexer is the underlying indexer wrapped by this cache.
indexer cache.Indexer
// groupVersionKind is the group-version-kind of the resource.
groupVersionKind schema.GroupVersionKind
// scopeName is the scope of the resource (namespaced or cluster-scoped).
scopeName apimeta.RESTScopeName
}
// Get checks the indexer for the object and writes a copy of it if found
func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out runtime.Object) error {
// Get checks the indexer for the object and writes a copy of it if found.
func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out client.Object) error {
if c.scopeName == apimeta.RESTScopeNameRoot {
key.Namespace = ""
}
storeKey := objectKeyToStoreKey(key)
// Lookup the object from the indexer cache
@@ -57,7 +64,7 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out runtime.O
// Not found, return an error
if !exists {
// Resource gets transformed into Kind in the error anyway, so this is fine
return errors.NewNotFound(schema.GroupResource{
return apierrors.NewNotFound(schema.GroupResource{
Group: c.groupVersionKind.Group,
Resource: c.groupVersionKind.Kind,
}, key.Name)
@@ -86,15 +93,16 @@ func (c *CacheReader) Get(_ context.Context, key client.ObjectKey, out runtime.O
return nil
}
// List lists items out of the indexer and writes them to out
func (c *CacheReader) List(_ context.Context, out runtime.Object, opts ...client.ListOption) error {
// List lists items out of the indexer and writes them to out.
func (c *CacheReader) List(_ context.Context, out client.ObjectList, opts ...client.ListOption) error {
var objs []interface{}
var err error
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
if listOpts.FieldSelector != nil {
switch {
case listOpts.FieldSelector != nil:
// TODO(directxman12): support more complicated field selectors by
// combining multiple indices, GetIndexers, etc
field, val, requiresExact := requiresExactMatch(listOpts.FieldSelector)
@@ -105,9 +113,9 @@ func (c *CacheReader) List(_ context.Context, out runtime.Object, opts ...client
// namespaced index key. Otherwise, ask for the non-namespaced variant by using the fake "all namespaces"
// namespace.
objs, err = c.indexer.ByIndex(FieldIndexName(field), KeyToNamespacedKey(listOpts.Namespace, val))
} else if listOpts.Namespace != "" {
case listOpts.Namespace != "":
objs, err = c.indexer.ByIndex(cache.NamespaceIndex, listOpts.Namespace)
} else {
default:
objs = c.indexer.List()
}
if err != nil {
@@ -118,8 +126,15 @@ func (c *CacheReader) List(_ context.Context, out runtime.Object, opts ...client
labelSel = listOpts.LabelSelector
}
limitSet := listOpts.Limit > 0
runtimeObjs := make([]runtime.Object, 0, len(objs))
for _, item := range objs {
for i, item := range objs {
// if the Limit option is set and the number of items
// listed exceeds this limit, then stop reading.
if limitSet && int64(i) >= listOpts.Limit {
break
}
obj, isObj := item.(runtime.Object)
if !isObj {
return fmt.Errorf("cache contained %T, which is not an Object", obj)
@@ -172,7 +187,7 @@ func FieldIndexName(field string) string {
return "field:" + field
}
// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces
// noNamespaceNamespace is used as the "namespace" when we want to list across all namespaces.
const allNamespacesNamespace = "__all_namespaces"
// KeyToNamespacedKey prefixes the given index key with a namespace

View File

@@ -49,41 +49,43 @@ func NewInformersMap(config *rest.Config,
scheme *runtime.Scheme,
mapper meta.RESTMapper,
resync time.Duration,
namespace string) *InformersMap {
namespace string,
selectors SelectorsByGVK,
) *InformersMap {
return &InformersMap{
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace),
structured: newStructuredInformersMap(config, scheme, mapper, resync, namespace, selectors),
unstructured: newUnstructuredInformersMap(config, scheme, mapper, resync, namespace, selectors),
metadata: newMetadataInformersMap(config, scheme, mapper, resync, namespace, selectors),
Scheme: scheme,
}
}
// Start calls Run on each of the informers and sets started to true. Blocks on the stop channel.
func (m *InformersMap) Start(stop <-chan struct{}) error {
go m.structured.Start(stop)
go m.unstructured.Start(stop)
go m.metadata.Start(stop)
<-stop
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
func (m *InformersMap) Start(ctx context.Context) error {
go m.structured.Start(ctx)
go m.unstructured.Start(ctx)
go m.metadata.Start(ctx)
<-ctx.Done()
return nil
}
// WaitForCacheSync waits until all the caches have been started and synced.
func (m *InformersMap) WaitForCacheSync(stop <-chan struct{}) bool {
func (m *InformersMap) WaitForCacheSync(ctx context.Context) bool {
syncedFuncs := append([]cache.InformerSynced(nil), m.structured.HasSyncedFuncs()...)
syncedFuncs = append(syncedFuncs, m.unstructured.HasSyncedFuncs()...)
syncedFuncs = append(syncedFuncs, m.metadata.HasSyncedFuncs()...)
if !m.structured.waitForStarted(stop) {
if !m.structured.waitForStarted(ctx) {
return false
}
if !m.unstructured.waitForStarted(stop) {
if !m.unstructured.waitForStarted(ctx) {
return false
}
if !m.metadata.waitForStarted(stop) {
if !m.metadata.waitForStarted(ctx) {
return false
}
return cache.WaitForCacheSync(stop, syncedFuncs...)
return cache.WaitForCacheSync(ctx.Done(), syncedFuncs...)
}
// Get will create a new Informer and add it to the map of InformersMap if none exists. Returns
@@ -104,16 +106,19 @@ func (m *InformersMap) Get(ctx context.Context, gvk schema.GroupVersionKind, obj
}
// newStructuredInformersMap creates a new InformersMap for structured objects.
func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createStructuredListWatch)
func newStructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createStructuredListWatch)
}
// newUnstructuredInformersMap creates a new InformersMap for unstructured objects.
func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createUnstructuredListWatch)
func newUnstructuredInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createUnstructuredListWatch)
}
// newMetadataInformersMap creates a new InformersMap for metadata-only objects.
func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration, namespace string) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, createMetadataListWatch)
func newMetadataInformersMap(config *rest.Config, scheme *runtime.Scheme, mapper meta.RESTMapper, resync time.Duration,
namespace string, selectors SelectorsByGVK) *specificInformersMap {
return newSpecificInformersMap(config, scheme, mapper, resync, namespace, selectors, createMetadataListWatch)
}

View File

@@ -34,11 +34,14 @@ import (
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// clientListWatcherFunc knows how to create a ListWatcher
func init() {
rand.Seed(time.Now().UnixNano())
}
// clientListWatcherFunc knows how to create a ListWatcher.
type createListWatcherFunc func(gvk schema.GroupVersionKind, ip *specificInformersMap) (*cache.ListWatch, error)
// newSpecificInformersMap returns a new specificInformersMap (like
@@ -48,6 +51,7 @@ func newSpecificInformersMap(config *rest.Config,
mapper meta.RESTMapper,
resync time.Duration,
namespace string,
selectors SelectorsByGVK,
createListWatcher createListWatcherFunc) *specificInformersMap {
ip := &specificInformersMap{
config: config,
@@ -60,11 +64,12 @@ func newSpecificInformersMap(config *rest.Config,
startWait: make(chan struct{}),
createListWatcher: createListWatcher,
namespace: namespace,
selectors: selectors,
}
return ip
}
// MapEntry contains the cached data for an Informer
// MapEntry contains the cached data for an Informer.
type MapEntry struct {
// Informer is the cached informer
Informer cache.SharedIndexInformer
@@ -120,35 +125,39 @@ type specificInformersMap struct {
// namespace is the namespace that all ListWatches are restricted to
// default or empty string means all namespaces
namespace string
// selectors are the label or field selectors that will be added to the
// ListWatch ListOptions.
selectors SelectorsByGVK
}
// Start calls Run on each of the informers and sets started to true. Blocks on the stop channel.
// Start calls Run on each of the informers and sets started to true. Blocks on the context.
// It doesn't return start because it can't return an error, and it's not a runnable directly.
func (ip *specificInformersMap) Start(stop <-chan struct{}) {
func (ip *specificInformersMap) Start(ctx context.Context) {
func() {
ip.mu.Lock()
defer ip.mu.Unlock()
// Set the stop channel so it can be passed to informers that are added later
ip.stop = stop
ip.stop = ctx.Done()
// Start each informer
for _, informer := range ip.informersByGVK {
go informer.Informer.Run(stop)
go informer.Informer.Run(ctx.Done())
}
// Set started to true so we immediately start any informers added later.
ip.started = true
close(ip.startWait)
}()
<-stop
<-ctx.Done()
}
func (ip *specificInformersMap) waitForStarted(stop <-chan struct{}) bool {
func (ip *specificInformersMap) waitForStarted(ctx context.Context) bool {
select {
case <-ip.startWait:
return true
case <-stop:
case <-ctx.Done():
return false
}
}
@@ -212,9 +221,20 @@ func (ip *specificInformersMap) addInformerToMap(gvk schema.GroupVersionKind, ob
ni := cache.NewSharedIndexInformer(lw, obj, resyncPeriod(ip.resync)(), cache.Indexers{
cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
})
rm, err := ip.mapper.RESTMapping(gvk.GroupKind(), gvk.Version)
if err != nil {
return nil, false, err
}
switch obj.(type) {
case *metav1.PartialObjectMetadata, *metav1.PartialObjectMetadataList:
ni = metadataSharedIndexInformerPreserveGVK(gvk, ni)
default:
}
i := &MapEntry{
Informer: ni,
Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk},
Reader: CacheReader{indexer: ni.GetIndexer(), groupVersionKind: gvk, scopeName: rm.Scope.Name()},
}
ip.informersByGVK[gvk] = i
@@ -236,7 +256,7 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
return nil, err
}
client, err := apiutil.RESTClientForGVK(gvk, ip.config, ip.codecs)
client, err := apiutil.RESTClientForGVK(gvk, false, ip.config, ip.codecs)
if err != nil {
return nil, err
}
@@ -252,6 +272,7 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
res := listObj.DeepCopyObject()
isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
err := client.Get().NamespaceIfScoped(ip.namespace, isNamespaceScoped).Resource(mapping.Resource.Resource).VersionedParams(&opts, ip.paramCodec).Do(ctx).Into(res)
@@ -259,6 +280,7 @@ func createStructuredListWatch(gvk schema.GroupVersionKind, ip *specificInformer
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
isNamespaceScoped := ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot
@@ -274,7 +296,12 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
if err != nil {
return nil, err
}
dynamicClient, err := dynamic.NewForConfig(ip.config)
// If the rest configuration has a negotiated serializer passed in,
// we should remove it and use the one that the dynamic client sets for us.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
dynamicClient, err := dynamic.NewForConfig(cfg)
if err != nil {
return nil, err
}
@@ -285,6 +312,7 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
// Create a new ListWatch for the obj
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return dynamicClient.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts)
}
@@ -292,6 +320,7 @@ func createUnstructuredListWatch(gvk schema.GroupVersionKind, ip *specificInform
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
@@ -310,8 +339,13 @@ func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersM
return nil, err
}
// Always clear the negotiated serializer and use the one
// set from the metadata client.
cfg := rest.CopyConfig(ip.config)
cfg.NegotiatedSerializer = nil
// grab the metadata client
client, err := metadata.NewForConfig(ip.config)
client, err := metadata.NewForConfig(cfg)
if err != nil {
return nil, err
}
@@ -320,9 +354,10 @@ func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersM
// pass in their own contexts instead of relying on this fixed one here.
ctx := context.TODO()
// create the relevant listwaatch
// create the relevant listwatch
return &cache.ListWatch{
ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
ip.selectors[gvk].ApplyToList(&opts)
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
return client.Resource(mapping.Resource).Namespace(ip.namespace).List(ctx, opts)
}
@@ -330,6 +365,7 @@ func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersM
},
// Setup the watch function
WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
ip.selectors[gvk].ApplyToList(&opts)
// Watch needs to be set to true separately
opts.Watch = true
if ip.namespace != "" && mapping.Scope.Name() != meta.RESTScopeNameRoot {
@@ -346,7 +382,7 @@ func createMetadataListWatch(gvk schema.GroupVersionKind, ip *specificInformersM
func resyncPeriod(resync time.Duration) func() time.Duration {
return func() time.Duration {
// the factor will fall into [0.9, 1.1)
factor := rand.Float64()/5.0 + 0.9
factor := rand.Float64()/5.0 + 0.9 //nolint:gosec
return time.Duration(float64(resync.Nanoseconds()) * factor)
}
}

View File

@@ -0,0 +1,71 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
"time"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/tools/cache"
)
func metadataSharedIndexInformerPreserveGVK(gvk schema.GroupVersionKind, si cache.SharedIndexInformer) cache.SharedIndexInformer {
return &sharedInformerWrapper{
gvk: gvk,
SharedIndexInformer: si,
}
}
type sharedInformerWrapper struct {
gvk schema.GroupVersionKind
cache.SharedIndexInformer
}
func (s *sharedInformerWrapper) AddEventHandler(handler cache.ResourceEventHandler) {
s.SharedIndexInformer.AddEventHandler(&handlerPreserveGVK{s.gvk, handler})
}
func (s *sharedInformerWrapper) AddEventHandlerWithResyncPeriod(handler cache.ResourceEventHandler, resyncPeriod time.Duration) {
s.SharedIndexInformer.AddEventHandlerWithResyncPeriod(&handlerPreserveGVK{s.gvk, handler}, resyncPeriod)
}
type handlerPreserveGVK struct {
gvk schema.GroupVersionKind
cache.ResourceEventHandler
}
func (h *handlerPreserveGVK) resetGroupVersionKind(obj interface{}) {
if v, ok := obj.(schema.ObjectKind); ok {
v.SetGroupVersionKind(h.gvk)
}
}
func (h *handlerPreserveGVK) OnAdd(obj interface{}) {
h.resetGroupVersionKind(obj)
h.ResourceEventHandler.OnAdd(obj)
}
func (h *handlerPreserveGVK) OnUpdate(oldObj, newObj interface{}) {
h.resetGroupVersionKind(oldObj)
h.resetGroupVersionKind(newObj)
h.ResourceEventHandler.OnUpdate(oldObj, newObj)
}
func (h *handlerPreserveGVK) OnDelete(obj interface{}) {
h.resetGroupVersionKind(obj)
h.ResourceEventHandler.OnDelete(obj)
}

View File

@@ -0,0 +1,43 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// SelectorsByGVK associate a GroupVersionKind to a field/label selector.
type SelectorsByGVK map[schema.GroupVersionKind]Selector
// Selector specify the label/field selector to fill in ListOptions.
type Selector struct {
Label labels.Selector
Field fields.Selector
}
// ApplyToList fill in ListOptions LabelSelector and FieldSelector if needed.
func (s Selector) ApplyToList(listOpts *metav1.ListOptions) {
if s.Label != nil {
listOpts.LabelSelector = s.Label.String()
}
if s.Field != nil {
listOpts.FieldSelector = s.Field.String()
}
}

View File

@@ -22,21 +22,25 @@ import (
"time"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/meta"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/rest"
toolscache "k8s.io/client-go/tools/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/objectutil"
)
// NewCacheFunc - Function for creating a new cache from the options and a rest config
// NewCacheFunc - Function for creating a new cache from the options and a rest config.
type NewCacheFunc func(config *rest.Config, opts Options) (Cache, error)
// a new global namespaced cache to handle cluster scoped resources.
const globalCache = "_cluster-scope"
// MultiNamespacedCacheBuilder - Builder function to create a new multi-namespaced cache.
// This will scope the cache to a list of namespaces. Listing for all namespaces
// will list for all the namespaces that this knows about. Note that this is not intended
// will list for all the namespaces that this knows about. By default this will create
// a global cache for cluster scoped resource. Note that this is not intended
// to be used for excluding namespaces, this is better done via a Predicate. Also note that
// you may face performance issues when using this with a high number of namespaces.
func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
@@ -45,7 +49,15 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
if err != nil {
return nil, err
}
caches := map[string]Cache{}
// create a cache for cluster scoped resources
gCache, err := New(config, opts)
if err != nil {
return nil, fmt.Errorf("error creating global cache %v", err)
}
for _, ns := range namespaces {
opts.Namespace = ns
c, err := New(config, opts)
@@ -54,7 +66,7 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
}
caches[ns] = c
}
return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme}, nil
return &multiNamespaceCache{namespaceToCache: caches, Scheme: opts.Scheme, RESTMapper: opts.Mapper, clusterCache: gCache}, nil
}
}
@@ -65,13 +77,32 @@ func MultiNamespacedCacheBuilder(namespaces []string) NewCacheFunc {
type multiNamespaceCache struct {
namespaceToCache map[string]Cache
Scheme *runtime.Scheme
RESTMapper apimeta.RESTMapper
clusterCache Cache
}
var _ Cache = &multiNamespaceCache{}
// Methods for multiNamespaceCache to conform to the Informers interface
func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj runtime.Object) (Informer, error) {
// Methods for multiNamespaceCache to conform to the Informers interface.
func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj client.Object) (Informer, error) {
informers := map[string]Informer{}
// If the object is clusterscoped, get the informer from clusterCache,
// if not use the namespaced caches.
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return nil, err
}
if !isNamespaced {
clusterCacheInf, err := c.clusterCache.GetInformer(ctx, obj)
if err != nil {
return nil, err
}
informers[globalCache] = clusterCacheInf
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformer(ctx, obj)
if err != nil {
@@ -79,11 +110,29 @@ func (c *multiNamespaceCache) GetInformer(ctx context.Context, obj runtime.Objec
}
informers[ns] = informer
}
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
}
func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema.GroupVersionKind) (Informer, error) {
informers := map[string]Informer{}
// If the object is clusterscoped, get the informer from clusterCache,
// if not use the namespaced caches.
isNamespaced, err := objectutil.IsAPINamespacedWithGVK(gvk, c.Scheme, c.RESTMapper)
if err != nil {
return nil, err
}
if !isNamespaced {
clusterCacheInf, err := c.clusterCache.GetInformerForKind(ctx, gvk)
if err != nil {
return nil, err
}
informers[globalCache] = clusterCacheInf
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
}
for ns, cache := range c.namespaceToCache {
informer, err := cache.GetInformerForKind(ctx, gvk)
if err != nil {
@@ -91,33 +140,58 @@ func (c *multiNamespaceCache) GetInformerForKind(ctx context.Context, gvk schema
}
informers[ns] = informer
}
return &multiNamespaceInformer{namespaceToInformer: informers}, nil
}
func (c *multiNamespaceCache) Start(stopCh <-chan struct{}) error {
func (c *multiNamespaceCache) Start(ctx context.Context) error {
// start global cache
go func() {
err := c.clusterCache.Start(ctx)
if err != nil {
log.Error(err, "cluster scoped cache failed to start")
}
}()
// start namespaced caches
for ns, cache := range c.namespaceToCache {
go func(ns string, cache Cache) {
err := cache.Start(stopCh)
err := cache.Start(ctx)
if err != nil {
log.Error(err, "multinamespace cache failed to start namespaced informer", "namespace", ns)
}
}(ns, cache)
}
<-stopCh
<-ctx.Done()
return nil
}
func (c *multiNamespaceCache) WaitForCacheSync(stop <-chan struct{}) bool {
func (c *multiNamespaceCache) WaitForCacheSync(ctx context.Context) bool {
synced := true
for _, cache := range c.namespaceToCache {
if s := cache.WaitForCacheSync(stop); !s {
if s := cache.WaitForCacheSync(ctx); !s {
synced = s
}
}
// check if cluster scoped cache has synced
if !c.clusterCache.WaitForCacheSync(ctx) {
synced = false
}
return synced
}
func (c *multiNamespaceCache) IndexField(ctx context.Context, obj runtime.Object, field string, extractValue client.IndexerFunc) error {
func (c *multiNamespaceCache) IndexField(ctx context.Context, obj client.Object, field string, extractValue client.IndexerFunc) error {
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return nil //nolint:nilerr
}
if !isNamespaced {
return c.clusterCache.IndexField(ctx, obj, field, extractValue)
}
for _, cache := range c.namespaceToCache {
if err := cache.IndexField(ctx, obj, field, extractValue); err != nil {
return err
@@ -126,7 +200,17 @@ func (c *multiNamespaceCache) IndexField(ctx context.Context, obj runtime.Object
return nil
}
func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
isNamespaced, err := objectutil.IsAPINamespaced(obj, c.Scheme, c.RESTMapper)
if err != nil {
return err
}
if !isNamespaced {
// Look into the global cache to fetch the object
return c.clusterCache.Get(ctx, key, obj)
}
cache, ok := c.namespaceToCache[key.Namespace]
if !ok {
return fmt.Errorf("unable to get: %v because of unknown namespace for the cache", key)
@@ -135,9 +219,20 @@ func (c *multiNamespaceCache) Get(ctx context.Context, key client.ObjectKey, obj
}
// List multi namespace cache will get all the objects in the namespaces that the cache is watching if asked for all namespaces.
func (c *multiNamespaceCache) List(ctx context.Context, list runtime.Object, opts ...client.ListOption) error {
func (c *multiNamespaceCache) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
isNamespaced, err := objectutil.IsAPINamespaced(list, c.Scheme, c.RESTMapper)
if err != nil {
return err
}
if !isNamespaced {
// Look at the global cache to get the objects with the specified GVK
return c.clusterCache.List(ctx, list, opts...)
}
if listOpts.Namespace != corev1.NamespaceAll {
cache, ok := c.namespaceToCache[listOpts.Namespace]
if !ok {
@@ -146,7 +241,7 @@ func (c *multiNamespaceCache) List(ctx context.Context, list runtime.Object, opt
return cache.List(ctx, list, opts...)
}
listAccessor, err := meta.ListAccessor(list)
listAccessor, err := apimeta.ListAccessor(list)
if err != nil {
return err
}
@@ -155,10 +250,13 @@ func (c *multiNamespaceCache) List(ctx context.Context, list runtime.Object, opt
if err != nil {
return err
}
limitSet := listOpts.Limit > 0
var resourceVersion string
for _, cache := range c.namespaceToCache {
listObj := list.DeepCopyObject()
err = cache.List(ctx, listObj, opts...)
listObj := list.DeepCopyObject().(client.ObjectList)
err = cache.List(ctx, listObj, &listOpts)
if err != nil {
return err
}
@@ -166,41 +264,52 @@ func (c *multiNamespaceCache) List(ctx context.Context, list runtime.Object, opt
if err != nil {
return err
}
accessor, err := meta.ListAccessor(listObj)
accessor, err := apimeta.ListAccessor(listObj)
if err != nil {
return fmt.Errorf("object: %T must be a list type", list)
}
allItems = append(allItems, items...)
// The last list call should have the most correct resource version.
resourceVersion = accessor.GetResourceVersion()
if limitSet {
// decrement Limit by the number of items
// fetched from the current namespace.
listOpts.Limit -= int64(len(items))
// if a Limit was set and the number of
// items read has reached this set limit,
// then stop reading.
if listOpts.Limit == 0 {
break
}
}
}
listAccessor.SetResourceVersion(resourceVersion)
return apimeta.SetList(list, allItems)
}
// multiNamespaceInformer knows how to handle interacting with the underlying informer across multiple namespaces
// multiNamespaceInformer knows how to handle interacting with the underlying informer across multiple namespaces.
type multiNamespaceInformer struct {
namespaceToInformer map[string]Informer
}
var _ Informer = &multiNamespaceInformer{}
// AddEventHandler adds the handler to each namespaced informer
// AddEventHandler adds the handler to each namespaced informer.
func (i *multiNamespaceInformer) AddEventHandler(handler toolscache.ResourceEventHandler) {
for _, informer := range i.namespaceToInformer {
informer.AddEventHandler(handler)
}
}
// AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer
// AddEventHandlerWithResyncPeriod adds the handler with a resync period to each namespaced informer.
func (i *multiNamespaceInformer) AddEventHandlerWithResyncPeriod(handler toolscache.ResourceEventHandler, resyncPeriod time.Duration) {
for _, informer := range i.namespaceToInformer {
informer.AddEventHandlerWithResyncPeriod(handler, resyncPeriod)
}
}
// AddIndexers adds the indexer for each namespaced informer
// AddIndexers adds the indexer for each namespaced informer.
func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error {
for _, informer := range i.namespaceToInformer {
err := informer.AddIndexers(indexers)
@@ -211,7 +320,7 @@ func (i *multiNamespaceInformer) AddIndexers(indexers toolscache.Indexers) error
return nil
}
// HasSynced checks if each namespaced informer has synced
// HasSynced checks if each namespaced informer has synced.
func (i *multiNamespaceInformer) HasSynced() bool {
for _, informer := range i.namespaceToInformer {
if ok := informer.HasSynced(); !ok {

View File

@@ -1,5 +1,5 @@
/*
Copyright 2019 The Kubernetes Authors.
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ limitations under the License.
package certwatcher
import (
"context"
"crypto/tls"
"sync"
@@ -30,7 +31,7 @@ var log = logf.RuntimeLog.WithName("certwatcher")
// changes, it reads and parses both and calls an optional callback with the new
// certificate.
type CertWatcher struct {
sync.Mutex
sync.RWMutex
currentCert *tls.Certificate
watcher *fsnotify.Watcher
@@ -63,13 +64,13 @@ func New(certPath, keyPath string) (*CertWatcher, error) {
// GetCertificate fetches the currently loaded certificate, which may be nil.
func (cw *CertWatcher) GetCertificate(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
cw.Lock()
defer cw.Unlock()
cw.RLock()
defer cw.RUnlock()
return cw.currentCert, nil
}
// Start starts the watch on the certificate and key files.
func (cw *CertWatcher) Start(stopCh <-chan struct{}) error {
func (cw *CertWatcher) Start(ctx context.Context) error {
files := []string{cw.certPath, cw.keyPath}
for _, f := range files {
@@ -82,8 +83,8 @@ func (cw *CertWatcher) Start(stopCh <-chan struct{}) error {
log.Info("Starting certificate watcher")
// Block until the stop channel is closed.
<-stopCh
// Block until the context is done.
<-ctx.Done()
return cw.watcher.Close()
}

View File

@@ -1,5 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,17 +14,10 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package error
import "fmt"
// SecretError represents error with a secret.
type SecretError struct {
KustomizationPath string
// ErrorMsg is an error message
ErrorMsg string
}
func (e SecretError) Error() string {
return fmt.Sprintf("Kustomization file [%s] encounters a secret error: %s\n", e.KustomizationPath, e.ErrorMsg)
}
/*
Package certwatcher is a helper for reloading Certificates from disk to be used
with tls servers. It provides a helper func `GetCertificate` which can be
called from `tls.Config` and passed into your tls.Listener. For a detailed
example server view pkg/webhook/server.go.
*/
package certwatcher

View File

@@ -21,6 +21,7 @@ package apiutil
import (
"fmt"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,10 +29,33 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/discovery"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
)
var (
protobufScheme = runtime.NewScheme()
protobufSchemeLock sync.RWMutex
)
func init() {
// Currently only enabled for built-in resources which are guaranteed to implement Protocol Buffers.
// For custom resources, CRDs can not support Protocol Buffers but Aggregated API can.
// See doc: https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#advanced-features-and-flexibility
if err := clientgoscheme.AddToScheme(protobufScheme); err != nil {
panic(err)
}
}
// AddToProtobufScheme add the given SchemeBuilder into protobufScheme, which should
// be additional types that do support protobuf.
func AddToProtobufScheme(addToScheme func(*runtime.Scheme) error) error {
protobufSchemeLock.Lock()
defer protobufSchemeLock.Unlock()
return addToScheme(protobufScheme)
}
// NewDiscoveryRESTMapper constructs a new RESTMapper based on discovery
// information fetched by a new client with the given config.
func NewDiscoveryRESTMapper(c *rest.Config) (meta.RESTMapper, error) {
@@ -56,7 +80,7 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi
// (unstructured, partial, etc)
// check for PartialObjectMetadata, which is analogous to unstructured, but isn't handled by ObjectKinds
_, isPartial := obj.(*metav1.PartialObjectMetadata)
_, isPartial := obj.(*metav1.PartialObjectMetadata) //nolint:ifshort
_, isPartialList := obj.(*metav1.PartialObjectMetadataList)
if isPartial || isPartialList {
// we require that the GVK be populated in order to recognize the object
@@ -93,16 +117,25 @@ func GVKForObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersi
// RESTClientForGVK constructs a new rest.Interface capable of accessing the resource associated
// with the given GroupVersionKind. The REST client will be configured to use the negotiated serializer from
// baseConfig, if set, otherwise a default serializer will be set.
func RESTClientForGVK(gvk schema.GroupVersionKind, baseConfig *rest.Config, codecs serializer.CodecFactory) (rest.Interface, error) {
cfg := createRestConfig(gvk, baseConfig)
if cfg.NegotiatedSerializer == nil {
cfg.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: codecs}
}
return rest.RESTClientFor(cfg)
func RESTClientForGVK(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) (rest.Interface, error) {
return rest.RESTClientFor(createRestConfig(gvk, isUnstructured, baseConfig, codecs))
}
//createRestConfig copies the base config and updates needed fields for a new rest config
func createRestConfig(gvk schema.GroupVersionKind, baseConfig *rest.Config) *rest.Config {
// serializerWithDecodedGVK is a CodecFactory that overrides the DecoderToVersion of a WithoutConversionCodecFactory
// in order to avoid clearing the GVK from the decoded object.
//
// See https://github.com/kubernetes/kubernetes/issues/80609.
type serializerWithDecodedGVK struct {
serializer.WithoutConversionCodecFactory
}
// DecoderToVersion returns an decoder that does not do conversion.
func (f serializerWithDecodedGVK) DecoderToVersion(serializer runtime.Decoder, _ runtime.GroupVersioner) runtime.Decoder {
return serializer
}
// createRestConfig copies the base config and updates needed fields for a new rest config.
func createRestConfig(gvk schema.GroupVersionKind, isUnstructured bool, baseConfig *rest.Config, codecs serializer.CodecFactory) *rest.Config {
gv := gvk.GroupVersion()
cfg := rest.CopyConfig(baseConfig)
@@ -115,5 +148,24 @@ func createRestConfig(gvk schema.GroupVersionKind, baseConfig *rest.Config) *res
if cfg.UserAgent == "" {
cfg.UserAgent = rest.DefaultKubernetesUserAgent()
}
// TODO(FillZpp): In the long run, we want to check discovery or something to make sure that this is actually true.
if cfg.ContentType == "" && !isUnstructured {
protobufSchemeLock.RLock()
if protobufScheme.Recognizes(gvk) {
cfg.ContentType = runtime.ContentTypeProtobuf
}
protobufSchemeLock.RUnlock()
}
if cfg.NegotiatedSerializer == nil {
if isUnstructured {
// If the object is unstructured, we need to preserve the GVK information.
// Use our own custom serializer.
cfg.NegotiatedSerializer = serializerWithDecodedGVK{serializer.WithoutConversionCodecFactory{CodecFactory: codecs}}
} else {
cfg.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: codecs}
}
}
return cfg
}

View File

@@ -19,7 +19,6 @@ package apiutil
import (
"errors"
"sync"
"time"
"golang.org/x/time/rate"
"k8s.io/apimachinery/pkg/api/meta"
@@ -29,34 +28,12 @@ import (
"k8s.io/client-go/restmapper"
)
// ErrRateLimited is returned by a RESTMapper method if the number of API
// calls has exceeded a limit within a certain time period.
type ErrRateLimited struct {
// Duration to wait until the next API call can be made.
Delay time.Duration
}
func (e ErrRateLimited) Error() string {
return "too many API calls to the RESTMapper within a timeframe"
}
// DelayIfRateLimited returns the delay time until the next API call is
// allowed and true if err is of type ErrRateLimited. The zero
// time.Duration value and false are returned if err is not a ErrRateLimited.
func DelayIfRateLimited(err error) (time.Duration, bool) {
var rlerr ErrRateLimited
if errors.As(err, &rlerr) {
return rlerr.Delay, true
}
return 0, false
}
// dynamicRESTMapper is a RESTMapper that dynamically discovers resource
// types at runtime.
type dynamicRESTMapper struct {
mu sync.RWMutex // protects the following fields
staticMapper meta.RESTMapper
limiter *dynamicLimiter
limiter *rate.Limiter
newMapper func() (meta.RESTMapper, error)
lazy bool
@@ -64,13 +41,13 @@ type dynamicRESTMapper struct {
initOnce sync.Once
}
// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper
// DynamicRESTMapperOption is a functional option on the dynamicRESTMapper.
type DynamicRESTMapperOption func(*dynamicRESTMapper) error
// WithLimiter sets the RESTMapper's underlying limiter to lim.
func WithLimiter(lim *rate.Limiter) DynamicRESTMapperOption {
return func(drm *dynamicRESTMapper) error {
drm.limiter = &dynamicLimiter{lim}
drm.limiter = lim
return nil
}
}
@@ -103,9 +80,7 @@ func NewDynamicRESTMapper(cfg *rest.Config, opts ...DynamicRESTMapperOption) (me
return nil, err
}
drm := &dynamicRESTMapper{
limiter: &dynamicLimiter{
rate.NewLimiter(rate.Limit(defaultRefillRate), defaultLimitSize),
},
limiter: rate.NewLimiter(rate.Limit(defaultRefillRate), defaultLimitSize),
newMapper: func() (meta.RESTMapper, error) {
groupResources, err := restmapper.GetAPIGroupResources(client)
if err != nil {
@@ -161,12 +136,13 @@ func (drm *dynamicRESTMapper) init() (err error) {
// checkAndReload attempts to call the given callback, which is assumed to be dependent
// on the data in the restmapper.
//
// If the callback returns a NoKindMatchError, it will attempt to reload
// If the callback returns an error that matches the given error, it will attempt to reload
// the RESTMapper's data and re-call the callback once that's occurred.
// If the callback returns any other error, the function will return immediately regardless.
//
// It will take care
// ensuring that reloads are rate-limitted and that extraneous calls aren't made.
// It will take care of ensuring that reloads are rate-limited and that extraneous calls
// aren't made. If a reload would exceed the limiters rate, it returns the error return by
// the callback.
// It's thread-safe, and worries about thread-safety for the callback (so the callback does
// not need to attempt to lock the restmapper).
func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsReload func() error) error {
@@ -199,7 +175,9 @@ func (drm *dynamicRESTMapper) checkAndReload(needsReloadErr error, checkNeedsRel
}
// we're still stale, so grab a rate-limit token if we can...
if err := drm.limiter.checkRate(); err != nil {
if !drm.limiter.Allow() {
// return error from static mapper here, we have refreshed often enough (exceeding rate of provided limiter)
// so that client's can handle this the same way as a "normal" NoResourceMatchError / NoKindMatchError
return err
}
@@ -305,19 +283,3 @@ func (drm *dynamicRESTMapper) ResourceSingularizer(resource string) (string, err
})
return singular, err
}
// dynamicLimiter holds a rate limiter used to throttle chatty RESTMapper users.
type dynamicLimiter struct {
*rate.Limiter
}
// checkRate returns an ErrRateLimited if too many API calls have been made
// within the set limit.
func (b *dynamicLimiter) checkRate() error {
res := b.Reserve()
if res.Delay() == 0 {
return nil
}
res.Cancel()
return ErrRateLimited{res.Delay()}
}

View File

@@ -19,6 +19,7 @@ package client
import (
"context"
"fmt"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -29,16 +30,36 @@ import (
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/metadata"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
"sigs.k8s.io/controller-runtime/pkg/log"
)
// Options are creation options for a Client
// WarningHandlerOptions are options for configuring a
// warning handler for the client which is responsible
// for surfacing API Server warnings.
type WarningHandlerOptions struct {
// SuppressWarnings decides if the warnings from the
// API server are suppressed or surfaced in the client.
SuppressWarnings bool
// AllowDuplicateLogs does not deduplicate the to-be
// logged surfaced warnings messages. See
// log.WarningHandlerOptions for considerations
// regarding deuplication
AllowDuplicateLogs bool
}
// Options are creation options for a Client.
type Options struct {
// Scheme, if provided, will be used to map go structs to GroupVersionKinds
Scheme *runtime.Scheme
// Mapper, if provided, will be used to map GroupVersionKinds to Resources
Mapper meta.RESTMapper
// Opts is used to configure the warning handler responsible for
// surfacing and handling warnings messages sent by the API server.
Opts WarningHandlerOptions
}
// New returns a new Client using the provided config and Options.
@@ -52,10 +73,31 @@ type Options struct {
// case of unstructured types, the group, version, and kind will be extracted
// from the corresponding fields on the object.
func New(config *rest.Config, options Options) (Client, error) {
return newClient(config, options)
}
func newClient(config *rest.Config, options Options) (*client, error) {
if config == nil {
return nil, fmt.Errorf("must provide non-nil rest.Config to client.New")
}
if !options.Opts.SuppressWarnings {
// surface warnings
logger := log.Log.WithName("KubeAPIWarningLogger")
// Set a WarningHandler, the default WarningHandler
// is log.KubeAPIWarningLogger with deduplication enabled.
// See log.KubeAPIWarningLoggerOptions for considerations
// regarding deduplication.
rest.SetDefaultWarningHandler(
log.NewKubeAPIWarningLogger(
logger,
log.KubeAPIWarningLoggerOptions{
Deduplicate: !options.Opts.AllowDuplicateLogs,
},
),
)
}
// Init a scheme if none provided
if options.Scheme == nil {
options.Scheme = scheme.Scheme
@@ -71,11 +113,13 @@ func New(config *rest.Config, options Options) (Client, error) {
}
clientcache := &clientCache{
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
resourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
config: config,
scheme: options.Scheme,
mapper: options.Mapper,
codecs: serializer.NewCodecFactory(options.Scheme),
structuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
unstructuredResourceByType: make(map[schema.GroupVersionKind]*resourceMeta),
}
rawMetaClient, err := metadata.NewForConfig(config)
@@ -96,6 +140,8 @@ func New(config *rest.Config, options Options) (Client, error) {
client: rawMetaClient,
restMapper: options.Mapper,
},
scheme: options.Scheme,
mapper: options.Mapper,
}
return c, nil
@@ -109,10 +155,11 @@ type client struct {
typedClient typedClient
unstructuredClient unstructuredClient
metadataClient metadataClient
scheme *runtime.Scheme
mapper meta.RESTMapper
}
// resetGroupVersionKind is a helper function to restore and preserve GroupVersionKind on an object.
// TODO(vincepri): Remove this function and its calls once controller-runtime dependencies are upgraded to 1.16?
func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersionKind) {
if gvk != schema.EmptyObjectKind.GroupVersionKind() {
if v, ok := obj.(schema.ObjectKind); ok {
@@ -121,8 +168,18 @@ func (c *client) resetGroupVersionKind(obj runtime.Object, gvk schema.GroupVersi
}
}
// Create implements client.Client
func (c *client) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
// Scheme returns the scheme this client is using.
func (c *client) Scheme() *runtime.Scheme {
return c.scheme
}
// RESTMapper returns the scheme this client is using.
func (c *client) RESTMapper() meta.RESTMapper {
return c.mapper
}
// Create implements client.Client.
func (c *client) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Create(ctx, obj, opts...)
@@ -133,8 +190,8 @@ func (c *client) Create(ctx context.Context, obj runtime.Object, opts ...CreateO
}
}
// Update implements client.Client
func (c *client) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.Client.
func (c *client) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
@@ -146,8 +203,8 @@ func (c *client) Update(ctx context.Context, obj runtime.Object, opts ...UpdateO
}
}
// Delete implements client.Client
func (c *client) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
// Delete implements client.Client.
func (c *client) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Delete(ctx, obj, opts...)
@@ -158,8 +215,8 @@ func (c *client) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteO
}
}
// DeleteAllOf implements client.Client
func (c *client) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
// DeleteAllOf implements client.Client.
func (c *client) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.DeleteAllOf(ctx, obj, opts...)
@@ -170,8 +227,8 @@ func (c *client) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...De
}
}
// Patch implements client.Client
func (c *client) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (c *client) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
@@ -183,45 +240,69 @@ func (c *client) Patch(ctx context.Context, obj runtime.Object, patch Patch, opt
}
}
// Get implements client.Client
func (c *client) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
// Get implements client.Client.
func (c *client) Get(ctx context.Context, key ObjectKey, obj Object) error {
switch obj.(type) {
case *unstructured.Unstructured:
return c.unstructuredClient.Get(ctx, key, obj)
case *metav1.PartialObjectMetadata:
// Metadata only object should always preserve the GVK coming in from the caller.
defer c.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
return c.metadataClient.Get(ctx, key, obj)
default:
return c.typedClient.Get(ctx, key, obj)
}
}
// List implements client.Client
func (c *client) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
switch obj.(type) {
case *unstructured.Unstructured:
// List implements client.Client.
func (c *client) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
switch x := obj.(type) {
case *unstructured.UnstructuredList:
return c.unstructuredClient.List(ctx, obj, opts...)
case *metav1.PartialObjectMetadataList:
return c.metadataClient.List(ctx, obj, opts...)
// Metadata only object should always preserve the GVK.
gvk := obj.GetObjectKind().GroupVersionKind()
defer c.resetGroupVersionKind(obj, gvk)
// Call the list client.
if err := c.metadataClient.List(ctx, obj, opts...); err != nil {
return err
}
// Restore the GVK for each item in the list.
itemGVK := schema.GroupVersionKind{
Group: gvk.Group,
Version: gvk.Version,
// TODO: this is producing unsafe guesses that don't actually work,
// but it matches ~99% of the cases out there.
Kind: strings.TrimSuffix(gvk.Kind, "List"),
}
for i := range x.Items {
item := &x.Items[i]
item.SetGroupVersionKind(itemGVK)
}
return nil
default:
return c.typedClient.List(ctx, obj, opts...)
}
}
// Status implements client.StatusClient
// Status implements client.StatusClient.
func (c *client) Status() StatusWriter {
return &statusWriter{client: c}
}
// statusWriter is client.StatusWriter that writes status subresource
// statusWriter is client.StatusWriter that writes status subresource.
type statusWriter struct {
client *client
}
// ensure statusWriter implements client.StatusWriter
// ensure statusWriter implements client.StatusWriter.
var _ StatusWriter = &statusWriter{}
// Update implements client.StatusWriter
func (sw *statusWriter) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.StatusWriter.
func (sw *statusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:
@@ -233,8 +314,8 @@ func (sw *statusWriter) Update(ctx context.Context, obj runtime.Object, opts ...
}
}
// Patch implements client.Client
func (sw *statusWriter) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (sw *statusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
defer sw.client.resetGroupVersionKind(obj, obj.GetObjectKind().GroupVersionKind())
switch obj.(type) {
case *unstructured.Unstructured:

View File

@@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -29,7 +30,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// clientCache creates and caches rest clients and metadata for Kubernetes types
// clientCache creates and caches rest clients and metadata for Kubernetes types.
type clientCache struct {
// config is the rest.Config to talk to an apiserver
config *rest.Config
@@ -43,20 +44,22 @@ type clientCache struct {
// codecs are used to create a REST client for a gvk
codecs serializer.CodecFactory
// resourceByType caches type metadata
resourceByType map[schema.GroupVersionKind]*resourceMeta
mu sync.RWMutex
// structuredResourceByType caches structured type metadata
structuredResourceByType map[schema.GroupVersionKind]*resourceMeta
// unstructuredResourceByType caches unstructured type metadata
unstructuredResourceByType map[schema.GroupVersionKind]*resourceMeta
mu sync.RWMutex
}
// newResource maps obj to a Kubernetes Resource and constructs a client for that Resource.
// If the object is a list, the resource represents the item's type instead.
func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList bool) (*resourceMeta, error) {
func (c *clientCache) newResource(gvk schema.GroupVersionKind, isList, isUnstructured bool) (*resourceMeta, error) {
if strings.HasSuffix(gvk.Kind, "List") && isList {
// if this was a list, treat it as a request for the item's resource
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
client, err := apiutil.RESTClientForGVK(gvk, c.config, c.codecs)
client, err := apiutil.RESTClientForGVK(gvk, isUnstructured, c.config, c.codecs)
if err != nil {
return nil, err
}
@@ -75,10 +78,18 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
return nil, err
}
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
isUnstructured = isUnstructured || isUnstructuredList
// It's better to do creation work twice than to not let multiple
// people make requests at once
c.mu.RLock()
r, known := c.resourceByType[gvk]
resourceByType := c.structuredResourceByType
if isUnstructured {
resourceByType = c.unstructuredResourceByType
}
r, known := resourceByType[gvk]
c.mu.RUnlock()
if known {
@@ -88,15 +99,15 @@ func (c *clientCache) getResource(obj runtime.Object) (*resourceMeta, error) {
// Initialize a new Client
c.mu.Lock()
defer c.mu.Unlock()
r, err = c.newResource(gvk, meta.IsListType(obj))
r, err = c.newResource(gvk, meta.IsListType(obj), isUnstructured)
if err != nil {
return nil, err
}
c.resourceByType[gvk] = r
resourceByType[gvk] = r
return r, err
}
// getObjMeta returns objMeta containing both type and object metadata and state
// getObjMeta returns objMeta containing both type and object metadata and state.
func (c *clientCache) getObjMeta(obj runtime.Object) (*objMeta, error) {
r, err := c.getResource(obj)
if err != nil {
@@ -119,18 +130,17 @@ type resourceMeta struct {
mapping *meta.RESTMapping
}
// isNamespaced returns true if the type is namespaced
// isNamespaced returns true if the type is namespaced.
func (r *resourceMeta) isNamespaced() bool {
return r.mapping.Scope.Name() != meta.RESTScopeNameRoot
}
// resource returns the resource name of the type
// resource returns the resource name of the type.
func (r *resourceMeta) resource() string {
return r.mapping.Resource.Resource
}
// objMeta stores type and object information about a Kubernetes type
// objMeta stores type and object information about a Kubernetes type.
type objMeta struct {
// resourceMeta contains type information for the object
*resourceMeta

View File

@@ -1,3 +1,19 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
@@ -12,7 +28,7 @@ import (
var _ runtime.ParameterCodec = noConversionParamCodec{}
// noConversionParamCodec is a no-conversion codec for serializing parameters into URL query strings.
// it's useful in scenarios with the unstructured client and arbitrary resouces.
// it's useful in scenarios with the unstructured client and arbitrary resources.
type noConversionParamCodec struct{}
func (noConversionParamCodec) EncodeParameters(obj runtime.Object, to schema.GroupVersion) (url.Values, error) {

View File

@@ -30,19 +30,14 @@ import (
)
var (
kubeconfig, apiServerURL string
log = logf.RuntimeLog.WithName("client").WithName("config")
kubeconfig string
log = logf.RuntimeLog.WithName("client").WithName("config")
)
func init() {
// TODO: Fix this to allow double vendoring this library but still register flags on behalf of users
flag.StringVar(&kubeconfig, "kubeconfig", "",
"Paths to a kubeconfig. Only required if out-of-cluster.")
// This flag is deprecated, it'll be removed in a future iteration, please switch to --kubeconfig.
flag.StringVar(&apiServerURL, "master", "",
"(Deprecated: switch to `--kubeconfig`) The address of the Kubernetes API server. Overrides any value in kubeconfig. "+
"Only required if out-of-cluster.")
}
// GetConfig creates a *rest.Config for talking to a Kubernetes API server.
@@ -60,7 +55,7 @@ func init() {
//
// * In-cluster config if running in cluster
//
// * $HOME/.kube/config if exists
// * $HOME/.kube/config if exists.
func GetConfig() (*rest.Config, error) {
return GetConfigWithContext("")
}
@@ -80,7 +75,7 @@ func GetConfig() (*rest.Config, error) {
//
// * In-cluster config if running in cluster
//
// * $HOME/.kube/config if exists
// * $HOME/.kube/config if exists.
func GetConfigWithContext(context string) (*rest.Config, error) {
cfg, err := loadConfig(context)
if err != nil {
@@ -100,12 +95,11 @@ func GetConfigWithContext(context string) (*rest.Config, error) {
// test the precedence of loading the config.
var loadInClusterConfig = rest.InClusterConfig
// loadConfig loads a REST Config as per the rules specified in GetConfig
// loadConfig loads a REST Config as per the rules specified in GetConfig.
func loadConfig(context string) (*rest.Config, error) {
// If a flag is specified with the config location, use that
if len(kubeconfig) > 0 {
return loadConfigWithContext(apiServerURL, &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, context)
return loadConfigWithContext("", &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, context)
}
// If the recommended kubeconfig env variable is not specified,
@@ -134,7 +128,7 @@ func loadConfig(context string) (*rest.Config, error) {
loadingRules.Precedence = append(loadingRules.Precedence, path.Join(u.HomeDir, clientcmd.RecommendedHomeDir, clientcmd.RecommendedFileName))
}
return loadConfigWithContext(apiServerURL, loadingRules, context)
return loadConfigWithContext("", loadingRules, context)
}
func loadConfigWithContext(apiServerURL string, loader clientcmd.ClientConfigLoader, context string) (*rest.Config, error) {

View File

@@ -19,6 +19,7 @@ package client
import (
"context"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
)
@@ -35,47 +36,57 @@ type dryRunClient struct {
client Client
}
// Create implements client.Client
func (c *dryRunClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
// Scheme returns the scheme this client is using.
func (c *dryRunClient) Scheme() *runtime.Scheme {
return c.client.Scheme()
}
// RESTMapper returns the rest mapper this client is using.
func (c *dryRunClient) RESTMapper() meta.RESTMapper {
return c.client.RESTMapper()
}
// Create implements client.Client.
func (c *dryRunClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
return c.client.Create(ctx, obj, append(opts, DryRunAll)...)
}
// Update implements client.Client
func (c *dryRunClient) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.Client.
func (c *dryRunClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
return c.client.Update(ctx, obj, append(opts, DryRunAll)...)
}
// Delete implements client.Client
func (c *dryRunClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
// Delete implements client.Client.
func (c *dryRunClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
return c.client.Delete(ctx, obj, append(opts, DryRunAll)...)
}
// DeleteAllOf implements client.Client
func (c *dryRunClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
// DeleteAllOf implements client.Client.
func (c *dryRunClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
return c.client.DeleteAllOf(ctx, obj, append(opts, DryRunAll)...)
}
// Patch implements client.Client
func (c *dryRunClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (c *dryRunClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
return c.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...)
}
// Get implements client.Client
func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
// Get implements client.Client.
func (c *dryRunClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
return c.client.Get(ctx, key, obj)
}
// List implements client.Client
func (c *dryRunClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
// List implements client.Client.
func (c *dryRunClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
return c.client.List(ctx, obj, opts...)
}
// Status implements client.StatusClient
// Status implements client.StatusClient.
func (c *dryRunClient) Status() StatusWriter {
return &dryRunStatusWriter{client: c.client.Status()}
}
// ensure dryRunStatusWriter implements client.StatusWriter
// ensure dryRunStatusWriter implements client.StatusWriter.
var _ StatusWriter = &dryRunStatusWriter{}
// dryRunStatusWriter is client.StatusWriter that writes status subresource with dryRun mode
@@ -84,12 +95,12 @@ type dryRunStatusWriter struct {
client StatusWriter
}
// Update implements client.StatusWriter
func (sw *dryRunStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.StatusWriter.
func (sw *dryRunStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
return sw.client.Update(ctx, obj, append(opts, DryRunAll)...)
}
// Patch implements client.StatusWriter
func (sw *dryRunStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.StatusWriter.
func (sw *dryRunStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
return sw.client.Patch(ctx, obj, patch, append(opts, DryRunAll)...)
}

View File

@@ -23,14 +23,17 @@ import (
"fmt"
"strconv"
"strings"
"sync"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
utilrand "k8s.io/apimachinery/pkg/util/rand"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/testing"
@@ -41,14 +44,16 @@ import (
type versionedTracker struct {
testing.ObjectTracker
scheme *runtime.Scheme
}
type fakeClient struct {
tracker versionedTracker
scheme *runtime.Scheme
tracker versionedTracker
scheme *runtime.Scheme
schemeWriteLock sync.Mutex
}
var _ client.Client = &fakeClient{}
var _ client.WithWatch = &fakeClient{}
const (
maxNameLength = 63
@@ -58,33 +63,124 @@ const (
// NewFakeClient creates a new fake client for testing.
// You can choose to initialize it with a slice of runtime.Object.
// Deprecated: use NewFakeClientWithScheme. You should always be
// passing an explicit Scheme.
func NewFakeClient(initObjs ...runtime.Object) client.Client {
return NewFakeClientWithScheme(scheme.Scheme, initObjs...)
//
// Deprecated: Please use NewClientBuilder instead.
func NewFakeClient(initObjs ...runtime.Object) client.WithWatch {
return NewClientBuilder().WithRuntimeObjects(initObjs...).Build()
}
// NewFakeClientWithScheme creates a new fake client with the given scheme
// for testing.
// You can choose to initialize it with a slice of runtime.Object.
func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.Client {
tracker := testing.NewObjectTracker(clientScheme, scheme.Codecs.UniversalDecoder())
for _, obj := range initObjs {
err := tracker.Add(obj)
if err != nil {
//
// Deprecated: Please use NewClientBuilder instead.
func NewFakeClientWithScheme(clientScheme *runtime.Scheme, initObjs ...runtime.Object) client.WithWatch {
return NewClientBuilder().WithScheme(clientScheme).WithRuntimeObjects(initObjs...).Build()
}
// NewClientBuilder returns a new builder to create a fake client.
func NewClientBuilder() *ClientBuilder {
return &ClientBuilder{}
}
// ClientBuilder builds a fake client.
type ClientBuilder struct {
scheme *runtime.Scheme
initObject []client.Object
initLists []client.ObjectList
initRuntimeObjects []runtime.Object
}
// WithScheme sets this builder's internal scheme.
// If not set, defaults to client-go's global scheme.Scheme.
func (f *ClientBuilder) WithScheme(scheme *runtime.Scheme) *ClientBuilder {
f.scheme = scheme
return f
}
// WithObjects can be optionally used to initialize this fake client with client.Object(s).
func (f *ClientBuilder) WithObjects(initObjs ...client.Object) *ClientBuilder {
f.initObject = append(f.initObject, initObjs...)
return f
}
// WithLists can be optionally used to initialize this fake client with client.ObjectList(s).
func (f *ClientBuilder) WithLists(initLists ...client.ObjectList) *ClientBuilder {
f.initLists = append(f.initLists, initLists...)
return f
}
// WithRuntimeObjects can be optionally used to initialize this fake client with runtime.Object(s).
func (f *ClientBuilder) WithRuntimeObjects(initRuntimeObjs ...runtime.Object) *ClientBuilder {
f.initRuntimeObjects = append(f.initRuntimeObjects, initRuntimeObjs...)
return f
}
// Build builds and returns a new fake client.
func (f *ClientBuilder) Build() client.WithWatch {
if f.scheme == nil {
f.scheme = scheme.Scheme
}
tracker := versionedTracker{ObjectTracker: testing.NewObjectTracker(f.scheme, scheme.Codecs.UniversalDecoder()), scheme: f.scheme}
for _, obj := range f.initObject {
if err := tracker.Add(obj); err != nil {
panic(fmt.Errorf("failed to add object %v to fake client: %w", obj, err))
}
}
return &fakeClient{
tracker: versionedTracker{tracker},
scheme: clientScheme,
for _, obj := range f.initLists {
if err := tracker.Add(obj); err != nil {
panic(fmt.Errorf("failed to add list %v to fake client: %w", obj, err))
}
}
for _, obj := range f.initRuntimeObjects {
if err := tracker.Add(obj); err != nil {
panic(fmt.Errorf("failed to add runtime object %v to fake client: %w", obj, err))
}
}
return &fakeClient{
tracker: tracker,
scheme: f.scheme,
}
}
const trackerAddResourceVersion = "999"
func (t versionedTracker) Add(obj runtime.Object) error {
var objects []runtime.Object
if meta.IsListType(obj) {
var err error
objects, err = meta.ExtractList(obj)
if err != nil {
return err
}
} else {
objects = []runtime.Object{obj}
}
for _, obj := range objects {
accessor, err := meta.Accessor(obj)
if err != nil {
return fmt.Errorf("failed to get accessor for object: %w", err)
}
if accessor.GetResourceVersion() == "" {
// We use a "magic" value of 999 here because this field
// is parsed as uint and and 0 is already used in Update.
// As we can't go lower, go very high instead so this can
// be recognized
accessor.SetResourceVersion(trackerAddResourceVersion)
}
if err := t.ObjectTracker.Add(obj); err != nil {
return err
}
}
return nil
}
func (t versionedTracker) Create(gvr schema.GroupVersionResource, obj runtime.Object, ns string) error {
accessor, err := meta.Accessor(obj)
if err != nil {
return err
return fmt.Errorf("failed to get accessor for object: %v", err)
}
if accessor.GetName() == "" {
return apierrors.NewInvalid(
@@ -108,20 +204,42 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob
if err != nil {
return fmt.Errorf("failed to get accessor for object: %v", err)
}
if accessor.GetName() == "" {
return apierrors.NewInvalid(
obj.GetObjectKind().GroupVersionKind().GroupKind(),
accessor.GetName(),
field.ErrorList{field.Required(field.NewPath("metadata.name"), "name is required")})
}
gvk := obj.GetObjectKind().GroupVersionKind()
if gvk.Empty() {
gvk, err = apiutil.GVKForObject(obj, t.scheme)
if err != nil {
return err
}
}
oldObject, err := t.ObjectTracker.Get(gvr, ns, accessor.GetName())
if err != nil {
// If the resource is not found and the resource allows create on update, issue a
// create instead.
if apierrors.IsNotFound(err) && allowsCreateOnUpdate(gvk) {
return t.Create(gvr, obj, ns)
}
return err
}
oldAccessor, err := meta.Accessor(oldObject)
if err != nil {
return err
}
// If the new object does not have the resource version set and it allows unconditional update,
// default it to the resource version of the existing resource
if accessor.GetResourceVersion() == "" && allowsUnconditionalUpdate(gvk) {
accessor.SetResourceVersion(oldAccessor.GetResourceVersion())
}
if accessor.GetResourceVersion() != oldAccessor.GetResourceVersion() {
return apierrors.NewConflict(gvr.GroupResource(), accessor.GetName(), errors.New("object was modified"))
}
@@ -134,10 +252,13 @@ func (t versionedTracker) Update(gvr schema.GroupVersionResource, obj runtime.Ob
}
intResourceVersion++
accessor.SetResourceVersion(strconv.FormatUint(intResourceVersion, 10))
if !accessor.GetDeletionTimestamp().IsZero() && len(accessor.GetFinalizers()) == 0 {
return t.ObjectTracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
}
return t.ObjectTracker.Update(gvr, obj, ns)
}
func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.Object) error {
func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object) error {
gvr, err := getGVRFromObject(obj, c.scheme)
if err != nil {
return err
@@ -167,19 +288,42 @@ func (c *fakeClient) Get(ctx context.Context, key client.ObjectKey, obj runtime.
return err
}
func (c *fakeClient) List(ctx context.Context, obj runtime.Object, opts ...client.ListOption) error {
func (c *fakeClient) Watch(ctx context.Context, list client.ObjectList, opts ...client.ListOption) (watch.Interface, error) {
gvk, err := apiutil.GVKForObject(list, c.scheme)
if err != nil {
return nil, err
}
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
gvr, _ := meta.UnsafeGuessKindToResource(gvk)
return c.tracker.Watch(gvr, listOpts.Namespace)
}
func (c *fakeClient) List(ctx context.Context, obj client.ObjectList, opts ...client.ListOption) error {
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return err
}
OriginalKind := gvk.Kind
originalKind := gvk.Kind
if !strings.HasSuffix(gvk.Kind, "List") {
return fmt.Errorf("non-list type %T (kind %q) passed as output", obj, gvk)
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
if _, isUnstructuredList := obj.(*unstructured.UnstructuredList); isUnstructuredList && !c.scheme.Recognizes(gvk) {
// We need tor register the ListKind with UnstructuredList:
// https://github.com/kubernetes/kubernetes/blob/7b2776b89fb1be28d4e9203bdeec079be903c103/staging/src/k8s.io/client-go/dynamic/fake/simple.go#L44-L51
c.schemeWriteLock.Lock()
c.scheme.AddKnownTypeWithName(gvk.GroupVersion().WithKind(gvk.Kind+"List"), &unstructured.UnstructuredList{})
c.schemeWriteLock.Unlock()
}
// we need the non-list GVK, so chop off the "List" from the end of the kind
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
listOpts := client.ListOptions{}
listOpts.ApplyOptions(opts)
@@ -194,7 +338,7 @@ func (c *fakeClient) List(ctx context.Context, obj runtime.Object, opts ...clien
if err != nil {
return err
}
ta.SetKind(OriginalKind)
ta.SetKind(originalKind)
ta.SetAPIVersion(gvk.GroupVersion().String())
j, err := json.Marshal(o)
@@ -224,7 +368,16 @@ func (c *fakeClient) List(ctx context.Context, obj runtime.Object, opts ...clien
return nil
}
func (c *fakeClient) Create(ctx context.Context, obj runtime.Object, opts ...client.CreateOption) error {
func (c *fakeClient) Scheme() *runtime.Scheme {
return c.scheme
}
func (c *fakeClient) RESTMapper() meta.RESTMapper {
// TODO: Implement a fake RESTMapper.
return nil
}
func (c *fakeClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
createOptions := &client.CreateOptions{}
createOptions.ApplyOptions(opts)
@@ -254,7 +407,7 @@ func (c *fakeClient) Create(ctx context.Context, obj runtime.Object, opts ...cli
return c.tracker.Create(gvr, obj, accessor.GetNamespace())
}
func (c *fakeClient) Delete(ctx context.Context, obj runtime.Object, opts ...client.DeleteOption) error {
func (c *fakeClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
gvr, err := getGVRFromObject(obj, c.scheme)
if err != nil {
return err
@@ -266,11 +419,10 @@ func (c *fakeClient) Delete(ctx context.Context, obj runtime.Object, opts ...cli
delOptions := client.DeleteOptions{}
delOptions.ApplyOptions(opts)
//TODO: implement propagation
return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
return c.deleteObject(gvr, accessor)
}
func (c *fakeClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...client.DeleteAllOfOption) error {
func (c *fakeClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
gvk, err := apiutil.GVKForObject(obj, c.scheme)
if err != nil {
return err
@@ -298,7 +450,7 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts .
if err != nil {
return err
}
err = c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
err = c.deleteObject(gvr, accessor)
if err != nil {
return err
}
@@ -306,7 +458,7 @@ func (c *fakeClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts .
return nil
}
func (c *fakeClient) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
func (c *fakeClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
updateOptions := &client.UpdateOptions{}
updateOptions.ApplyOptions(opts)
@@ -327,7 +479,7 @@ func (c *fakeClient) Update(ctx context.Context, obj runtime.Object, opts ...cli
return c.tracker.Update(gvr, obj, accessor.GetNamespace())
}
func (c *fakeClient) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error {
func (c *fakeClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
patchOptions := &client.PatchOptions{}
patchOptions.ApplyOptions(opts)
@@ -383,6 +535,23 @@ func (c *fakeClient) Status() client.StatusWriter {
return &fakeStatusWriter{client: c}
}
func (c *fakeClient) deleteObject(gvr schema.GroupVersionResource, accessor metav1.Object) error {
old, err := c.tracker.Get(gvr, accessor.GetNamespace(), accessor.GetName())
if err == nil {
oldAccessor, err := meta.Accessor(old)
if err == nil {
if len(oldAccessor.GetFinalizers()) > 0 {
now := metav1.Now()
oldAccessor.SetDeletionTimestamp(&now)
return c.tracker.Update(gvr, old, accessor.GetNamespace())
}
}
}
//TODO: implement propagation
return c.tracker.Delete(gvr, accessor.GetNamespace(), accessor.GetName())
}
func getGVRFromObject(obj runtime.Object, scheme *runtime.Scheme) (schema.GroupVersionResource, error) {
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
@@ -396,14 +565,111 @@ type fakeStatusWriter struct {
client *fakeClient
}
func (sw *fakeStatusWriter) Update(ctx context.Context, obj runtime.Object, opts ...client.UpdateOption) error {
func (sw *fakeStatusWriter) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
// TODO(droot): This results in full update of the obj (spec + status). Need
// a way to update status field only.
return sw.client.Update(ctx, obj, opts...)
}
func (sw *fakeStatusWriter) Patch(ctx context.Context, obj runtime.Object, patch client.Patch, opts ...client.PatchOption) error {
func (sw *fakeStatusWriter) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
// TODO(droot): This results in full update of the obj (spec + status). Need
// a way to update status field only.
return sw.client.Patch(ctx, obj, patch, opts...)
}
func allowsUnconditionalUpdate(gvk schema.GroupVersionKind) bool {
switch gvk.Group {
case "apps":
switch gvk.Kind {
case "ControllerRevision", "DaemonSet", "Deployment", "ReplicaSet", "StatefulSet":
return true
}
case "autoscaling":
switch gvk.Kind {
case "HorizontalPodAutoscaler":
return true
}
case "batch":
switch gvk.Kind {
case "CronJob", "Job":
return true
}
case "certificates":
switch gvk.Kind {
case "Certificates":
return true
}
case "flowcontrol":
switch gvk.Kind {
case "FlowSchema", "PriorityLevelConfiguration":
return true
}
case "networking":
switch gvk.Kind {
case "Ingress", "IngressClass", "NetworkPolicy":
return true
}
case "policy":
switch gvk.Kind {
case "PodSecurityPolicy":
return true
}
case "rbac":
switch gvk.Kind {
case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding":
return true
}
case "scheduling":
switch gvk.Kind {
case "PriorityClass":
return true
}
case "settings":
switch gvk.Kind {
case "PodPreset":
return true
}
case "storage":
switch gvk.Kind {
case "StorageClass":
return true
}
case "":
switch gvk.Kind {
case "ConfigMap", "Endpoint", "Event", "LimitRange", "Namespace", "Node",
"PersistentVolume", "PersistentVolumeClaim", "Pod", "PodTemplate",
"ReplicationController", "ResourceQuota", "Secret", "Service",
"ServiceAccount", "EndpointSlice":
return true
}
}
return false
}
func allowsCreateOnUpdate(gvk schema.GroupVersionKind) bool {
switch gvk.Group {
case "coordination":
switch gvk.Kind {
case "Lease":
return true
}
case "node":
switch gvk.Kind {
case "RuntimeClass":
return true
}
case "rbac":
switch gvk.Kind {
case "ClusterRole", "ClusterRoleBinding", "Role", "RoleBinding":
return true
}
case "":
switch gvk.Kind {
case "Endpoint", "Event", "LimitRange", "Service":
return true
}
}
return false
}

View File

@@ -17,17 +17,23 @@ limitations under the License.
/*
Package fake provides a fake client for testing.
Deprecated: please use pkg/envtest for testing. This package will be dropped
before the v1.0.0 release.
An fake client is backed by its simple object store indexed by GroupVersionResource.
A fake client is backed by its simple object store indexed by GroupVersionResource.
You can create a fake client with optional objects.
client := NewFakeClient(initObjs...) // initObjs is a slice of runtime.Object
client := NewFakeClientWithScheme(scheme, initObjs...) // initObjs is a slice of runtime.Object
You can invoke the methods defined in the Client interface.
When it doubt, it's almost always better not to use this package and instead use
When in doubt, it's almost always better not to use this package and instead use
envtest.Environment with a real client and API server.
WARNING: ⚠️ Current Limitations / Known Issues with the fake Client ⚠️
- This client does not have a way to inject specific errors to test handled vs. unhandled errors.
- There is some support for sub resources which can cause issues with tests if you're trying to update
e.g. metadata and status in the same reconcile.
- No OpeanAPI validation is performed when creating or updating objects.
- ObjectMeta's `Generation` and `ResourceVersion` don't behave properly, Patch or Update
operations that rely on these fields will fail, or give false positives.
*/
package fake

View File

@@ -24,18 +24,15 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/watch"
)
// ObjectKey identifies a Kubernetes Object.
type ObjectKey = types.NamespacedName
// ObjectKeyFromObject returns the ObjectKey given a runtime.Object
func ObjectKeyFromObject(obj runtime.Object) (ObjectKey, error) {
accessor, err := meta.Accessor(obj)
if err != nil {
return ObjectKey{}, err
}
return ObjectKey{Namespace: accessor.GetNamespace(), Name: accessor.GetName()}, nil
// ObjectKeyFromObject returns the ObjectKey given a runtime.Object.
func ObjectKeyFromObject(obj Object) ObjectKey {
return ObjectKey{Namespace: obj.GetNamespace(), Name: obj.GetName()}
}
// Patch is a patch that can be applied to a Kubernetes object.
@@ -43,7 +40,7 @@ type Patch interface {
// Type is the PatchType of the patch.
Type() types.PatchType
// Data is the raw data representing the patch.
Data(obj runtime.Object) ([]byte, error)
Data(obj Object) ([]byte, error)
}
// TODO(directxman12): is there a sane way to deal with get/delete options?
@@ -53,32 +50,32 @@ type Reader interface {
// Get retrieves an obj for the given object key from the Kubernetes Cluster.
// obj must be a struct pointer so that obj can be updated with the response
// returned by the Server.
Get(ctx context.Context, key ObjectKey, obj runtime.Object) error
Get(ctx context.Context, key ObjectKey, obj Object) error
// List retrieves list of objects for a given namespace and list options. On a
// successful call, Items field in the list will be populated with the
// result returned from the server.
List(ctx context.Context, list runtime.Object, opts ...ListOption) error
List(ctx context.Context, list ObjectList, opts ...ListOption) error
}
// Writer knows how to create, delete, and update Kubernetes objects.
type Writer interface {
// Create saves the object obj in the Kubernetes cluster.
Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error
Create(ctx context.Context, obj Object, opts ...CreateOption) error
// Delete deletes the given obj from Kubernetes cluster.
Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error
Delete(ctx context.Context, obj Object, opts ...DeleteOption) error
// Update updates the given obj in the Kubernetes cluster. obj must be a
// struct pointer so that obj can be updated with the content returned by the Server.
Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error
Update(ctx context.Context, obj Object, opts ...UpdateOption) error
// Patch patches the given obj in the Kubernetes cluster. obj must be a
// struct pointer so that obj can be updated with the content returned by the Server.
Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error
Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error
// DeleteAllOf deletes all objects of the given type matching the given options.
DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error
DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error
}
// StatusClient knows how to create a client which can update status subresource
@@ -92,12 +89,12 @@ type StatusWriter interface {
// Update updates the fields corresponding to the status subresource for the
// given obj. obj must be a struct pointer so that obj can be updated
// with the content returned by the Server.
Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error
Update(ctx context.Context, obj Object, opts ...UpdateOption) error
// Patch patches the given object's subresource. obj must be a struct
// pointer so that obj can be updated with the content returned by the
// Server.
Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error
Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error
}
// Client knows how to perform CRUD operations on Kubernetes objects.
@@ -105,12 +102,25 @@ type Client interface {
Reader
Writer
StatusClient
// Scheme returns the scheme this client is using.
Scheme() *runtime.Scheme
// RESTMapper returns the rest this client is using.
RESTMapper() meta.RESTMapper
}
// WithWatch supports Watch on top of the CRUD operations supported by
// the normal Client. Its intended use-case are CLI apps that need to wait for
// events.
type WithWatch interface {
Client
Watch(ctx context.Context, obj ObjectList, opts ...ListOption) (watch.Interface, error)
}
// IndexerFunc knows how to take an object and turn it into a series
// of non-namespaced keys. Namespaced objects are automatically given
// namespaced and non-spaced variants, so keys do not need to include namespace.
type IndexerFunc func(runtime.Object) []string
type IndexerFunc func(Object) []string
// FieldIndexer knows how to index over a particular "field" such that it
// can later be used by a field selector.
@@ -122,7 +132,7 @@ type FieldIndexer interface {
// and "equality" in the field selector means that at least one key matches the value.
// The FieldIndexer will automatically take care of indexing over namespace
// and supporting efficient all-namespace queries.
IndexField(ctx context.Context, obj runtime.Object, field string, extractValue IndexerFunc) error
IndexField(ctx context.Context, obj Object, field string, extractValue IndexerFunc) error
}
// IgnoreNotFound returns nil on NotFound errors.

View File

@@ -1,17 +1,17 @@
/*
Copyright 2020 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
@@ -23,7 +23,6 @@ import (
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/client-go/metadata"
)
@@ -50,8 +49,8 @@ func (mc *metadataClient) getResourceInterface(gvk schema.GroupVersionKind, ns s
return mc.client.Resource(mapping.Resource).Namespace(ns), nil
}
// Delete implements client.Client
func (mc *metadataClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
// Delete implements client.Client.
func (mc *metadataClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
@@ -68,8 +67,8 @@ func (mc *metadataClient) Delete(ctx context.Context, obj runtime.Object, opts .
return resInt.Delete(ctx, metadata.Name, *deleteOpts.AsDeleteOptions())
}
// DeleteAllOf implements client.Client
func (mc *metadataClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
// DeleteAllOf implements client.Client.
func (mc *metadataClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
@@ -86,8 +85,8 @@ func (mc *metadataClient) DeleteAllOf(ctx context.Context, obj runtime.Object, o
return resInt.DeleteCollection(ctx, *deleteAllOfOpts.AsDeleteOptions(), *deleteAllOfOpts.AsListOptions())
}
// Patch implements client.Client
func (mc *metadataClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (mc *metadataClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
@@ -105,6 +104,8 @@ func (mc *metadataClient) Patch(ctx context.Context, obj runtime.Object, patch P
}
patchOpts := &PatchOptions{}
patchOpts.ApplyOptions(opts)
res, err := resInt.Patch(ctx, metadata.Name, patch.Type(), data, *patchOpts.AsPatchOptions())
if err != nil {
return err
@@ -114,8 +115,8 @@ func (mc *metadataClient) Patch(ctx context.Context, obj runtime.Object, patch P
return nil
}
// Get implements client.Client
func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
// Get implements client.Client.
func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
@@ -137,8 +138,8 @@ func (mc *metadataClient) Get(ctx context.Context, key ObjectKey, obj runtime.Ob
return nil
}
// List implements client.Client
func (mc *metadataClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
// List implements client.Client.
func (mc *metadataClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadataList)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)
@@ -166,7 +167,7 @@ func (mc *metadataClient) List(ctx context.Context, obj runtime.Object, opts ...
return nil
}
func (mc *metadataClient) PatchStatus(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
func (mc *metadataClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
metadata, ok := obj.(*metav1.PartialObjectMetadata)
if !ok {
return fmt.Errorf("metadata client did not understand object: %T", obj)

View File

@@ -0,0 +1,254 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"errors"
"fmt"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// NewNamespacedClient wraps an existing client enforcing the namespace value.
// All functions using this client will have the same namespace declared here.
func NewNamespacedClient(c Client, ns string) Client {
return &namespacedClient{
client: c,
namespace: ns,
}
}
var _ Client = &namespacedClient{}
// namespacedClient is a Client that wraps another Client in order to enforce the specified namespace value.
type namespacedClient struct {
namespace string
client Client
}
// Scheme returns the scheme this client is using.
func (n *namespacedClient) Scheme() *runtime.Scheme {
return n.client.Scheme()
}
// RESTMapper returns the scheme this client is using.
func (n *namespacedClient) RESTMapper() meta.RESTMapper {
return n.client.RESTMapper()
}
// isNamespaced returns true if the object is namespace scoped.
// For unstructured objects the gvk is found from the object itself.
// TODO: this is repetitive code. Remove this and use ojectutil.IsNamespaced.
func isNamespaced(c Client, obj runtime.Object) (bool, error) {
var gvk schema.GroupVersionKind
var err error
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
isUnstructured = isUnstructured || isUnstructuredList
if isUnstructured {
gvk = obj.GetObjectKind().GroupVersionKind()
} else {
gvk, err = apiutil.GVKForObject(obj, c.Scheme())
if err != nil {
return false, err
}
}
gk := schema.GroupKind{
Group: gvk.Group,
Kind: gvk.Kind,
}
restmapping, err := c.RESTMapper().RESTMapping(gk)
if err != nil {
return false, fmt.Errorf("failed to get restmapping: %w", err)
}
scope := restmapping.Scope.Name()
if scope == "" {
return false, errors.New("scope cannot be identified, empty scope returned")
}
if scope != meta.RESTScopeNameRoot {
return true, nil
}
return false, nil
}
// Create implements clinet.Client.
func (n *namespacedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != n.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), n.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(n.namespace)
}
return n.client.Create(ctx, obj, opts...)
}
// Update implements client.Client.
func (n *namespacedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != n.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), n.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(n.namespace)
}
return n.client.Update(ctx, obj, opts...)
}
// Delete implements client.Client.
func (n *namespacedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != n.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), n.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(n.namespace)
}
return n.client.Delete(ctx, obj, opts...)
}
// DeleteAllOf implements client.Client.
func (n *namespacedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
if isNamespaceScoped {
opts = append(opts, InNamespace(n.namespace))
}
return n.client.DeleteAllOf(ctx, obj, opts...)
}
// Patch implements client.Client.
func (n *namespacedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != n.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), n.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(n.namespace)
}
return n.client.Patch(ctx, obj, patch, opts...)
}
// Get implements client.Client.
func (n *namespacedClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
isNamespaceScoped, err := isNamespaced(n.client, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
if isNamespaceScoped {
if key.Namespace != "" && key.Namespace != n.namespace {
return fmt.Errorf("namespace %s provided for the object %s does not match the namesapce %s on the client", key.Namespace, obj.GetName(), n.namespace)
}
key.Namespace = n.namespace
}
return n.client.Get(ctx, key, obj)
}
// List implements client.Client.
func (n *namespacedClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
if n.namespace != "" {
opts = append(opts, InNamespace(n.namespace))
}
return n.client.List(ctx, obj, opts...)
}
// Status implements client.StatusClient.
func (n *namespacedClient) Status() StatusWriter {
return &namespacedClientStatusWriter{StatusClient: n.client.Status(), namespace: n.namespace, namespacedclient: n}
}
// ensure namespacedClientStatusWriter implements client.StatusWriter.
var _ StatusWriter = &namespacedClientStatusWriter{}
type namespacedClientStatusWriter struct {
StatusClient StatusWriter
namespace string
namespacedclient Client
}
// Update implements client.StatusWriter.
func (nsw *namespacedClientStatusWriter) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != nsw.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(nsw.namespace)
}
return nsw.StatusClient.Update(ctx, obj, opts...)
}
// Patch implements client.StatusWriter.
func (nsw *namespacedClientStatusWriter) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
isNamespaceScoped, err := isNamespaced(nsw.namespacedclient, obj)
if err != nil {
return fmt.Errorf("error finding the scope of the object: %v", err)
}
objectNamespace := obj.GetNamespace()
if objectNamespace != nsw.namespace && objectNamespace != "" {
return fmt.Errorf("namespace %s of the object %s does not match the namespace %s on the client", objectNamespace, obj.GetName(), nsw.namespace)
}
if isNamespaceScoped && objectNamespace == "" {
obj.SetNamespace(nsw.namespace)
}
return nsw.StatusClient.Patch(ctx, obj, patch, opts...)
}

View File

@@ -0,0 +1,77 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
// Object is a Kubernetes object, allows functions to work indistinctly with
// any resource that implements both Object interfaces.
//
// Semantically, these are objects which are both serializable (runtime.Object)
// and identifiable (metav1.Object) -- think any object which you could write
// as YAML or JSON, and then `kubectl create`.
//
// Code-wise, this means that any object which embeds both ObjectMeta (which
// provides metav1.Object) and TypeMeta (which provides half of runtime.Object)
// and has a `DeepCopyObject` implementation (the other half of runtime.Object)
// will implement this by default.
//
// For example, nearly all the built-in types are Objects, as well as all
// KubeBuilder-generated CRDs (unless you do something real funky to them).
//
// By and large, most things that implement runtime.Object also implement
// Object -- it's very rare to have *just* a runtime.Object implementation (the
// cases tend to be funky built-in types like Webhook payloads that don't have
// a `metadata` field).
//
// Notice that XYZList types are distinct: they implement ObjectList instead.
type Object interface {
metav1.Object
runtime.Object
}
// ObjectList is a Kubernetes object list, allows functions to work
// indistinctly with any resource that implements both runtime.Object and
// metav1.ListInterface interfaces.
//
// Semantically, this is any object which may be serialized (ObjectMeta), and
// is a kubernetes list wrapper (has items, pagination fields, etc) -- think
// the wrapper used in a response from a `kubectl list --output yaml` call.
//
// Code-wise, this means that any object which embedds both ListMeta (which
// provides metav1.ListInterface) and TypeMeta (which provides half of
// runtime.Object) and has a `DeepCopyObject` implementation (the other half of
// runtime.Object) will implement this by default.
//
// For example, nearly all the built-in XYZList types are ObjectLists, as well
// as the XYZList types for all KubeBuilder-generated CRDs (unless you do
// something real funky to them).
//
// By and large, most things that are XYZList and implement runtime.Object also
// implement ObjectList -- it's very rare to have *just* a runtime.Object
// implementation (the cases tend to be funky built-in types like Webhook
// payloads that don't have a `metadata` field).
//
// This is similar to Object, which is almost always implemented by the items
// in the list themselves.
type ObjectList interface {
metav1.ListInterface
runtime.Object
}

View File

@@ -158,7 +158,7 @@ func (o *CreateOptions) ApplyOptions(opts []CreateOption) *CreateOptions {
return o
}
// ApplyToCreate implements CreateOption
// ApplyToCreate implements CreateOption.
func (o *CreateOptions) ApplyToCreate(co *CreateOptions) {
if o.DryRun != nil {
co.DryRun = o.DryRun
@@ -173,11 +173,6 @@ func (o *CreateOptions) ApplyToCreate(co *CreateOptions) {
var _ CreateOption = &CreateOptions{}
// CreateDryRunAll sets the "dry run" option to "all".
//
// Deprecated: Use DryRunAll
var CreateDryRunAll = DryRunAll
// }}}
// {{{ Delete Options
@@ -244,7 +239,7 @@ func (o *DeleteOptions) ApplyOptions(opts []DeleteOption) *DeleteOptions {
var _ DeleteOption = &DeleteOptions{}
// ApplyToDelete implements DeleteOption
// ApplyToDelete implements DeleteOption.
func (o *DeleteOptions) ApplyToDelete(do *DeleteOptions) {
if o.GracePeriodSeconds != nil {
do.GracePeriodSeconds = o.GracePeriodSeconds
@@ -354,7 +349,7 @@ type ListOptions struct {
var _ ListOption = &ListOptions{}
// ApplyToList implements ListOption for ListOptions
// ApplyToList implements ListOption for ListOptions.
func (o *ListOptions) ApplyToList(lo *ListOptions) {
if o.LabelSelector != nil {
lo.LabelSelector = o.LabelSelector
@@ -460,14 +455,6 @@ func (m MatchingLabelsSelector) ApplyToDeleteAllOf(opts *DeleteAllOfOptions) {
m.ApplyToList(&opts.ListOptions)
}
// MatchingField filters the list operation on the given field selector
// (or index in the case of cached lists).
//
// Deprecated: Use MatchingFields
func MatchingField(name, val string) MatchingFields {
return MatchingFields{name: val}
}
// MatchingFields filters the list/delete operation on the given field Set
// (or index in the case of cached lists).
type MatchingFields fields.Set
@@ -582,7 +569,7 @@ func (o *UpdateOptions) ApplyOptions(opts []UpdateOption) *UpdateOptions {
var _ UpdateOption = &UpdateOptions{}
// ApplyToUpdate implements UpdateOption
// ApplyToUpdate implements UpdateOption.
func (o *UpdateOptions) ApplyToUpdate(uo *UpdateOptions) {
if o.DryRun != nil {
uo.DryRun = o.DryRun
@@ -595,11 +582,6 @@ func (o *UpdateOptions) ApplyToUpdate(uo *UpdateOptions) {
}
}
// UpdateDryRunAll sets the "dry run" option to "all".
//
// Deprecated: Use DryRunAll
var UpdateDryRunAll = DryRunAll
// }}}
// {{{ Patch Options
@@ -654,7 +636,7 @@ func (o *PatchOptions) AsPatchOptions() *metav1.PatchOptions {
var _ PatchOption = &PatchOptions{}
// ApplyToPatch implements PatchOptions
// ApplyToPatch implements PatchOptions.
func (o *PatchOptions) ApplyToPatch(po *PatchOptions) {
if o.DryRun != nil {
po.DryRun = o.DryRun
@@ -682,11 +664,6 @@ func (forceOwnership) ApplyToPatch(opts *PatchOptions) {
opts.Force = &definitelyTrue
}
// PatchDryRunAll sets the "dry run" option to "all".
//
// Deprecated: Use DryRunAll
var PatchDryRunAll = DryRunAll
// }}}
// {{{ DeleteAllOf Options
@@ -711,7 +688,7 @@ func (o *DeleteAllOfOptions) ApplyOptions(opts []DeleteAllOfOption) *DeleteAllOf
var _ DeleteAllOfOption = &DeleteAllOfOptions{}
// ApplyToDeleteAllOf implements DeleteAllOfOption
// ApplyToDeleteAllOf implements DeleteAllOfOption.
func (o *DeleteAllOfOptions) ApplyToDeleteAllOf(do *DeleteAllOfOptions) {
o.ApplyToList(&do.ListOptions)
o.ApplyToDelete(&do.DeleteOptions)

View File

@@ -20,20 +20,18 @@ import (
"fmt"
jsonpatch "github.com/evanphx/json-patch"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/json"
"k8s.io/apimachinery/pkg/util/strategicpatch"
)
var (
// Apply uses server-side apply to patch the given object.
Apply = applyPatch{}
Apply Patch = applyPatch{}
// Merge uses the raw object as a merge patch, without modifications.
// Use MergeFrom if you wish to compute a diff instead.
Merge = mergePatch{}
Merge Patch = mergePatch{}
)
type patch struct {
@@ -47,7 +45,7 @@ func (s *patch) Type() types.PatchType {
}
// Data implements Patch.
func (s *patch) Data(obj runtime.Object) ([]byte, error) {
func (s *patch) Data(obj Object) ([]byte, error) {
return s.data, nil
}
@@ -56,13 +54,6 @@ func RawPatch(patchType types.PatchType, data []byte) Patch {
return &patch{patchType, data}
}
// ConstantPatch constructs a new Patch with the given PatchType and data.
//
// Deprecated: use RawPatch instead
func ConstantPatch(patchType types.PatchType, data []byte) Patch {
return RawPatch(patchType, data)
}
// MergeFromWithOptimisticLock can be used if clients want to make sure a patch
// is being applied to the latest resource version of an object.
//
@@ -94,68 +85,97 @@ type MergeFromOptions struct {
}
type mergeFromPatch struct {
from runtime.Object
opts MergeFromOptions
patchType types.PatchType
createPatch func(originalJSON, modifiedJSON []byte, dataStruct interface{}) ([]byte, error)
from Object
opts MergeFromOptions
}
// Type implements patch.
// Type implements Patch.
func (s *mergeFromPatch) Type() types.PatchType {
return types.MergePatchType
return s.patchType
}
// Data implements Patch.
func (s *mergeFromPatch) Data(obj runtime.Object) ([]byte, error) {
originalJSON, err := json.Marshal(s.from)
if err != nil {
return nil, err
}
modifiedJSON, err := json.Marshal(obj)
if err != nil {
return nil, err
}
data, err := jsonpatch.CreateMergePatch(originalJSON, modifiedJSON)
if err != nil {
return nil, err
}
func (s *mergeFromPatch) Data(obj Object) ([]byte, error) {
original := s.from
modified := obj
if s.opts.OptimisticLock {
dataMap := map[string]interface{}{}
if err := json.Unmarshal(data, &dataMap); err != nil {
return nil, err
}
fromMeta, ok := s.from.(metav1.Object)
if !ok {
return nil, fmt.Errorf("cannot use OptimisticLock, from object %q is not a valid metav1.Object", s.from)
}
resourceVersion := fromMeta.GetResourceVersion()
if len(resourceVersion) == 0 {
return nil, fmt.Errorf("cannot use OptimisticLock, from object %q does not have any resource version we can use", s.from)
}
u := &unstructured.Unstructured{Object: dataMap}
u.SetResourceVersion(resourceVersion)
data, err = json.Marshal(u)
if err != nil {
return nil, err
version := original.GetResourceVersion()
if len(version) == 0 {
return nil, fmt.Errorf("cannot use OptimisticLock, object %q does not have any resource version we can use", original)
}
original = original.DeepCopyObject().(Object)
original.SetResourceVersion("")
modified = modified.DeepCopyObject().(Object)
modified.SetResourceVersion(version)
}
originalJSON, err := json.Marshal(original)
if err != nil {
return nil, err
}
modifiedJSON, err := json.Marshal(modified)
if err != nil {
return nil, err
}
data, err := s.createPatch(originalJSON, modifiedJSON, obj)
if err != nil {
return nil, err
}
return data, nil
}
func createMergePatch(originalJSON, modifiedJSON []byte, _ interface{}) ([]byte, error) {
return jsonpatch.CreateMergePatch(originalJSON, modifiedJSON)
}
func createStrategicMergePatch(originalJSON, modifiedJSON []byte, dataStruct interface{}) ([]byte, error) {
return strategicpatch.CreateTwoWayMergePatch(originalJSON, modifiedJSON, dataStruct)
}
// MergeFrom creates a Patch that patches using the merge-patch strategy with the given object as base.
func MergeFrom(obj runtime.Object) Patch {
return &mergeFromPatch{from: obj}
// The difference between MergeFrom and StrategicMergeFrom lays in the handling of modified list fields.
// When using MergeFrom, existing lists will be completely replaced by new lists.
// When using StrategicMergeFrom, the list field's `patchStrategy` is respected if specified in the API type,
// e.g. the existing list is not replaced completely but rather merged with the new one using the list's `patchMergeKey`.
// See https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/ for more details on
// the difference between merge-patch and strategic-merge-patch.
func MergeFrom(obj Object) Patch {
return &mergeFromPatch{patchType: types.MergePatchType, createPatch: createMergePatch, from: obj}
}
// MergeFromWithOptions creates a Patch that patches using the merge-patch strategy with the given object as base.
func MergeFromWithOptions(obj runtime.Object, opts ...MergeFromOption) Patch {
// See MergeFrom for more details.
func MergeFromWithOptions(obj Object, opts ...MergeFromOption) Patch {
options := &MergeFromOptions{}
for _, opt := range opts {
opt.ApplyToMergeFrom(options)
}
return &mergeFromPatch{from: obj, opts: *options}
return &mergeFromPatch{patchType: types.MergePatchType, createPatch: createMergePatch, from: obj, opts: *options}
}
// StrategicMergeFrom creates a Patch that patches using the strategic-merge-patch strategy with the given object as base.
// The difference between MergeFrom and StrategicMergeFrom lays in the handling of modified list fields.
// When using MergeFrom, existing lists will be completely replaced by new lists.
// When using StrategicMergeFrom, the list field's `patchStrategy` is respected if specified in the API type,
// e.g. the existing list is not replaced completely but rather merged with the new one using the list's `patchMergeKey`.
// See https://kubernetes.io/docs/tasks/manage-kubernetes-objects/update-api-object-kubectl-patch/ for more details on
// the difference between merge-patch and strategic-merge-patch.
// Please note, that CRDs don't support strategic-merge-patch, see
// https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#advanced-features-and-flexibility
func StrategicMergeFrom(obj Object, opts ...MergeFromOption) Patch {
options := &MergeFromOptions{}
for _, opt := range opts {
opt.ApplyToMergeFrom(options)
}
return &mergeFromPatch{patchType: types.StrategicMergePatchType, createPatch: createStrategicMergePatch, from: obj, opts: *options}
}
// mergePatch uses a raw merge strategy to patch the object.
@@ -167,7 +187,7 @@ func (p mergePatch) Type() types.PatchType {
}
// Data implements Patch.
func (p mergePatch) Data(obj runtime.Object) ([]byte, error) {
func (p mergePatch) Data(obj Object) ([]byte, error) {
// NB(directxman12): we might technically want to be using an actual encoder
// here (in case some more performant encoder is introduced) but this is
// correct and sufficient for our uses (it's what the JSON serializer in
@@ -184,7 +204,7 @@ func (p applyPatch) Type() types.PatchType {
}
// Data implements Patch.
func (p applyPatch) Data(obj runtime.Object) ([]byte, error) {
func (p applyPatch) Data(obj Object) ([]byte, error) {
// NB(directxman12): we might technically want to be using an actual encoder
// here (in case some more performant encoder is introduced) but this is
// correct and sufficient for our uses (it's what the JSON serializer in

View File

@@ -18,43 +18,123 @@ package client
import (
"context"
"strings"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// DelegatingClient forms a Client by composing separate reader, writer and
// NewDelegatingClientInput encapsulates the input parameters to create a new delegating client.
type NewDelegatingClientInput struct {
CacheReader Reader
Client Client
UncachedObjects []Object
CacheUnstructured bool
}
// NewDelegatingClient creates a new delegating client.
//
// A delegating client forms a Client by composing separate reader, writer and
// statusclient interfaces. This way, you can have an Client that reads from a
// cache and writes to the API server.
type DelegatingClient struct {
func NewDelegatingClient(in NewDelegatingClientInput) (Client, error) {
uncachedGVKs := map[schema.GroupVersionKind]struct{}{}
for _, obj := range in.UncachedObjects {
gvk, err := apiutil.GVKForObject(obj, in.Client.Scheme())
if err != nil {
return nil, err
}
uncachedGVKs[gvk] = struct{}{}
}
return &delegatingClient{
scheme: in.Client.Scheme(),
mapper: in.Client.RESTMapper(),
Reader: &delegatingReader{
CacheReader: in.CacheReader,
ClientReader: in.Client,
scheme: in.Client.Scheme(),
uncachedGVKs: uncachedGVKs,
cacheUnstructured: in.CacheUnstructured,
},
Writer: in.Client,
StatusClient: in.Client,
}, nil
}
type delegatingClient struct {
Reader
Writer
StatusClient
scheme *runtime.Scheme
mapper meta.RESTMapper
}
// DelegatingReader forms a Reader that will cause Get and List requests for
// Scheme returns the scheme this client is using.
func (d *delegatingClient) Scheme() *runtime.Scheme {
return d.scheme
}
// RESTMapper returns the rest mapper this client is using.
func (d *delegatingClient) RESTMapper() meta.RESTMapper {
return d.mapper
}
// delegatingReader forms a Reader that will cause Get and List requests for
// unstructured types to use the ClientReader while requests for any other type
// of object with use the CacheReader. This avoids accidentally caching the
// entire cluster in the common case of loading arbitrary unstructured objects
// (e.g. from OwnerReferences).
type DelegatingReader struct {
type delegatingReader struct {
CacheReader Reader
ClientReader Reader
uncachedGVKs map[schema.GroupVersionKind]struct{}
scheme *runtime.Scheme
cacheUnstructured bool
}
func (d *delegatingReader) shouldBypassCache(obj runtime.Object) (bool, error) {
gvk, err := apiutil.GVKForObject(obj, d.scheme)
if err != nil {
return false, err
}
// TODO: this is producing unsafe guesses that don't actually work,
// but it matches ~99% of the cases out there.
if meta.IsListType(obj) {
gvk.Kind = strings.TrimSuffix(gvk.Kind, "List")
}
if _, isUncached := d.uncachedGVKs[gvk]; isUncached {
return true, nil
}
if !d.cacheUnstructured {
_, isUnstructured := obj.(*unstructured.Unstructured)
_, isUnstructuredList := obj.(*unstructured.UnstructuredList)
return isUnstructured || isUnstructuredList, nil
}
return false, nil
}
// Get retrieves an obj for a given object key from the Kubernetes Cluster.
func (d *DelegatingReader) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
_, isUnstructured := obj.(*unstructured.Unstructured)
if isUnstructured {
func (d *delegatingReader) Get(ctx context.Context, key ObjectKey, obj Object) error {
if isUncached, err := d.shouldBypassCache(obj); err != nil {
return err
} else if isUncached {
return d.ClientReader.Get(ctx, key, obj)
}
return d.CacheReader.Get(ctx, key, obj)
}
// List retrieves list of objects for a given namespace and list options.
func (d *DelegatingReader) List(ctx context.Context, list runtime.Object, opts ...ListOption) error {
_, isUnstructured := list.(*unstructured.UnstructuredList)
if isUnstructured {
func (d *delegatingReader) List(ctx context.Context, list ObjectList, opts ...ListOption) error {
if isUncached, err := d.shouldBypassCache(list); err != nil {
return err
} else if isUncached {
return d.ClientReader.List(ctx, list, opts...)
}
return d.CacheReader.List(ctx, list, opts...)

View File

@@ -22,6 +22,10 @@ import (
"k8s.io/apimachinery/pkg/runtime"
)
var _ Reader = &typedClient{}
var _ Writer = &typedClient{}
var _ StatusWriter = &typedClient{}
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type typedClient struct {
@@ -29,8 +33,8 @@ type typedClient struct {
paramCodec runtime.ParameterCodec
}
// Create implements client.Client
func (c *typedClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
// Create implements client.Client.
func (c *typedClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -47,8 +51,8 @@ func (c *typedClient) Create(ctx context.Context, obj runtime.Object, opts ...Cr
Into(obj)
}
// Update implements client.Client
func (c *typedClient) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.Client.
func (c *typedClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -66,8 +70,8 @@ func (c *typedClient) Update(ctx context.Context, obj runtime.Object, opts ...Up
Into(obj)
}
// Delete implements client.Client
func (c *typedClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
// Delete implements client.Client.
func (c *typedClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -85,8 +89,8 @@ func (c *typedClient) Delete(ctx context.Context, obj runtime.Object, opts ...De
Error()
}
// DeleteAllOf implements client.Client
func (c *typedClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
// DeleteAllOf implements client.Client.
func (c *typedClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -104,8 +108,8 @@ func (c *typedClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts
Error()
}
// Patch implements client.Client
func (c *typedClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (c *typedClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -127,8 +131,8 @@ func (c *typedClient) Patch(ctx context.Context, obj runtime.Object, patch Patch
Into(obj)
}
// Get implements client.Client
func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
// Get implements client.Client.
func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
r, err := c.cache.getResource(obj)
if err != nil {
return err
@@ -139,8 +143,8 @@ func (c *typedClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object
Name(key.Name).Do(ctx).Into(obj)
}
// List implements client.Client
func (c *typedClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
// List implements client.Client.
func (c *typedClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
r, err := c.cache.getResource(obj)
if err != nil {
return err
@@ -156,7 +160,7 @@ func (c *typedClient) List(ctx context.Context, obj runtime.Object, opts ...List
}
// UpdateStatus used by StatusWriter to write status.
func (c *typedClient) UpdateStatus(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
func (c *typedClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err
@@ -177,7 +181,7 @@ func (c *typedClient) UpdateStatus(ctx context.Context, obj runtime.Object, opts
}
// PatchStatus used by StatusWriter to write status.
func (c *typedClient) PatchStatus(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
func (c *typedClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
o, err := c.cache.getObjMeta(obj)
if err != nil {
return err

View File

@@ -25,6 +25,10 @@ import (
"k8s.io/apimachinery/pkg/runtime"
)
var _ Reader = &unstructuredClient{}
var _ Writer = &unstructuredClient{}
var _ StatusWriter = &unstructuredClient{}
// client is a client.Client that reads and writes directly from/to an API server. It lazily initializes
// new clients at the time they are used, and caches the client.
type unstructuredClient struct {
@@ -32,8 +36,8 @@ type unstructuredClient struct {
paramCodec runtime.ParameterCodec
}
// Create implements client.Client
func (uc *unstructuredClient) Create(ctx context.Context, obj runtime.Object, opts ...CreateOption) error {
// Create implements client.Client.
func (uc *unstructuredClient) Create(ctx context.Context, obj Object, opts ...CreateOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -60,8 +64,8 @@ func (uc *unstructuredClient) Create(ctx context.Context, obj runtime.Object, op
return result
}
// Update implements client.Client
func (uc *unstructuredClient) Update(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
// Update implements client.Client.
func (uc *unstructuredClient) Update(ctx context.Context, obj Object, opts ...UpdateOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -89,8 +93,8 @@ func (uc *unstructuredClient) Update(ctx context.Context, obj runtime.Object, op
return result
}
// Delete implements client.Client
func (uc *unstructuredClient) Delete(ctx context.Context, obj runtime.Object, opts ...DeleteOption) error {
// Delete implements client.Client.
func (uc *unstructuredClient) Delete(ctx context.Context, obj Object, opts ...DeleteOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -112,8 +116,8 @@ func (uc *unstructuredClient) Delete(ctx context.Context, obj runtime.Object, op
Error()
}
// DeleteAllOf implements client.Client
func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj runtime.Object, opts ...DeleteAllOfOption) error {
// DeleteAllOf implements client.Client.
func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj Object, opts ...DeleteAllOfOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -135,8 +139,8 @@ func (uc *unstructuredClient) DeleteAllOf(ctx context.Context, obj runtime.Objec
Error()
}
// Patch implements client.Client
func (uc *unstructuredClient) Patch(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
// Patch implements client.Client.
func (uc *unstructuredClient) Patch(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -163,8 +167,8 @@ func (uc *unstructuredClient) Patch(ctx context.Context, obj runtime.Object, pat
Into(obj)
}
// Get implements client.Client
func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj runtime.Object) error {
// Get implements client.Client.
func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj Object) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -189,8 +193,8 @@ func (uc *unstructuredClient) Get(ctx context.Context, key ObjectKey, obj runtim
return result
}
// List implements client.Client
func (uc *unstructuredClient) List(ctx context.Context, obj runtime.Object, opts ...ListOption) error {
// List implements client.Client.
func (uc *unstructuredClient) List(ctx context.Context, obj ObjectList, opts ...ListOption) error {
u, ok := obj.(*unstructured.UnstructuredList)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -217,7 +221,7 @@ func (uc *unstructuredClient) List(ctx context.Context, obj runtime.Object, opts
Into(obj)
}
func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj runtime.Object, opts ...UpdateOption) error {
func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj Object, opts ...UpdateOption) error {
_, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)
@@ -239,7 +243,7 @@ func (uc *unstructuredClient) UpdateStatus(ctx context.Context, obj runtime.Obje
Into(obj)
}
func (uc *unstructuredClient) PatchStatus(ctx context.Context, obj runtime.Object, patch Patch, opts ...PatchOption) error {
func (uc *unstructuredClient) PatchStatus(ctx context.Context, obj Object, patch Patch, opts ...PatchOption) error {
u, ok := obj.(*unstructured.Unstructured)
if !ok {
return fmt.Errorf("unstructured client did not understand object: %T", obj)

View File

@@ -0,0 +1,118 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package client
import (
"context"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/rest"
)
// NewWithWatch returns a new WithWatch.
func NewWithWatch(config *rest.Config, options Options) (WithWatch, error) {
client, err := newClient(config, options)
if err != nil {
return nil, err
}
dynamicClient, err := dynamic.NewForConfig(config)
if err != nil {
return nil, err
}
return &watchingClient{client: client, dynamic: dynamicClient}, nil
}
type watchingClient struct {
*client
dynamic dynamic.Interface
}
func (w *watchingClient) Watch(ctx context.Context, list ObjectList, opts ...ListOption) (watch.Interface, error) {
switch l := list.(type) {
case *unstructured.UnstructuredList:
return w.unstructuredWatch(ctx, l, opts...)
case *metav1.PartialObjectMetadataList:
return w.metadataWatch(ctx, l, opts...)
default:
return w.typedWatch(ctx, l, opts...)
}
}
func (w *watchingClient) listOpts(opts ...ListOption) ListOptions {
listOpts := ListOptions{}
listOpts.ApplyOptions(opts)
if listOpts.Raw == nil {
listOpts.Raw = &metav1.ListOptions{}
}
listOpts.Raw.Watch = true
return listOpts
}
func (w *watchingClient) metadataWatch(ctx context.Context, obj *metav1.PartialObjectMetadataList, opts ...ListOption) (watch.Interface, error) {
gvk := obj.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
listOpts := w.listOpts(opts...)
resInt, err := w.client.metadataClient.getResourceInterface(gvk, listOpts.Namespace)
if err != nil {
return nil, err
}
return resInt.Watch(ctx, *listOpts.AsListOptions())
}
func (w *watchingClient) unstructuredWatch(ctx context.Context, obj *unstructured.UnstructuredList, opts ...ListOption) (watch.Interface, error) {
gvk := obj.GroupVersionKind()
if strings.HasSuffix(gvk.Kind, "List") {
gvk.Kind = gvk.Kind[:len(gvk.Kind)-4]
}
r, err := w.client.unstructuredClient.cache.getResource(obj)
if err != nil {
return nil, err
}
listOpts := w.listOpts(opts...)
if listOpts.Namespace != "" && r.isNamespaced() {
return w.dynamic.Resource(r.mapping.Resource).Namespace(listOpts.Namespace).Watch(ctx, *listOpts.AsListOptions())
}
return w.dynamic.Resource(r.mapping.Resource).Watch(ctx, *listOpts.AsListOptions())
}
func (w *watchingClient) typedWatch(ctx context.Context, obj ObjectList, opts ...ListOption) (watch.Interface, error) {
r, err := w.client.typedClient.cache.getResource(obj)
if err != nil {
return nil, err
}
listOpts := w.listOpts(opts...)
return r.Get().
NamespaceIfScoped(listOpts.Namespace, r.isNamespaced()).
Resource(r.resource()).
VersionedParams(listOpts.AsListOptions(), w.client.typedClient.paramCodec).
Watch(ctx)
}

View File

@@ -0,0 +1,270 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cluster
import (
"context"
"errors"
"time"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
)
// Cluster provides various methods to interact with a cluster.
type Cluster interface {
// SetFields will set any dependencies on an object for which the object has implemented the inject
// interface - e.g. inject.Client.
// Deprecated: use the equivalent Options field to set a field. This method will be removed in v0.10.
SetFields(interface{}) error
// GetConfig returns an initialized Config
GetConfig() *rest.Config
// GetScheme returns an initialized Scheme
GetScheme() *runtime.Scheme
// GetClient returns a client configured with the Config. This client may
// not be a fully "direct" client -- it may read from a cache, for
// instance. See Options.NewClient for more information on how the default
// implementation works.
GetClient() client.Client
// GetFieldIndexer returns a client.FieldIndexer configured with the client
GetFieldIndexer() client.FieldIndexer
// GetCache returns a cache.Cache
GetCache() cache.Cache
// GetEventRecorderFor returns a new EventRecorder for the provided name
GetEventRecorderFor(name string) record.EventRecorder
// GetRESTMapper returns a RESTMapper
GetRESTMapper() meta.RESTMapper
// GetAPIReader returns a reader that will be configured to use the API server.
// This should be used sparingly and only when the client does not fit your
// use case.
GetAPIReader() client.Reader
// Start starts the cluster
Start(ctx context.Context) error
}
// Options are the possible options that can be configured for a Cluster.
type Options struct {
// Scheme is the scheme used to resolve runtime.Objects to GroupVersionKinds / Resources
// Defaults to the kubernetes/client-go scheme.Scheme, but it's almost always better
// idea to pass your own scheme in. See the documentation in pkg/scheme for more information.
Scheme *runtime.Scheme
// MapperProvider provides the rest mapper used to map go types to Kubernetes APIs
MapperProvider func(c *rest.Config) (meta.RESTMapper, error)
// Logger is the logger that should be used by this Cluster.
// If none is set, it defaults to log.Log global logger.
Logger logr.Logger
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
SyncPeriod *time.Duration
// Namespace if specified restricts the manager's cache to watch objects in
// the desired namespace Defaults to all namespaces
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources the cache
// will only hold objects from the desired namespace.
Namespace string
// NewCache is the function that will create the cache to be used
// by the manager. If not set this will use the default new cache function.
NewCache cache.NewCacheFunc
// NewClient is the func that creates the client to be used by the manager.
// If not set this will create the default DelegatingClient that will
// use the cache for reads and the client for writes.
NewClient NewClientFunc
// ClientDisableCacheFor tells the client that, if any cache is used, to bypass it
// for the given objects.
ClientDisableCacheFor []client.Object
// DryRunClient specifies whether the client should be configured to enforce
// dryRun mode.
DryRunClient bool
// EventBroadcaster records Events emitted by the manager and sends them to the Kubernetes API
// Use this to customize the event correlator and spam filter
//
// Deprecated: using this may cause goroutine leaks if the lifetime of your manager or controllers
// is shorter than the lifetime of your process.
EventBroadcaster record.EventBroadcaster
// makeBroadcaster allows deferring the creation of the broadcaster to
// avoid leaking goroutines if we never call Start on this manager. It also
// returns whether or not this is a "owned" broadcaster, and as such should be
// stopped with the manager.
makeBroadcaster intrec.EventBroadcasterProducer
// Dependency injection for testing
newRecorderProvider func(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster intrec.EventBroadcasterProducer) (*intrec.Provider, error)
}
// Option can be used to manipulate Options.
type Option func(*Options)
// New constructs a brand new cluster.
func New(config *rest.Config, opts ...Option) (Cluster, error) {
if config == nil {
return nil, errors.New("must specify Config")
}
options := Options{}
for _, opt := range opts {
opt(&options)
}
options = setOptionsDefaults(options)
// Create the mapper provider
mapper, err := options.MapperProvider(config)
if err != nil {
options.Logger.Error(err, "Failed to get API Group-Resources")
return nil, err
}
// Create the cache for the cached read client and registering informers
cache, err := options.NewCache(config, cache.Options{Scheme: options.Scheme, Mapper: mapper, Resync: options.SyncPeriod, Namespace: options.Namespace})
if err != nil {
return nil, err
}
clientOptions := client.Options{Scheme: options.Scheme, Mapper: mapper}
apiReader, err := client.New(config, clientOptions)
if err != nil {
return nil, err
}
writeObj, err := options.NewClient(cache, config, clientOptions, options.ClientDisableCacheFor...)
if err != nil {
return nil, err
}
if options.DryRunClient {
writeObj = client.NewDryRunClient(writeObj)
}
// Create the recorder provider to inject event recorders for the components.
// TODO(directxman12): the log for the event provider should have a context (name, tags, etc) specific
// to the particular controller that it's being injected into, rather than a generic one like is here.
recorderProvider, err := options.newRecorderProvider(config, options.Scheme, options.Logger.WithName("events"), options.makeBroadcaster)
if err != nil {
return nil, err
}
return &cluster{
config: config,
scheme: options.Scheme,
cache: cache,
fieldIndexes: cache,
client: writeObj,
apiReader: apiReader,
recorderProvider: recorderProvider,
mapper: mapper,
logger: options.Logger,
}, nil
}
// setOptionsDefaults set default values for Options fields.
func setOptionsDefaults(options Options) Options {
// Use the Kubernetes client-go scheme if none is specified
if options.Scheme == nil {
options.Scheme = scheme.Scheme
}
if options.MapperProvider == nil {
options.MapperProvider = func(c *rest.Config) (meta.RESTMapper, error) {
return apiutil.NewDynamicRESTMapper(c)
}
}
// Allow users to define how to create a new client
if options.NewClient == nil {
options.NewClient = DefaultNewClient
}
// Allow newCache to be mocked
if options.NewCache == nil {
options.NewCache = cache.New
}
// Allow newRecorderProvider to be mocked
if options.newRecorderProvider == nil {
options.newRecorderProvider = intrec.NewProvider
}
// This is duplicated with pkg/manager, we need it here to provide
// the user with an EventBroadcaster and there for the Leader election
if options.EventBroadcaster == nil {
// defer initialization to avoid leaking by default
options.makeBroadcaster = func() (record.EventBroadcaster, bool) {
return record.NewBroadcaster(), true
}
} else {
options.makeBroadcaster = func() (record.EventBroadcaster, bool) {
return options.EventBroadcaster, false
}
}
if options.Logger == nil {
options.Logger = logf.RuntimeLog.WithName("cluster")
}
return options
}
// NewClientFunc allows a user to define how to create a client.
type NewClientFunc func(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error)
// DefaultNewClient creates the default caching client.
func DefaultNewClient(cache cache.Cache, config *rest.Config, options client.Options, uncachedObjects ...client.Object) (client.Client, error) {
c, err := client.New(config, options)
if err != nil {
return nil, err
}
return client.NewDelegatingClient(client.NewDelegatingClientInput{
CacheReader: cache,
Client: c,
UncachedObjects: uncachedObjects,
})
}

View File

@@ -0,0 +1,128 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cluster
import (
"context"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
intrec "sigs.k8s.io/controller-runtime/pkg/internal/recorder"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
type cluster struct {
// config is the rest.config used to talk to the apiserver. Required.
config *rest.Config
// scheme is the scheme injected into Controllers, EventHandlers, Sources and Predicates. Defaults
// to scheme.scheme.
scheme *runtime.Scheme
cache cache.Cache
// TODO(directxman12): Provide an escape hatch to get individual indexers
// client is the client injected into Controllers (and EventHandlers, Sources and Predicates).
client client.Client
// apiReader is the reader that will make requests to the api server and not the cache.
apiReader client.Reader
// fieldIndexes knows how to add field indexes over the Cache used by this controller,
// which can later be consumed via field selectors from the injected client.
fieldIndexes client.FieldIndexer
// recorderProvider is used to generate event recorders that will be injected into Controllers
// (and EventHandlers, Sources and Predicates).
recorderProvider *intrec.Provider
// mapper is used to map resources to kind, and map kind and version.
mapper meta.RESTMapper
// Logger is the logger that should be used by this manager.
// If none is set, it defaults to log.Log global logger.
logger logr.Logger
}
func (c *cluster) SetFields(i interface{}) error {
if _, err := inject.ConfigInto(c.config, i); err != nil {
return err
}
if _, err := inject.ClientInto(c.client, i); err != nil {
return err
}
if _, err := inject.APIReaderInto(c.apiReader, i); err != nil {
return err
}
if _, err := inject.SchemeInto(c.scheme, i); err != nil {
return err
}
if _, err := inject.CacheInto(c.cache, i); err != nil {
return err
}
if _, err := inject.MapperInto(c.mapper, i); err != nil {
return err
}
return nil
}
func (c *cluster) GetConfig() *rest.Config {
return c.config
}
func (c *cluster) GetClient() client.Client {
return c.client
}
func (c *cluster) GetScheme() *runtime.Scheme {
return c.scheme
}
func (c *cluster) GetFieldIndexer() client.FieldIndexer {
return c.fieldIndexes
}
func (c *cluster) GetCache() cache.Cache {
return c.cache
}
func (c *cluster) GetEventRecorderFor(name string) record.EventRecorder {
return c.recorderProvider.GetEventRecorderFor(name)
}
func (c *cluster) GetRESTMapper() meta.RESTMapper {
return c.mapper
}
func (c *cluster) GetAPIReader() client.Reader {
return c.apiReader
}
func (c *cluster) GetLogger() logr.Logger {
return c.logger
}
func (c *cluster) Start(ctx context.Context) error {
defer c.recorderProvider.Stop(ctx)
return c.cache.Start(ctx)
}

View File

@@ -0,0 +1,112 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package config
import (
"fmt"
ioutil "io/ioutil"
"sync"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"sigs.k8s.io/controller-runtime/pkg/config/v1alpha1"
)
// ControllerManagerConfiguration defines the functions necessary to parse a config file
// and to configure the Options struct for the ctrl.Manager.
type ControllerManagerConfiguration interface {
runtime.Object
// Complete returns the versioned configuration
Complete() (v1alpha1.ControllerManagerConfigurationSpec, error)
}
// DeferredFileLoader is used to configure the decoder for loading controller
// runtime component config types.
type DeferredFileLoader struct {
ControllerManagerConfiguration
path string
scheme *runtime.Scheme
once sync.Once
err error
}
// File will set up the deferred file loader for the configuration
// this will also configure the defaults for the loader if nothing is
//
// Defaults:
// Path: "./config.yaml"
// Kind: GenericControllerManagerConfiguration
func File() *DeferredFileLoader {
scheme := runtime.NewScheme()
utilruntime.Must(v1alpha1.AddToScheme(scheme))
return &DeferredFileLoader{
path: "./config.yaml",
ControllerManagerConfiguration: &v1alpha1.ControllerManagerConfiguration{},
scheme: scheme,
}
}
// Complete will use sync.Once to set the scheme.
func (d *DeferredFileLoader) Complete() (v1alpha1.ControllerManagerConfigurationSpec, error) {
d.once.Do(d.loadFile)
if d.err != nil {
return v1alpha1.ControllerManagerConfigurationSpec{}, d.err
}
return d.ControllerManagerConfiguration.Complete()
}
// AtPath will set the path to load the file for the decoder.
func (d *DeferredFileLoader) AtPath(path string) *DeferredFileLoader {
d.path = path
return d
}
// OfKind will set the type to be used for decoding the file into.
func (d *DeferredFileLoader) OfKind(obj ControllerManagerConfiguration) *DeferredFileLoader {
d.ControllerManagerConfiguration = obj
return d
}
// InjectScheme will configure the scheme to be used for decoding the file.
func (d *DeferredFileLoader) InjectScheme(scheme *runtime.Scheme) error {
d.scheme = scheme
return nil
}
// loadFile is used from the mutex.Once to load the file.
func (d *DeferredFileLoader) loadFile() {
if d.scheme == nil {
d.err = fmt.Errorf("scheme not supplied to controller configuration loader")
return
}
content, err := ioutil.ReadFile(d.path)
if err != nil {
d.err = fmt.Errorf("could not read file at %s", d.path)
return
}
codecs := serializer.NewCodecFactory(d.scheme)
// Regardless of if the bytes are of any external version,
// it will be read successfully and converted into the internal version
if err = runtime.DecodeInto(codecs.UniversalDecoder(), content, d.ControllerManagerConfiguration); err != nil {
d.err = fmt.Errorf("could not decode file into runtime.Object")
}
}

View File

@@ -0,0 +1,25 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package config contains functionality for interacting with ComponentConfig
// files
//
// DeferredFileLoader
//
// This uses a deferred file decoding allowing you to chain your configuration
// setup. You can pass this into manager.Options#File and it will load your
// config.
package config

View File

@@ -1,5 +1,5 @@
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,12 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package defaultconfig
const (
namespaceFieldSpecs = `
namespace:
- path: metadata/namespace
create: true
`
)
// Package v1alpha1 provides the ControllerManagerConfiguration used for
// configuring ctrl.Manager
// +kubebuilder:object:generate=true
package v1alpha1

View File

@@ -0,0 +1,37 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)
var (
// GroupVersion is group version used to register these objects.
GroupVersion = schema.GroupVersion{Group: "controller-runtime.sigs.k8s.io", Version: "v1alpha1"}
// SchemeBuilder is used to add go types to the GroupVersionKind scheme.
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)
func init() {
SchemeBuilder.Register(&ControllerManagerConfiguration{})
}

View File

@@ -0,0 +1,157 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
configv1alpha1 "k8s.io/component-base/config/v1alpha1"
)
// ControllerManagerConfigurationSpec defines the desired state of GenericControllerManagerConfiguration.
type ControllerManagerConfigurationSpec struct {
// SyncPeriod determines the minimum frequency at which watched resources are
// reconciled. A lower period will correct entropy more quickly, but reduce
// responsiveness to change if there are many watched resources. Change this
// value only if you know what you are doing. Defaults to 10 hours if unset.
// there will a 10 percent jitter between the SyncPeriod of all controllers
// so that all controllers will not send list requests simultaneously.
// +optional
SyncPeriod *metav1.Duration `json:"syncPeriod,omitempty"`
// LeaderElection is the LeaderElection config to be used when configuring
// the manager.Manager leader election
// +optional
LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
// CacheNamespace if specified restricts the manager's cache to watch objects in
// the desired namespace Defaults to all namespaces
//
// Note: If a namespace is specified, controllers can still Watch for a
// cluster-scoped resource (e.g Node). For namespaced resources the cache
// will only hold objects from the desired namespace.
// +optional
CacheNamespace string `json:"cacheNamespace,omitempty"`
// GracefulShutdownTimeout is the duration given to runnable to stop before the manager actually returns on stop.
// To disable graceful shutdown, set to time.Duration(0)
// To use graceful shutdown without timeout, set to a negative duration, e.G. time.Duration(-1)
// The graceful shutdown is skipped for safety reasons in case the leader election lease is lost.
GracefulShutdownTimeout *metav1.Duration `json:"gracefulShutDown,omitempty"`
// Controller contains global configuration options for controllers
// registered within this manager.
// +optional
Controller *ControllerConfigurationSpec `json:"controller,omitempty"`
// Metrics contains thw controller metrics configuration
// +optional
Metrics ControllerMetrics `json:"metrics,omitempty"`
// Health contains the controller health configuration
// +optional
Health ControllerHealth `json:"health,omitempty"`
// Webhook contains the controllers webhook configuration
// +optional
Webhook ControllerWebhook `json:"webhook,omitempty"`
}
// ControllerConfigurationSpec defines the global configuration for
// controllers registered with the manager.
type ControllerConfigurationSpec struct {
// GroupKindConcurrency is a map from a Kind to the number of concurrent reconciliation
// allowed for that controller.
//
// When a controller is registered within this manager using the builder utilities,
// users have to specify the type the controller reconciles in the For(...) call.
// If the object's kind passed matches one of the keys in this map, the concurrency
// for that controller is set to the number specified.
//
// The key is expected to be consistent in form with GroupKind.String(),
// e.g. ReplicaSet in apps group (regardless of version) would be `ReplicaSet.apps`.
//
// +optional
GroupKindConcurrency map[string]int `json:"groupKindConcurrency,omitempty"`
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
// +optional
CacheSyncTimeout *time.Duration `json:"cacheSyncTimeout,omitempty"`
}
// ControllerMetrics defines the metrics configs.
type ControllerMetrics struct {
// BindAddress is the TCP address that the controller should bind to
// for serving prometheus metrics.
// It can be set to "0" to disable the metrics serving.
// +optional
BindAddress string `json:"bindAddress,omitempty"`
}
// ControllerHealth defines the health configs.
type ControllerHealth struct {
// HealthProbeBindAddress is the TCP address that the controller should bind to
// for serving health probes
// +optional
HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"`
// ReadinessEndpointName, defaults to "readyz"
// +optional
ReadinessEndpointName string `json:"readinessEndpointName,omitempty"`
// LivenessEndpointName, defaults to "healthz"
// +optional
LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
}
// ControllerWebhook defines the webhook server for the controller.
type ControllerWebhook struct {
// Port is the port that the webhook server serves at.
// It is used to set webhook.Server.Port.
// +optional
Port *int `json:"port,omitempty"`
// Host is the hostname that the webhook server binds to.
// It is used to set webhook.Server.Host.
// +optional
Host string `json:"host,omitempty"`
// CertDir is the directory that contains the server key and certificate.
// if not set, webhook server would look up the server key and certificate in
// {TempDir}/k8s-webhook-server/serving-certs. The server key and certificate
// must be named tls.key and tls.crt, respectively.
// +optional
CertDir string `json:"certDir,omitempty"`
}
// +kubebuilder:object:root=true
// ControllerManagerConfiguration is the Schema for the GenericControllerManagerConfigurations API.
type ControllerManagerConfiguration struct {
metav1.TypeMeta `json:",inline"`
// ControllerManagerConfiguration returns the contfigurations for controllers
ControllerManagerConfigurationSpec `json:",inline"`
}
// Complete returns the configuration for controller-runtime.
func (c *ControllerManagerConfigurationSpec) Complete() (ControllerManagerConfigurationSpec, error) {
return *c, nil
}

View File

@@ -0,0 +1,152 @@
// +build !ignore_autogenerated
// Code generated by controller-gen. DO NOT EDIT.
package v1alpha1
import (
"k8s.io/apimachinery/pkg/apis/meta/v1"
runtime "k8s.io/apimachinery/pkg/runtime"
configv1alpha1 "k8s.io/component-base/config/v1alpha1"
timex "time"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerConfigurationSpec) DeepCopyInto(out *ControllerConfigurationSpec) {
*out = *in
if in.GroupKindConcurrency != nil {
in, out := &in.GroupKindConcurrency, &out.GroupKindConcurrency
*out = make(map[string]int, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.CacheSyncTimeout != nil {
in, out := &in.CacheSyncTimeout, &out.CacheSyncTimeout
*out = new(timex.Duration)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerConfigurationSpec.
func (in *ControllerConfigurationSpec) DeepCopy() *ControllerConfigurationSpec {
if in == nil {
return nil
}
out := new(ControllerConfigurationSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerHealth) DeepCopyInto(out *ControllerHealth) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerHealth.
func (in *ControllerHealth) DeepCopy() *ControllerHealth {
if in == nil {
return nil
}
out := new(ControllerHealth)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerManagerConfiguration) DeepCopyInto(out *ControllerManagerConfiguration) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ControllerManagerConfigurationSpec.DeepCopyInto(&out.ControllerManagerConfigurationSpec)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfiguration.
func (in *ControllerManagerConfiguration) DeepCopy() *ControllerManagerConfiguration {
if in == nil {
return nil
}
out := new(ControllerManagerConfiguration)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *ControllerManagerConfiguration) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerManagerConfigurationSpec) DeepCopyInto(out *ControllerManagerConfigurationSpec) {
*out = *in
if in.SyncPeriod != nil {
in, out := &in.SyncPeriod, &out.SyncPeriod
*out = new(v1.Duration)
**out = **in
}
if in.LeaderElection != nil {
in, out := &in.LeaderElection, &out.LeaderElection
*out = new(configv1alpha1.LeaderElectionConfiguration)
(*in).DeepCopyInto(*out)
}
if in.GracefulShutdownTimeout != nil {
in, out := &in.GracefulShutdownTimeout, &out.GracefulShutdownTimeout
*out = new(v1.Duration)
**out = **in
}
if in.Controller != nil {
in, out := &in.Controller, &out.Controller
*out = new(ControllerConfigurationSpec)
(*in).DeepCopyInto(*out)
}
out.Metrics = in.Metrics
out.Health = in.Health
in.Webhook.DeepCopyInto(&out.Webhook)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerManagerConfigurationSpec.
func (in *ControllerManagerConfigurationSpec) DeepCopy() *ControllerManagerConfigurationSpec {
if in == nil {
return nil
}
out := new(ControllerManagerConfigurationSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerMetrics) DeepCopyInto(out *ControllerMetrics) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerMetrics.
func (in *ControllerMetrics) DeepCopy() *ControllerMetrics {
if in == nil {
return nil
}
out := new(ControllerMetrics)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ControllerWebhook) DeepCopyInto(out *ControllerWebhook) {
*out = *in
if in.Port != nil {
in, out := &in.Port, &out.Port
*out = new(int)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ControllerWebhook.
func (in *ControllerWebhook) DeepCopy() *ControllerWebhook {
if in == nil {
return nil
}
out := new(ControllerWebhook)
in.DeepCopyInto(out)
return out
}

View File

@@ -17,7 +17,9 @@ limitations under the License.
package controller
import (
"context"
"fmt"
"time"
"github.com/go-logr/logr"
"k8s.io/client-go/util/workqueue"
@@ -30,7 +32,7 @@ import (
"sigs.k8s.io/controller-runtime/pkg/source"
)
// Options are the arguments for creating a new Controller
// Options are the arguments for creating a new Controller.
type Options struct {
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
MaxConcurrentReconciles int
@@ -43,8 +45,13 @@ type Options struct {
// The overall is a token bucket and the per-item is exponential.
RateLimiter ratelimiter.RateLimiter
// Log is the logger used for this controller.
// Log is the logger used for this controller and passed to each reconciliation
// request via the context field.
Log logr.Logger
// CacheSyncTimeout refers to the time limit set to wait for syncing caches.
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
}
// Controller implements a Kubernetes API. A Controller manages a work queue fed reconcile.Requests
@@ -63,9 +70,12 @@ type Controller interface {
// EventHandler if all provided Predicates evaluate to true.
Watch(src source.Source, eventhandler handler.EventHandler, predicates ...predicate.Predicate) error
// Start starts the controller. Start blocks until stop is closed or a
// Start starts the controller. Start blocks until the context is closed or a
// controller has an error starting.
Start(stop <-chan struct{}) error
Start(ctx context.Context) error
// GetLogger returns this controller logger prefilled with basic information.
GetLogger() logr.Logger
}
// New returns a new Controller registered with the Manager. The Manager will ensure that shared Caches have
@@ -91,16 +101,20 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
return nil, fmt.Errorf("must specify Name for Controller")
}
if options.Log == nil {
options.Log = mgr.GetLogger()
}
if options.MaxConcurrentReconciles <= 0 {
options.MaxConcurrentReconciles = 1
}
if options.RateLimiter == nil {
options.RateLimiter = workqueue.DefaultControllerRateLimiter()
if options.CacheSyncTimeout == 0 {
options.CacheSyncTimeout = 2 * time.Minute
}
if options.Log == nil {
options.Log = mgr.GetLogger()
if options.RateLimiter == nil {
options.RateLimiter = workqueue.DefaultControllerRateLimiter()
}
// Inject dependencies into Reconciler
@@ -115,8 +129,9 @@ func NewUnmanaged(name string, mgr manager.Manager, options Options) (Controller
return workqueue.NewNamedRateLimitingQueue(options.RateLimiter, name)
},
MaxConcurrentReconciles: options.MaxConcurrentReconciles,
CacheSyncTimeout: options.CacheSyncTimeout,
SetFields: mgr.SetFields,
Name: name,
Log: options.Log.WithName("controller").WithValues("controller", name),
Log: options.Log.WithName("controller").WithName(name),
}, nil
}

View File

@@ -19,11 +19,12 @@ package controllerutil
import (
"context"
"fmt"
"reflect"
"k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/utils/pointer"
@@ -33,7 +34,7 @@ import (
// AlreadyOwnedError is an error returned if the object you are trying to assign
// a controller reference is already owned by another controller Object is the
// subject and Owner is the reference for the current owner
// subject and Owner is the reference for the current owner.
type AlreadyOwnedError struct {
Object metav1.Object
Owner metav1.OwnerReference
@@ -43,10 +44,10 @@ func (e *AlreadyOwnedError) Error() string {
return fmt.Sprintf("Object %s/%s is already owned by another %s controller %s", e.Object.GetNamespace(), e.Object.GetName(), e.Owner.Kind, e.Owner.Name)
}
func newAlreadyOwnedError(Object metav1.Object, Owner metav1.OwnerReference) *AlreadyOwnedError {
func newAlreadyOwnedError(obj metav1.Object, owner metav1.OwnerReference) *AlreadyOwnedError {
return &AlreadyOwnedError{
Object: Object,
Owner: Owner,
Object: obj,
Owner: owner,
}
}
@@ -117,13 +118,11 @@ func SetOwnerReference(owner, object metav1.Object, scheme *runtime.Scheme) erro
// Update owner references and return.
upsertOwnerRef(ref, object)
return nil
}
func upsertOwnerRef(ref metav1.OwnerReference, object metav1.Object) {
owners := object.GetOwnerReferences()
idx := indexOwnerRef(owners, ref)
if idx == -1 {
if idx := indexOwnerRef(owners, ref); idx == -1 {
owners = append(owners, ref)
} else {
owners[idx] = ref
@@ -155,7 +154,7 @@ func validateOwner(owner, object metav1.Object) error {
return nil
}
// Returns true if a and b point to the same object
// Returns true if a and b point to the same object.
func referSameObject(a, b metav1.OwnerReference) bool {
aGV, err := schema.ParseGroupVersion(a.APIVersion)
if err != nil {
@@ -170,16 +169,20 @@ func referSameObject(a, b metav1.OwnerReference) bool {
return aGV.Group == bGV.Group && a.Kind == b.Kind && a.Name == b.Name
}
// OperationResult is the action result of a CreateOrUpdate call
// OperationResult is the action result of a CreateOrUpdate call.
type OperationResult string
const ( // They should complete the sentence "Deployment default/foo has been ..."
// OperationResultNone means that the resource has not been changed
// OperationResultNone means that the resource has not been changed.
OperationResultNone OperationResult = "unchanged"
// OperationResultCreated means that a new resource is created
// OperationResultCreated means that a new resource is created.
OperationResultCreated OperationResult = "created"
// OperationResultUpdated means that an existing resource is updated
// OperationResultUpdated means that an existing resource is updated.
OperationResultUpdated OperationResult = "updated"
// OperationResultUpdatedStatus means that an existing resource and its status is updated.
OperationResultUpdatedStatus OperationResult = "updatedStatus"
// OperationResultUpdatedStatusOnly means that only an existing status is updated.
OperationResultUpdatedStatusOnly OperationResult = "updatedStatusOnly"
)
// CreateOrUpdate creates or updates the given object in the Kubernetes
@@ -189,14 +192,10 @@ const ( // They should complete the sentence "Deployment default/foo has been ..
// The MutateFn is called regardless of creating or updating an object.
//
// It returns the executed operation and an error.
func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object, f MutateFn) (OperationResult, error) {
key, err := client.ObjectKeyFromObject(obj)
if err != nil {
return OperationResultNone, err
}
func CreateOrUpdate(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) {
key := client.ObjectKeyFromObject(obj)
if err := c.Get(ctx, key, obj); err != nil {
if !errors.IsNotFound(err) {
if !apierrors.IsNotFound(err) {
return OperationResultNone, err
}
if err := mutate(f, key, obj); err != nil {
@@ -208,7 +207,7 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object, f
return OperationResultCreated, nil
}
existing := obj.DeepCopyObject()
existing := obj.DeepCopyObject() //nolint:ifshort
if err := mutate(f, key, obj); err != nil {
return OperationResultNone, err
}
@@ -223,12 +222,124 @@ func CreateOrUpdate(ctx context.Context, c client.Client, obj runtime.Object, f
return OperationResultUpdated, nil
}
// mutate wraps a MutateFn and applies validation to its result
func mutate(f MutateFn, key client.ObjectKey, obj runtime.Object) error {
// CreateOrPatch creates or patches the given object in the Kubernetes
// cluster. The object's desired state must be reconciled with the before
// state inside the passed in callback MutateFn.
//
// The MutateFn is called regardless of creating or updating an object.
//
// It returns the executed operation and an error.
func CreateOrPatch(ctx context.Context, c client.Client, obj client.Object, f MutateFn) (OperationResult, error) {
key := client.ObjectKeyFromObject(obj)
if err := c.Get(ctx, key, obj); err != nil {
if !apierrors.IsNotFound(err) {
return OperationResultNone, err
}
if f != nil {
if err := mutate(f, key, obj); err != nil {
return OperationResultNone, err
}
}
if err := c.Create(ctx, obj); err != nil {
return OperationResultNone, err
}
return OperationResultCreated, nil
}
// Create patches for the object and its possible status.
objPatch := client.MergeFrom(obj.DeepCopyObject().(client.Object))
statusPatch := client.MergeFrom(obj.DeepCopyObject().(client.Object))
// Create a copy of the original object as well as converting that copy to
// unstructured data.
before, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())
if err != nil {
return OperationResultNone, err
}
// Attempt to extract the status from the resource for easier comparison later
beforeStatus, hasBeforeStatus, err := unstructured.NestedFieldCopy(before, "status")
if err != nil {
return OperationResultNone, err
}
// If the resource contains a status then remove it from the unstructured
// copy to avoid unnecessary patching later.
if hasBeforeStatus {
unstructured.RemoveNestedField(before, "status")
}
// Mutate the original object.
if f != nil {
if err := mutate(f, key, obj); err != nil {
return OperationResultNone, err
}
}
// Convert the resource to unstructured to compare against our before copy.
after, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return OperationResultNone, err
}
// Attempt to extract the status from the resource for easier comparison later
afterStatus, hasAfterStatus, err := unstructured.NestedFieldCopy(after, "status")
if err != nil {
return OperationResultNone, err
}
// If the resource contains a status then remove it from the unstructured
// copy to avoid unnecessary patching later.
if hasAfterStatus {
unstructured.RemoveNestedField(after, "status")
}
result := OperationResultNone
if !reflect.DeepEqual(before, after) {
// Only issue a Patch if the before and after resources (minus status) differ
if err := c.Patch(ctx, obj, objPatch); err != nil {
return result, err
}
result = OperationResultUpdated
}
if (hasBeforeStatus || hasAfterStatus) && !reflect.DeepEqual(beforeStatus, afterStatus) {
// Only issue a Status Patch if the resource has a status and the beforeStatus
// and afterStatus copies differ
if result == OperationResultUpdated {
// If Status was replaced by Patch before, set it to afterStatus
objectAfterPatch, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
if err != nil {
return result, err
}
if err = unstructured.SetNestedField(objectAfterPatch, afterStatus, "status"); err != nil {
return result, err
}
// If Status was replaced by Patch before, restore patched structure to the obj
if err = runtime.DefaultUnstructuredConverter.FromUnstructured(objectAfterPatch, obj); err != nil {
return result, err
}
}
if err := c.Status().Patch(ctx, obj, statusPatch); err != nil {
return result, err
}
if result == OperationResultUpdated {
result = OperationResultUpdatedStatus
} else {
result = OperationResultUpdatedStatusOnly
}
}
return result, nil
}
// mutate wraps a MutateFn and applies validation to its result.
func mutate(f MutateFn, key client.ObjectKey, obj client.Object) error {
if err := f(); err != nil {
return err
}
if newKey, err := client.ObjectKeyFromObject(obj); err != nil || key != newKey {
if newKey := client.ObjectKeyFromObject(obj); key != newKey {
return fmt.Errorf("MutateFn cannot mutate object name and/or object namespace")
}
return nil
@@ -238,7 +349,7 @@ func mutate(f MutateFn, key client.ObjectKey, obj runtime.Object) error {
type MutateFn func() error
// AddFinalizer accepts an Object and adds the provided finalizer if not present.
func AddFinalizer(o Object, finalizer string) {
func AddFinalizer(o client.Object, finalizer string) {
f := o.GetFinalizers()
for _, e := range f {
if e == finalizer {
@@ -248,21 +359,8 @@ func AddFinalizer(o Object, finalizer string) {
o.SetFinalizers(append(f, finalizer))
}
// AddFinalizerWithError tries to convert a runtime object to a metav1 object and add the provided finalizer.
// It returns an error if the provided object cannot provide an accessor.
//
// Deprecated: Use AddFinalizer instead. Check is performing on compile time.
func AddFinalizerWithError(o runtime.Object, finalizer string) error {
m, err := meta.Accessor(o)
if err != nil {
return err
}
AddFinalizer(m.(Object), finalizer)
return nil
}
// RemoveFinalizer accepts an Object and removes the provided finalizer if present.
func RemoveFinalizer(o Object, finalizer string) {
func RemoveFinalizer(o client.Object, finalizer string) {
f := o.GetFinalizers()
for i := 0; i < len(f); i++ {
if f[i] == finalizer {
@@ -273,21 +371,8 @@ func RemoveFinalizer(o Object, finalizer string) {
o.SetFinalizers(f)
}
// RemoveFinalizerWithError tries to convert a runtime object to a metav1 object and remove the provided finalizer.
// It returns an error if the provided object cannot provide an accessor.
//
// Deprecated: Use RemoveFinalizer instead. Check is performing on compile time.
func RemoveFinalizerWithError(o runtime.Object, finalizer string) error {
m, err := meta.Accessor(o)
if err != nil {
return err
}
RemoveFinalizer(m.(Object), finalizer)
return nil
}
// ContainsFinalizer checks an Object that the provided finalizer is present.
func ContainsFinalizer(o Object, finalizer string) bool {
func ContainsFinalizer(o client.Object, finalizer string) bool {
f := o.GetFinalizers()
for _, e := range f {
if e == finalizer {
@@ -299,7 +384,6 @@ func ContainsFinalizer(o Object, finalizer string) bool {
// Object allows functions to work indistinctly with any resource that
// implements both Object interfaces.
type Object interface {
metav1.Object
runtime.Object
}
//
// Deprecated: Use client.Object instead.
type Object = client.Object

View File

@@ -20,12 +20,16 @@ import (
"bufio"
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -34,18 +38,31 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/wait"
k8syaml "k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"k8s.io/client-go/util/retry"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/conversion"
"sigs.k8s.io/yaml"
)
// CRDInstallOptions are the options for installing CRDs
// CRDInstallOptions are the options for installing CRDs.
type CRDInstallOptions struct {
// Scheme is used to determine if conversion webhooks should be enabled
// for a particular CRD / object.
//
// Conversion webhooks are going to be enabled if an object in the scheme
// implements Hub and Spoke conversions.
//
// If nil, scheme.Scheme is used.
Scheme *runtime.Scheme
// Paths is a list of paths to the directories or files containing CRDs
Paths []string
// CRDs is a list of CRDs to install
CRDs []runtime.Object
CRDs []client.Object
// ErrorIfPathMissing will cause an error if a Path does not exist
ErrorIfPathMissing bool
@@ -60,34 +77,45 @@ type CRDInstallOptions struct {
// uninstalled when terminating the test environment.
// Defaults to false.
CleanUpAfterUse bool
// WebhookOptions contains the conversion webhook information to install
// on the CRDs. This field is usually inherited by the EnvTest options.
//
// If you're passing this field manually, you need to make sure that
// the CA information and host port is filled in properly.
WebhookOptions WebhookInstallOptions
}
const defaultPollInterval = 100 * time.Millisecond
const defaultMaxWait = 10 * time.Second
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]runtime.Object, error) {
// InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory.
func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]client.Object, error) {
defaultCRDOptions(&options)
// Read the CRD yamls into options.CRDs
if err := readCRDFiles(&options); err != nil {
return nil, fmt.Errorf("unable to read CRD files: %w", err)
}
if err := modifyConversionWebhooks(options.CRDs, options.Scheme, options.WebhookOptions); err != nil {
return nil, err
}
// Create the CRDs in the apiserver
if err := CreateCRDs(config, options.CRDs); err != nil {
return options.CRDs, err
return options.CRDs, fmt.Errorf("unable to create CRD instances: %w", err)
}
// Wait for the CRDs to appear as Resources in the apiserver
if err := WaitForCRDs(config, options.CRDs, options); err != nil {
return options.CRDs, err
return options.CRDs, fmt.Errorf("something went wrong waiting for CRDs to appear as API resources: %w", err)
}
return options.CRDs, nil
}
// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs
// readCRDFiles reads the directories of CRDs in options.Paths and adds the CRD structs to options.CRDs.
func readCRDFiles(options *CRDInstallOptions) error {
if len(options.Paths) > 0 {
crdList, err := renderCRDs(options)
@@ -100,8 +128,11 @@ func readCRDFiles(options *CRDInstallOptions) error {
return nil
}
// defaultCRDOptions sets the default values for CRDs
// defaultCRDOptions sets the default values for CRDs.
func defaultCRDOptions(o *CRDInstallOptions) {
if o.Scheme == nil {
o.Scheme = scheme.Scheme
}
if o.MaxTime == 0 {
o.MaxTime = defaultMaxWait
}
@@ -110,8 +141,8 @@ func defaultCRDOptions(o *CRDInstallOptions) {
}
}
// WaitForCRDs waits for the CRDs to appear in discovery
func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallOptions) error {
// WaitForCRDs waits for the CRDs to appear in discovery.
func WaitForCRDs(config *rest.Config, crds []client.Object, options CRDInstallOptions) error {
// Add each CRD to a map of GroupVersion to Resource
waitingFor := map[schema.GroupVersion]*sets.String{}
for _, crd := range runtimeCRDListToUnstructured(crds) {
@@ -128,14 +159,17 @@ func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallO
if err != nil {
return err
}
if crdVersion != "" {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
versions, _, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
versions, found, err := unstructured.NestedSlice(crd.Object, "spec", "versions")
if err != nil {
return err
}
// gvs should be added here only if single version is found. If multiple version is found we will add those version
// based on the version is served or not.
if crdVersion != "" && !found {
gvs = append(gvs, schema.GroupVersion{Group: crdGroup, Version: crdVersion})
}
for _, version := range versions {
versionMap, ok := version.(map[string]interface{})
if !ok {
@@ -170,7 +204,7 @@ func WaitForCRDs(config *rest.Config, crds []runtime.Object, options CRDInstallO
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not
// poller checks if all the resources have been found in discovery, and returns false if not.
type poller struct {
// config is used to get discovery
config *rest.Config
@@ -179,7 +213,7 @@ type poller struct {
waitingFor map[schema.GroupVersion]*sets.String
}
// poll checks if all the resources have been found in discovery, and returns false if not
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *poller) poll() (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
cs, err := clientset.NewForConfig(p.config)
@@ -199,7 +233,7 @@ func (p *poller) poll() (done bool, err error) {
// TODO: Maybe the controller-runtime client should be able to do this...
resourceList, err := cs.Discovery().ServerResourcesForGroupVersion(gv.Group + "/" + gv.Version)
if err != nil {
return false, nil
return false, nil //nolint:nilerr
}
// Remove each found resource from the resources set that we are waiting for
@@ -215,9 +249,8 @@ func (p *poller) poll() (done bool, err error) {
return allFound, nil
}
// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory
// UninstallCRDs uninstalls a collection of CRDs by reading the crd yaml files from a directory.
func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
// Read the CRD yamls into options.CRDs
if err := readCRDFiles(&options); err != nil {
return err
@@ -243,11 +276,11 @@ func UninstallCRDs(config *rest.Config, options CRDInstallOptions) error {
return nil
}
// CreateCRDs creates the CRDs
func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
// CreateCRDs creates the CRDs.
func CreateCRDs(config *rest.Config, crds []client.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
return fmt.Errorf("unable to create client: %w", err)
}
// Create each CRD
@@ -258,14 +291,19 @@ func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
switch {
case apierrors.IsNotFound(err):
if err := cs.Create(context.TODO(), crd); err != nil {
return err
return fmt.Errorf("unable to create CRD %q: %w", crd.GetName(), err)
}
case err != nil:
return err
return fmt.Errorf("unable to get CRD %q to check if it exists: %w", crd.GetName(), err)
default:
log.V(1).Info("CRD already exists, updating", "crd", crd.GetName())
crd.SetResourceVersion(existingCrd.GetResourceVersion())
if err := cs.Update(context.TODO(), crd); err != nil {
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
if err := cs.Get(context.TODO(), client.ObjectKey{Name: crd.GetName()}, existingCrd); err != nil {
return err
}
crd.SetResourceVersion(existingCrd.GetResourceVersion())
return cs.Update(context.TODO(), crd)
}); err != nil {
return err
}
}
@@ -274,7 +312,7 @@ func CreateCRDs(config *rest.Config, crds []runtime.Object) error {
}
// renderCRDs iterate through options.Paths and extract all CRD files.
func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
func renderCRDs(options *CRDInstallOptions) ([]client.Object, error) {
var (
err error
info os.FileInfo
@@ -301,10 +339,8 @@ func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
if !info.IsDir() {
filePath, files = filepath.Dir(path), []os.FileInfo{info}
} else {
if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, err
}
log.V(1).Info("reading CRDs from path", "path", path)
@@ -325,14 +361,200 @@ func renderCRDs(options *CRDInstallOptions) ([]runtime.Object, error) {
}
// Converting map to a list to return
var res []runtime.Object
res := []client.Object{}
for _, obj := range crds {
res = append(res, obj)
}
return res, nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs
// modifyConversionWebhooks takes all the registered CustomResourceDefinitions and applies modifications
// to conditionally enable webhooks if the type is registered within the scheme.
//
// The complexity of this function is high mostly due to all the edge cases that we need to handle:
// CRDv1beta1, CRDv1, and their unstructured counterpart.
//
// We should be able to simplify this code once we drop support for v1beta1 and standardize around the typed CRDv1 object.
func modifyConversionWebhooks(crds []client.Object, scheme *runtime.Scheme, webhookOptions WebhookInstallOptions) error { //nolint:gocyclo
if len(webhookOptions.LocalServingCAData) == 0 {
return nil
}
// Determine all registered convertible types.
convertibles := map[schema.GroupKind]struct{}{}
for gvk := range scheme.AllKnownTypes() {
obj, err := scheme.New(gvk)
if err != nil {
return err
}
if ok, err := conversion.IsConvertible(scheme, obj); ok && err == nil {
convertibles[gvk.GroupKind()] = struct{}{}
}
}
// generate host port.
hostPort, err := webhookOptions.generateHostPort()
if err != nil {
return err
}
url := pointer.StringPtr(fmt.Sprintf("https://%s/convert", hostPort))
for _, crd := range crds {
switch c := crd.(type) {
case *apiextensionsv1beta1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if c.Spec.PreserveUnknownFields == nil || *c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1beta1.WebhookConverter
c.Spec.Conversion.WebhookClientConfig.Service = nil
c.Spec.Conversion.WebhookClientConfig = &apiextensionsv1beta1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *apiextensionsv1.CustomResourceDefinition:
// Continue if we're preserving unknown fields.
if c.Spec.PreserveUnknownFields {
continue
}
// Continue if the GroupKind isn't registered as being convertible.
if _, ok := convertibles[schema.GroupKind{
Group: c.Spec.Group,
Kind: c.Spec.Names.Kind,
}]; !ok {
continue
}
c.Spec.Conversion.Strategy = apiextensionsv1.WebhookConverter
c.Spec.Conversion.Webhook.ClientConfig.Service = nil
c.Spec.Conversion.Webhook.ClientConfig = &apiextensionsv1.WebhookClientConfig{
Service: nil,
URL: url,
CABundle: webhookOptions.LocalServingCAData,
}
case *unstructured.Unstructured:
webhookClientConfig := map[string]interface{}{
"url": *url,
"caBundle": base64.StdEncoding.EncodeToString(webhookOptions.LocalServingCAData),
}
switch c.GroupVersionKind().Version {
case "v1beta1":
// Continue if we're preserving unknown fields.
//
// preserveUnknownFields defaults to true if `nil` in v1beta1.
if preserve, found, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve || !found {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1beta1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1beta1"},
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhookClientConfig"); err != nil {
return err
}
case "v1":
if preserve, _, err := unstructured.NestedBool(c.Object, "spec", "preserveUnknownFields"); preserve {
continue
} else if err != nil {
return err
}
// Continue if the GroupKind isn't registered as being convertible.
group, found, err := unstructured.NestedString(c.Object, "spec", "group")
if !found {
continue
} else if err != nil {
return err
}
kind, found, err := unstructured.NestedString(c.Object, "spec", "names", "kind")
if !found {
continue
} else if err != nil {
return err
}
if _, ok := convertibles[schema.GroupKind{
Group: group,
Kind: kind,
}]; !ok {
continue
}
// Set the strategy.
if err := unstructured.SetNestedField(
c.Object,
string(apiextensionsv1.WebhookConverter),
"spec", "conversion", "strategy"); err != nil {
return err
}
// Set the conversion review versions.
if err := unstructured.SetNestedStringSlice(
c.Object,
[]string{"v1", "v1beta1"},
"spec", "conversion", "webhook", "conversionReviewVersions"); err != nil {
return err
}
// Set the client configuration.
if err := unstructured.SetNestedMap(
c.Object,
webhookClientConfig,
"spec", "conversion", "webhook", "clientConfig"); err != nil {
return err
}
}
}
}
return nil
}
// readCRDs reads the CRDs from files and Unmarshals them into structs.
func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructured, error) {
var crds []*unstructured.Unstructured
@@ -378,9 +600,9 @@ func readCRDs(basePath string, files []os.FileInfo) ([]*unstructured.Unstructure
return crds, nil
}
// readDocuments reads documents from file
// readDocuments reads documents from file.
func readDocuments(fp string) ([][]byte, error) {
b, err := ioutil.ReadFile(fp)
b, err := ioutil.ReadFile(fp) //nolint:gosec
if err != nil {
return nil, err
}

View File

@@ -1,3 +1,19 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package envtest
import (
@@ -5,6 +21,7 @@ import (
apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var (
@@ -38,7 +55,7 @@ func mergePaths(s1, s2 []string) []string {
// mergeCRDs merges two CRD slices using their names.
// This function makes no guarantees about order of the merged slice.
func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
func mergeCRDs(s1, s2 []client.Object) []client.Object {
m := make(map[string]*unstructured.Unstructured)
for _, obj := range runtimeCRDListToUnstructured(s1) {
m[obj.GetName()] = obj
@@ -46,7 +63,7 @@ func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
for _, obj := range runtimeCRDListToUnstructured(s2) {
m[obj.GetName()] = obj
}
merged := make([]runtime.Object, len(m))
merged := make([]client.Object, len(m))
i := 0
for _, obj := range m {
merged[i] = obj
@@ -55,7 +72,7 @@ func mergeCRDs(s1, s2 []runtime.Object) []runtime.Object {
return merged
}
func runtimeCRDListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
func runtimeCRDListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
u := &unstructured.Unstructured{}

View File

@@ -33,21 +33,21 @@ var _ ginkgo.Reporter = NewlineReporter{}
// See issue https://github.com/jstemmer/go-junit-report/issues/31
type NewlineReporter struct{}
// SpecSuiteWillBegin implements ginkgo.Reporter
// SpecSuiteWillBegin implements ginkgo.Reporter.
func (NewlineReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summary *types.SuiteSummary) {
}
// BeforeSuiteDidRun implements ginkgo.Reporter
// BeforeSuiteDidRun implements ginkgo.Reporter.
func (NewlineReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {}
// AfterSuiteDidRun implements ginkgo.Reporter
// AfterSuiteDidRun implements ginkgo.Reporter.
func (NewlineReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {}
// SpecWillRun implements ginkgo.Reporter
// SpecWillRun implements ginkgo.Reporter.
func (NewlineReporter) SpecWillRun(specSummary *types.SpecSummary) {}
// SpecDidComplete implements ginkgo.Reporter
// SpecDidComplete implements ginkgo.Reporter.
func (NewlineReporter) SpecDidComplete(specSummary *types.SpecSummary) {}
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:".
func (NewlineReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) { fmt.Printf("\n") }

View File

@@ -73,35 +73,35 @@ func (pr *prowReporter) SpecSuiteWillBegin(config config.GinkgoConfigType, summa
}
}
// BeforeSuiteDidRun implements ginkgo.Reporter
// BeforeSuiteDidRun implements ginkgo.Reporter.
func (pr *prowReporter) BeforeSuiteDidRun(setupSummary *types.SetupSummary) {
if pr.junitReporter != nil {
pr.junitReporter.BeforeSuiteDidRun(setupSummary)
}
}
// AfterSuiteDidRun implements ginkgo.Reporter
// AfterSuiteDidRun implements ginkgo.Reporter.
func (pr *prowReporter) AfterSuiteDidRun(setupSummary *types.SetupSummary) {
if pr.junitReporter != nil {
pr.junitReporter.AfterSuiteDidRun(setupSummary)
}
}
// SpecWillRun implements ginkgo.Reporter
// SpecWillRun implements ginkgo.Reporter.
func (pr *prowReporter) SpecWillRun(specSummary *types.SpecSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecWillRun(specSummary)
}
}
// SpecDidComplete implements ginkgo.Reporter
// SpecDidComplete implements ginkgo.Reporter.
func (pr *prowReporter) SpecDidComplete(specSummary *types.SpecSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecDidComplete(specSummary)
}
}
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:"
// SpecSuiteDidEnd Prints a newline between "35 Passed | 0 Failed | 0 Pending | 0 Skipped" and "--- PASS:".
func (pr *prowReporter) SpecSuiteDidEnd(summary *types.SuiteSummary) {
if pr.junitReporter != nil {
pr.junitReporter.SpecSuiteDidEnd(summary)

View File

@@ -19,14 +19,16 @@ package envtest
import (
"fmt"
"os"
"path/filepath"
"strings"
"time"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/controlplane"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
logf "sigs.k8s.io/controller-runtime/pkg/internal/log"
)
@@ -46,46 +48,74 @@ It's possible to override some defaults, by setting the following environment va
*/
const (
envUseExistingCluster = "USE_EXISTING_CLUSTER"
envKubeAPIServerBin = "TEST_ASSET_KUBE_APISERVER"
envEtcdBin = "TEST_ASSET_ETCD"
envKubectlBin = "TEST_ASSET_KUBECTL"
envKubebuilderPath = "KUBEBUILDER_ASSETS"
envStartTimeout = "KUBEBUILDER_CONTROLPLANE_START_TIMEOUT"
envStopTimeout = "KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT"
envAttachOutput = "KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT"
defaultKubebuilderPath = "/usr/local/kubebuilder/bin"
StartTimeout = 60
StopTimeout = 60
envUseExistingCluster = "USE_EXISTING_CLUSTER"
envStartTimeout = "KUBEBUILDER_CONTROLPLANE_START_TIMEOUT"
envStopTimeout = "KUBEBUILDER_CONTROLPLANE_STOP_TIMEOUT"
envAttachOutput = "KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT"
StartTimeout = 60
StopTimeout = 60
defaultKubebuilderControlPlaneStartTimeout = 20 * time.Second
defaultKubebuilderControlPlaneStopTimeout = 20 * time.Second
)
// Default binary path for test framework
func defaultAssetPath(binary string) string {
assetPath := os.Getenv(envKubebuilderPath)
if assetPath == "" {
assetPath = defaultKubebuilderPath
}
return filepath.Join(assetPath, binary)
// internal types we expose as part of our public API.
type (
// ControlPlane is the re-exported ControlPlane type from the internal testing package.
ControlPlane = controlplane.ControlPlane
}
// APIServer is the re-exported APIServer from the internal testing package.
APIServer = controlplane.APIServer
// ControlPlane is the re-exported ControlPlane type from the internal integration package
type ControlPlane = integration.ControlPlane
// Etcd is the re-exported Etcd from the internal testing package.
Etcd = controlplane.Etcd
// APIServer is the re-exported APIServer type from the internal integration package
type APIServer = integration.APIServer
// User represents a Kubernetes user to provision for auth purposes.
User = controlplane.User
// Etcd is the re-exported Etcd type from the internal integration package
type Etcd = integration.Etcd
// AuthenticatedUser represets a Kubernetes user that's been provisioned.
AuthenticatedUser = controlplane.AuthenticatedUser
// ListenAddr indicates the address and port that the API server should listen on.
ListenAddr = process.ListenAddr
// SecureServing contains details describing how the API server should serve
// its secure endpoint.
SecureServing = controlplane.SecureServing
// Authn is an authentication method that can be used with the control plane to
// provision users.
Authn = controlplane.Authn
// Arguments allows configuring a process's flags.
Arguments = process.Arguments
// Arg is a single flag with one or more values.
Arg = process.Arg
)
var (
// EmptyArguments constructs a new set of flags with nothing set.
//
// This is mostly useful for testing helper methods -- you'll want to call
// Configure on the APIServer (or etcd) to configure their arguments.
EmptyArguments = process.EmptyArguments
)
// Environment creates a Kubernetes test environment that will start / stop the Kubernetes control plane and
// install extension APIs
// install extension APIs.
type Environment struct {
// ControlPlane is the ControlPlane including the apiserver and etcd
ControlPlane integration.ControlPlane
ControlPlane controlplane.ControlPlane
// Scheme is used to determine if conversion webhooks should be enabled
// for a particular CRD / object.
//
// Conversion webhooks are going to be enabled if an object in the scheme
// implements Hub and Spoke conversions.
//
// If nil, scheme.Scheme is used.
Scheme *runtime.Scheme
// Config can be used to talk to the apiserver. It's automatically
// populated if not set using the standard controller-runtime config
@@ -106,14 +136,18 @@ type Environment struct {
// CRDs is a list of CRDs to install.
// If both this field and CRDs field in CRDInstallOptions are specified, the
// values are merged.
CRDs []runtime.Object
CRDs []client.Object
// CRDDirectoryPaths is a list of paths containing CRD yaml or json configs.
// If both this field and Paths field in CRDInstallOptions are specified, the
// values are merged.
CRDDirectoryPaths []string
// UseExisting indicates that this environments should use an
// BinaryAssetsDirectory is the path where the binaries required for the envtest are
// located in the local environment. This field can be overridden by setting KUBEBUILDER_ASSETS.
BinaryAssetsDirectory string
// UseExistingCluster indicates that this environments should use an
// existing kubeconfig, instead of trying to stand up a new control plane.
// This is useful in cases that need aggregated API servers and the like.
UseExistingCluster *bool
@@ -129,6 +163,8 @@ type Environment struct {
ControlPlaneStopTimeout time.Duration
// KubeAPIServerFlags is the set of flags passed while starting the api server.
//
// Deprecated: use ControlPlane.GetAPIServer().Configure() instead.
KubeAPIServerFlags []string
// AttachControlPlaneOutput indicates if control plane output will be attached to os.Stdout and os.Stderr.
@@ -146,38 +182,19 @@ func (te *Environment) Stop() error {
return err
}
}
if err := te.WebhookInstallOptions.Cleanup(); err != nil {
return err
}
if te.useExistingCluster() {
return nil
}
err := te.WebhookInstallOptions.Cleanup()
if err != nil {
return err
}
return te.ControlPlane.Stop()
}
// getAPIServerFlags returns flags to be used with the Kubernetes API server.
// it returns empty slice for api server defined defaults to be applied if no args specified
func (te Environment) getAPIServerFlags() []string {
// Set default API server flags if not set.
if len(te.KubeAPIServerFlags) == 0 {
return []string{}
}
// Check KubeAPIServerFlags contains service-cluster-ip-range, if not, set default value to service-cluster-ip-range
containServiceClusterIPRange := false
for _, flag := range te.KubeAPIServerFlags {
if strings.Contains(flag, "service-cluster-ip-range") {
containServiceClusterIPRange = true
break
}
}
if !containServiceClusterIPRange {
te.KubeAPIServerFlags = append(te.KubeAPIServerFlags, "--service-cluster-ip-range=10.0.0.0/24")
}
return te.KubeAPIServerFlags
}
// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on
// Start starts a local Kubernetes server and updates te.ApiserverPort with the port it is listening on.
func (te *Environment) Start() (*rest.Config, error) {
if te.useExistingCluster() {
log.V(1).Info("using existing cluster")
@@ -189,25 +206,36 @@ func (te *Environment) Start() (*rest.Config, error) {
var err error
te.Config, err = config.GetConfig()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to get configuration for existing cluster: %w", err)
}
}
} else {
if te.ControlPlane.APIServer == nil {
te.ControlPlane.APIServer = &integration.APIServer{Args: te.getAPIServerFlags()}
apiServer := te.ControlPlane.GetAPIServer()
if len(apiServer.Args) == 0 { //nolint:staticcheck
// pass these through separately from above in case something like
// AddUser defaults APIServer.
//
// TODO(directxman12): if/when we feel like making a bigger
// breaking change here, just make APIServer and Etcd non-pointers
// in ControlPlane.
// NB(directxman12): we still pass these in so that things work if the
// user manually specifies them, but in most cases we expect them to
// be nil so that we use the new .Configure() logic.
apiServer.Args = te.KubeAPIServerFlags //nolint:staticcheck
}
if te.ControlPlane.Etcd == nil {
te.ControlPlane.Etcd = &integration.Etcd{}
te.ControlPlane.Etcd = &controlplane.Etcd{}
}
if os.Getenv(envAttachOutput) == "true" {
te.AttachControlPlaneOutput = true
}
if te.ControlPlane.APIServer.Out == nil && te.AttachControlPlaneOutput {
te.ControlPlane.APIServer.Out = os.Stdout
if apiServer.Out == nil && te.AttachControlPlaneOutput {
apiServer.Out = os.Stdout
}
if te.ControlPlane.APIServer.Err == nil && te.AttachControlPlaneOutput {
te.ControlPlane.APIServer.Err = os.Stderr
if apiServer.Err == nil && te.AttachControlPlaneOutput {
apiServer.Err = os.Stderr
}
if te.ControlPlane.Etcd.Out == nil && te.AttachControlPlaneOutput {
te.ControlPlane.Etcd.Out = os.Stdout
@@ -216,55 +244,78 @@ func (te *Environment) Start() (*rest.Config, error) {
te.ControlPlane.Etcd.Err = os.Stderr
}
if os.Getenv(envKubeAPIServerBin) == "" {
te.ControlPlane.APIServer.Path = defaultAssetPath("kube-apiserver")
}
if os.Getenv(envEtcdBin) == "" {
te.ControlPlane.Etcd.Path = defaultAssetPath("etcd")
}
if os.Getenv(envKubectlBin) == "" {
// we can't just set the path manually (it's behind a function), so set the environment variable instead
if err := os.Setenv(envKubectlBin, defaultAssetPath("kubectl")); err != nil {
return nil, err
}
}
apiServer.Path = process.BinPathFinder("kube-apiserver", te.BinaryAssetsDirectory)
te.ControlPlane.Etcd.Path = process.BinPathFinder("etcd", te.BinaryAssetsDirectory)
te.ControlPlane.KubectlPath = process.BinPathFinder("kubectl", te.BinaryAssetsDirectory)
if err := te.defaultTimeouts(); err != nil {
return nil, fmt.Errorf("failed to default controlplane timeouts: %w", err)
}
te.ControlPlane.Etcd.StartTimeout = te.ControlPlaneStartTimeout
te.ControlPlane.Etcd.StopTimeout = te.ControlPlaneStopTimeout
te.ControlPlane.APIServer.StartTimeout = te.ControlPlaneStartTimeout
te.ControlPlane.APIServer.StopTimeout = te.ControlPlaneStopTimeout
apiServer.StartTimeout = te.ControlPlaneStartTimeout
apiServer.StopTimeout = te.ControlPlaneStopTimeout
log.V(1).Info("starting control plane", "api server flags", te.ControlPlane.APIServer.Args)
log.V(1).Info("starting control plane")
if err := te.startControlPlane(); err != nil {
return nil, err
return nil, fmt.Errorf("unable to start control plane itself: %w", err)
}
// Create the *rest.Config for creating new clients
te.Config = &rest.Config{
Host: te.ControlPlane.APIURL().Host,
baseConfig := &rest.Config{
// gotta go fast during tests -- we don't really care about overwhelming our test API server
QPS: 1000.0,
Burst: 2000.0,
}
adminInfo := User{Name: "admin", Groups: []string{"system:masters"}}
adminUser, err := te.ControlPlane.AddUser(adminInfo, baseConfig)
if err != nil {
return te.Config, fmt.Errorf("unable to provision admin user: %w", err)
}
te.Config = adminUser.Config()
}
// Set the default scheme if nil.
if te.Scheme == nil {
te.Scheme = scheme.Scheme
}
// Call PrepWithoutInstalling to setup certificates first
// and have them available to patch CRD conversion webhook as well.
if err := te.WebhookInstallOptions.PrepWithoutInstalling(); err != nil {
return nil, err
}
log.V(1).Info("installing CRDs")
te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs)
te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths)
te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing
te.CRDInstallOptions.WebhookOptions = te.WebhookInstallOptions
crds, err := InstallCRDs(te.Config, te.CRDInstallOptions)
if err != nil {
return te.Config, err
return te.Config, fmt.Errorf("unable to install CRDs onto control plane: %w", err)
}
te.CRDs = crds
log.V(1).Info("installing webhooks")
err = te.WebhookInstallOptions.Install(te.Config)
if err := te.WebhookInstallOptions.Install(te.Config); err != nil {
return nil, fmt.Errorf("unable to install webhooks onto control plane: %w", err)
}
return te.Config, nil
}
return te.Config, err
// AddUser provisions a new user for connecting to this Environment. The user will
// have the specified name & belong to the specified groups.
//
// If you specify a "base" config, the returned REST Config will contain those
// settings as well as any required by the authentication method. You can use
// this to easily specify options like QPS.
//
// This is effectively a convinience alias for ControlPlane.AddUser -- see that
// for more low-level details.
func (te *Environment) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) {
return te.ControlPlane.AddUser(user, baseConfig)
}
func (te *Environment) startControlPlane() error {
@@ -319,4 +370,6 @@ func (te *Environment) useExistingCluster() bool {
// DefaultKubeAPIServerFlags exposes the default args for the APIServer so that
// you can use those to append your own additional arguments.
var DefaultKubeAPIServerFlags = integration.APIServerDefaultArgs
//
// Deprecated: use APIServer.Configure() instead.
var DefaultKubeAPIServerFlags = controlplane.APIServerDefaultArgs //nolint:staticcheck

View File

@@ -23,7 +23,6 @@ import (
"path/filepath"
"time"
"k8s.io/apimachinery/pkg/api/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -33,21 +32,21 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/yaml"
)
// WebhookInstallOptions are the options for installing mutating or validating webhooks
// WebhookInstallOptions are the options for installing mutating or validating webhooks.
type WebhookInstallOptions struct {
// Paths is a list of paths to the directories containing the mutating or validating webhooks yaml or json configs.
DirectoryPaths []string
// Paths is a list of paths to the directories or files containing the mutating or validating webhooks yaml or json configs.
Paths []string
// MutatingWebhooks is a list of MutatingWebhookConfigurations to install
MutatingWebhooks []runtime.Object
MutatingWebhooks []client.Object
// ValidatingWebhooks is a list of ValidatingWebhookConfigurations to install
ValidatingWebhooks []runtime.Object
ValidatingWebhooks []client.Object
// IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true
IgnoreErrorIfPathMissing bool
@@ -67,6 +66,9 @@ type WebhookInstallOptions struct {
// CAData is the CA that can be used to trust the serving certificates in LocalServingCertDir.
LocalServingCAData []byte
// LocalServingHostExternalName is the hostname to use to reach the webhook server.
LocalServingHostExternalName string
// MaxTime is the max time to wait
MaxTime time.Duration
@@ -76,8 +78,11 @@ type WebhookInstallOptions struct {
// ModifyWebhookDefinitions modifies webhook definitions by:
// - applying CABundle based on the provided tinyca
// - if webhook client config uses service spec, it's removed and replaced with direct url
func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error {
// - if webhook client config uses service spec, it's removed and replaced with direct url.
func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error {
caData := o.LocalServingCAData
// generate host port.
hostPort, err := o.generateHostPort()
if err != nil {
return err
@@ -137,13 +142,19 @@ func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort strin
}
func (o *WebhookInstallOptions) generateHostPort() (string, error) {
port, host, err := addr.Suggest()
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
if o.LocalServingPort == 0 {
port, host, err := addr.Suggest(o.LocalServingHost)
if err != nil {
return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err)
}
o.LocalServingPort = port
o.LocalServingHost = host
}
o.LocalServingPort = port
o.LocalServingHost = host
return net.JoinHostPort(host, fmt.Sprintf("%d", port)), nil
host := o.LocalServingHostExternalName
if host == "" {
host = o.LocalServingHost
}
return net.JoinHostPort(host, fmt.Sprintf("%d", o.LocalServingPort)), nil
}
// PrepWithoutInstalling does the setup parts of Install (populating host-port,
@@ -152,40 +163,33 @@ func (o *WebhookInstallOptions) generateHostPort() (string, error) {
// controller-runtime, where we need a random host-port & caData for webhook
// tests, but may be useful in similar scenarios.
func (o *WebhookInstallOptions) PrepWithoutInstalling() error {
hookCA, err := o.setupCA()
if err != nil {
return err
}
if err := parseWebhookDirs(o); err != nil {
if err := o.setupCA(); err != nil {
return err
}
err = o.ModifyWebhookDefinitions(hookCA)
if err != nil {
if err := parseWebhook(o); err != nil {
return err
}
return nil
return o.ModifyWebhookDefinitions()
}
// Install installs specified webhooks to the API server
// Install installs specified webhooks to the API server.
func (o *WebhookInstallOptions) Install(config *rest.Config) error {
if err := o.PrepWithoutInstalling(); err != nil {
return err
if len(o.LocalServingCAData) == 0 {
if err := o.PrepWithoutInstalling(); err != nil {
return err
}
}
if err := createWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks); err != nil {
return err
}
if err := WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o); err != nil {
return err
}
return nil
return WaitForWebhooks(config, o.MutatingWebhooks, o.ValidatingWebhooks, *o)
}
// Cleanup cleans up cert directories
// Cleanup cleans up cert directories.
func (o *WebhookInstallOptions) Cleanup() error {
if o.LocalServingCertDir != "" {
return os.RemoveAll(o.LocalServingCertDir)
@@ -193,12 +197,11 @@ func (o *WebhookInstallOptions) Cleanup() error {
return nil
}
// WaitForWebhooks waits for the Webhooks to be available through API server
// WaitForWebhooks waits for the Webhooks to be available through API server.
func WaitForWebhooks(config *rest.Config,
mutatingWebhooks []runtime.Object,
validatingWebhooks []runtime.Object,
mutatingWebhooks []client.Object,
validatingWebhooks []client.Object,
options WebhookInstallOptions) error {
waitingFor := map[schema.GroupVersionKind]*sets.String{}
for _, hook := range runtimeListToUnstructured(append(validatingWebhooks, mutatingWebhooks...)) {
@@ -213,7 +216,7 @@ func WaitForWebhooks(config *rest.Config,
return wait.PollImmediate(options.PollInterval, options.MaxTime, p.poll)
}
// poller checks if all the resources have been found in discovery, and returns false if not
// poller checks if all the resources have been found in discovery, and returns false if not.
type webhookPoller struct {
// config is used to get discovery
config *rest.Config
@@ -222,7 +225,7 @@ type webhookPoller struct {
waitingFor map[schema.GroupVersionKind]*sets.String
}
// poll checks if all the resources have been found in discovery, and returns false if not
// poll checks if all the resources have been found in discovery, and returns false if not.
func (p *webhookPoller) poll() (done bool, err error) {
// Create a new clientset to avoid any client caching of discovery
c, err := client.New(p.config, client.Options{})
@@ -248,7 +251,7 @@ func (p *webhookPoller) poll() (done bool, err error) {
names.Delete(name)
}
if errors.IsNotFound(err) {
if apierrors.IsNotFound(err) {
allFound = false
}
if err != nil {
@@ -259,41 +262,42 @@ func (p *webhookPoller) poll() (done bool, err error) {
return allFound, nil
}
// setupCA creates CA for testing and writes them to disk
func (o *WebhookInstallOptions) setupCA() ([]byte, error) {
hookCA, err := integration.NewTinyCA()
// setupCA creates CA for testing and writes them to disk.
func (o *WebhookInstallOptions) setupCA() error {
hookCA, err := certs.NewTinyCA()
if err != nil {
return nil, fmt.Errorf("unable to set up webhook CA: %v", err)
return fmt.Errorf("unable to set up webhook CA: %v", err)
}
hookCert, err := hookCA.NewServingCert()
names := []string{"localhost", o.LocalServingHost, o.LocalServingHostExternalName}
hookCert, err := hookCA.NewServingCert(names...)
if err != nil {
return nil, fmt.Errorf("unable to set up webhook serving certs: %v", err)
return fmt.Errorf("unable to set up webhook serving certs: %v", err)
}
localServingCertsDir, err := ioutil.TempDir("", "envtest-serving-certs-")
o.LocalServingCertDir = localServingCertsDir
if err != nil {
return nil, fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
return fmt.Errorf("unable to create directory for webhook serving certs: %v", err)
}
certData, keyData, err := hookCert.AsBytes()
if err != nil {
return nil, fmt.Errorf("unable to marshal webhook serving certs: %v", err)
return fmt.Errorf("unable to marshal webhook serving certs: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.crt"), certData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving cert to disk: %v", err)
}
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil {
return nil, fmt.Errorf("unable to write webhook serving key to disk: %v", err)
if err := ioutil.WriteFile(filepath.Join(localServingCertsDir, "tls.key"), keyData, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to write webhook serving key to disk: %v", err)
}
o.LocalServingCAData = certData
return certData, nil
return err
}
func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []runtime.Object) error {
func createWebhooks(config *rest.Config, mutHooks []client.Object, valHooks []client.Object) error {
cs, err := client.New(config, client.Options{})
if err != nil {
return err
@@ -315,7 +319,7 @@ func createWebhooks(config *rest.Config, mutHooks []runtime.Object, valHooks []r
return nil
}
// ensureCreated creates or update object if already exists in the cluster
// ensureCreated creates or update object if already exists in the cluster.
func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
existing := obj.DeepCopy()
err := cs.Get(context.Background(), client.ObjectKey{Name: obj.GetName()}, existing)
@@ -336,10 +340,10 @@ func ensureCreated(cs client.Client, obj *unstructured.Unstructured) error {
return nil
}
// parseWebhookDirs reads the directories of Webhooks in options.DirectoryPaths and adds the Webhook structs to options
func parseWebhookDirs(options *WebhookInstallOptions) error {
if len(options.DirectoryPaths) > 0 {
for _, path := range options.DirectoryPaths {
// parseWebhook reads the directories or files of Webhooks in options.Paths and adds the Webhook structs to options.
func parseWebhook(options *WebhookInstallOptions) error {
if len(options.Paths) > 0 {
for _, path := range options.Paths {
_, err := os.Stat(path)
if options.IgnoreErrorIfPathMissing && os.IsNotExist(err) {
continue // skip this path
@@ -359,21 +363,27 @@ func parseWebhookDirs(options *WebhookInstallOptions) error {
}
// readWebhooks reads the Webhooks from files and Unmarshals them into structs
// returns slice of mutating and validating webhook configurations
func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
// returns slice of mutating and validating webhook configurations.
func readWebhooks(path string) ([]client.Object, []client.Object, error) {
// Get the webhook files
var files []os.FileInfo
var err error
log.V(1).Info("reading Webhooks from path", "path", path)
if files, err = ioutil.ReadDir(path); err != nil {
info, err := os.Stat(path)
if err != nil {
return nil, nil, err
}
if !info.IsDir() {
path, files = filepath.Dir(path), []os.FileInfo{info}
} else if files, err = ioutil.ReadDir(path); err != nil {
return nil, nil, err
}
// file extensions that may contain Webhooks
resourceExtensions := sets.NewString(".json", ".yaml", ".yml")
var mutHooks []runtime.Object
var valHooks []runtime.Object
var mutHooks []client.Object
var valHooks []client.Object
for _, file := range files {
// Only parse allowlisted file types
if !resourceExtensions.Has(filepath.Ext(file.Name())) {
@@ -425,7 +435,7 @@ func readWebhooks(path string) ([]runtime.Object, []runtime.Object, error) {
return mutHooks, valHooks, nil
}
func runtimeListToUnstructured(l []runtime.Object) []*unstructured.Unstructured {
func runtimeListToUnstructured(l []client.Object) []*unstructured.Unstructured {
res := []*unstructured.Unstructured{}
for _, obj := range l {
m, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj.DeepCopyObject())

View File

@@ -16,45 +16,30 @@ limitations under the License.
package event
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
)
import "sigs.k8s.io/controller-runtime/pkg/client"
// CreateEvent is an event where a Kubernetes object was created. CreateEvent should be generated
// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler.
type CreateEvent struct {
// Meta is the ObjectMeta of the Kubernetes Type that was created
Meta metav1.Object
// Object is the object from the event
Object runtime.Object
Object client.Object
}
// UpdateEvent is an event where a Kubernetes object was updated. UpdateEvent should be generated
// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler.
type UpdateEvent struct {
// MetaOld is the ObjectMeta of the Kubernetes Type that was updated (before the update)
MetaOld metav1.Object
// ObjectOld is the object from the event
ObjectOld runtime.Object
// MetaNew is the ObjectMeta of the Kubernetes Type that was updated (after the update)
MetaNew metav1.Object
ObjectOld client.Object
// ObjectNew is the object from the event
ObjectNew runtime.Object
ObjectNew client.Object
}
// DeleteEvent is an event where a Kubernetes object was deleted. DeleteEvent should be generated
// by a source.Source and transformed into a reconcile.Request by an handler.EventHandler.
type DeleteEvent struct {
// Meta is the ObjectMeta of the Kubernetes Type that was deleted
Meta metav1.Object
// Object is the object from the event
Object runtime.Object
Object client.Object
// DeleteStateUnknown is true if the Delete event was missed but we identified the object
// as having been deleted.
@@ -65,9 +50,6 @@ type DeleteEvent struct {
// GenericEvent should be generated by a source.Source and transformed into a reconcile.Request by an
// handler.EventHandler.
type GenericEvent struct {
// Meta is the ObjectMeta of a Kubernetes Type this event is for
Meta metav1.Object
// Object is the object from the event
Object runtime.Object
Object client.Object
}

View File

@@ -26,6 +26,8 @@ import (
var enqueueLog = logf.RuntimeLog.WithName("eventhandler").WithName("EnqueueRequestForObject")
type empty struct{}
var _ EventHandler = &EnqueueRequestForObject{}
// EnqueueRequestForObject enqueues a Request containing the Name and Namespace of the object that is the source of the Event.
@@ -33,59 +35,56 @@ var _ EventHandler = &EnqueueRequestForObject{}
// Controllers that have associated Resources (e.g. CRDs) to reconcile the associated Resource.
type EnqueueRequestForObject struct{}
// Create implements EventHandler
// Create implements EventHandler.
func (e *EnqueueRequestForObject) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
if evt.Meta == nil {
if evt.Object == nil {
enqueueLog.Error(nil, "CreateEvent received with no metadata", "event", evt)
return
}
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Name: evt.Meta.GetName(),
Namespace: evt.Meta.GetNamespace(),
Name: evt.Object.GetName(),
Namespace: evt.Object.GetNamespace(),
}})
}
// Update implements EventHandler
// Update implements EventHandler.
func (e *EnqueueRequestForObject) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
if evt.MetaOld != nil {
switch {
case evt.ObjectNew != nil:
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Name: evt.MetaOld.GetName(),
Namespace: evt.MetaOld.GetNamespace(),
Name: evt.ObjectNew.GetName(),
Namespace: evt.ObjectNew.GetNamespace(),
}})
} else {
enqueueLog.Error(nil, "UpdateEvent received with no old metadata", "event", evt)
}
if evt.MetaNew != nil {
case evt.ObjectOld != nil:
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Name: evt.MetaNew.GetName(),
Namespace: evt.MetaNew.GetNamespace(),
Name: evt.ObjectOld.GetName(),
Namespace: evt.ObjectOld.GetNamespace(),
}})
} else {
enqueueLog.Error(nil, "UpdateEvent received with no new metadata", "event", evt)
default:
enqueueLog.Error(nil, "UpdateEvent received with no metadata", "event", evt)
}
}
// Delete implements EventHandler
// Delete implements EventHandler.
func (e *EnqueueRequestForObject) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
if evt.Meta == nil {
if evt.Object == nil {
enqueueLog.Error(nil, "DeleteEvent received with no metadata", "event", evt)
return
}
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Name: evt.Meta.GetName(),
Namespace: evt.Meta.GetNamespace(),
Name: evt.Object.GetName(),
Namespace: evt.Object.GetNamespace(),
}})
}
// Generic implements EventHandler
// Generic implements EventHandler.
func (e *EnqueueRequestForObject) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
if evt.Meta == nil {
if evt.Object == nil {
enqueueLog.Error(nil, "GenericEvent received with no metadata", "event", evt)
return
}
q.Add(reconcile.Request{NamespacedName: types.NamespacedName{
Name: evt.Meta.GetName(),
Namespace: evt.Meta.GetNamespace(),
Name: evt.Object.GetName(),
Namespace: evt.Object.GetNamespace(),
}})
}

View File

@@ -17,15 +17,16 @@ limitations under the License.
package handler
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
)
var _ EventHandler = &EnqueueRequestsFromMapFunc{}
// MapFunc is the signature required for enqueueing requests from a generic function.
// This type is usually used with EnqueueRequestsFromMapFunc when registering an event handler.
type MapFunc func(client.Object) []reconcile.Request
// EnqueueRequestsFromMapFunc enqueues Requests by running a transformation function that outputs a collection
// of reconcile.Requests on each Event. The reconcile.Requests may be for an arbitrary set of objects
@@ -37,69 +38,60 @@ var _ EventHandler = &EnqueueRequestsFromMapFunc{}
//
// For UpdateEvents which contain both a new and old object, the transformation function is run on both
// objects and both sets of Requests are enqueue.
type EnqueueRequestsFromMapFunc struct {
func EnqueueRequestsFromMapFunc(fn MapFunc) EventHandler {
return &enqueueRequestsFromMapFunc{
toRequests: fn,
}
}
var _ EventHandler = &enqueueRequestsFromMapFunc{}
type enqueueRequestsFromMapFunc struct {
// Mapper transforms the argument into a slice of keys to be reconciled
ToRequests Mapper
toRequests MapFunc
}
// Create implements EventHandler
func (e *EnqueueRequestsFromMapFunc) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
e.mapAndEnqueue(q, MapObject{Meta: evt.Meta, Object: evt.Object})
// Create implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
}
// Update implements EventHandler
func (e *EnqueueRequestsFromMapFunc) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
e.mapAndEnqueue(q, MapObject{Meta: evt.MetaOld, Object: evt.ObjectOld})
e.mapAndEnqueue(q, MapObject{Meta: evt.MetaNew, Object: evt.ObjectNew})
// Update implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.ObjectOld, reqs)
e.mapAndEnqueue(q, evt.ObjectNew, reqs)
}
// Delete implements EventHandler
func (e *EnqueueRequestsFromMapFunc) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
e.mapAndEnqueue(q, MapObject{Meta: evt.Meta, Object: evt.Object})
// Delete implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
}
// Generic implements EventHandler
func (e *EnqueueRequestsFromMapFunc) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
e.mapAndEnqueue(q, MapObject{Meta: evt.Meta, Object: evt.Object})
// Generic implements EventHandler.
func (e *enqueueRequestsFromMapFunc) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
reqs := map[reconcile.Request]empty{}
e.mapAndEnqueue(q, evt.Object, reqs)
}
func (e *EnqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInterface, object MapObject) {
for _, req := range e.ToRequests.Map(object) {
q.Add(req)
func (e *enqueueRequestsFromMapFunc) mapAndEnqueue(q workqueue.RateLimitingInterface, object client.Object, reqs map[reconcile.Request]empty) {
for _, req := range e.toRequests(object) {
_, ok := reqs[req]
if !ok {
q.Add(req)
reqs[req] = empty{}
}
}
}
// EnqueueRequestsFromMapFunc can inject fields into the mapper.
// InjectFunc implements inject.Injector.
func (e *EnqueueRequestsFromMapFunc) InjectFunc(f inject.Func) error {
func (e *enqueueRequestsFromMapFunc) InjectFunc(f inject.Func) error {
if f == nil {
return nil
}
return f(e.ToRequests)
}
// Mapper maps an object to a collection of keys to be enqueued
type Mapper interface {
// Map maps an object
Map(MapObject) []reconcile.Request
}
// MapObject contains information from an event to be transformed into a Request.
type MapObject struct {
// Meta is the meta data for an object from an event.
Meta metav1.Object
// Object is the object from an event.
Object runtime.Object
}
var _ Mapper = ToRequestsFunc(nil)
// ToRequestsFunc implements Mapper using a function.
type ToRequestsFunc func(MapObject) []reconcile.Request
// Map implements Mapper
func (m ToRequestsFunc) Map(i MapObject) []reconcile.Request {
return m(i)
return f(e.toRequests)
}

View File

@@ -57,33 +57,39 @@ type EnqueueRequestForOwner struct {
mapper meta.RESTMapper
}
// Create implements EventHandler
// Create implements EventHandler.
func (e *EnqueueRequestForOwner) Create(evt event.CreateEvent, q workqueue.RateLimitingInterface) {
for _, req := range e.getOwnerReconcileRequest(evt.Meta) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
q.Add(req)
}
}
// Update implements EventHandler
// Update implements EventHandler.
func (e *EnqueueRequestForOwner) Update(evt event.UpdateEvent, q workqueue.RateLimitingInterface) {
for _, req := range e.getOwnerReconcileRequest(evt.MetaOld) {
q.Add(req)
}
for _, req := range e.getOwnerReconcileRequest(evt.MetaNew) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.ObjectOld, reqs)
e.getOwnerReconcileRequest(evt.ObjectNew, reqs)
for req := range reqs {
q.Add(req)
}
}
// Delete implements EventHandler
// Delete implements EventHandler.
func (e *EnqueueRequestForOwner) Delete(evt event.DeleteEvent, q workqueue.RateLimitingInterface) {
for _, req := range e.getOwnerReconcileRequest(evt.Meta) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
q.Add(req)
}
}
// Generic implements EventHandler
// Generic implements EventHandler.
func (e *EnqueueRequestForOwner) Generic(evt event.GenericEvent, q workqueue.RateLimitingInterface) {
for _, req := range e.getOwnerReconcileRequest(evt.Meta) {
reqs := map[reconcile.Request]empty{}
e.getOwnerReconcileRequest(evt.Object, reqs)
for req := range reqs {
q.Add(req)
}
}
@@ -99,29 +105,27 @@ func (e *EnqueueRequestForOwner) parseOwnerTypeGroupKind(scheme *runtime.Scheme)
}
// Expect only 1 kind. If there is more than one kind this is probably an edge case such as ListOptions.
if len(kinds) != 1 {
err := fmt.Errorf("Expected exactly 1 kind for OwnerType %T, but found %s kinds", e.OwnerType, kinds)
log.Error(nil, "Expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType), "kinds", kinds)
err := fmt.Errorf("expected exactly 1 kind for OwnerType %T, but found %s kinds", e.OwnerType, kinds)
log.Error(nil, "expected exactly 1 kind for OwnerType", "owner type", fmt.Sprintf("%T", e.OwnerType), "kinds", kinds)
return err
}
// Cache the Group and Kind for the OwnerType
e.groupKind = schema.GroupKind{Group: kinds[0].Group, Kind: kinds[0].Kind}
return nil
}
// getOwnerReconcileRequest looks at object and returns a slice of reconcile.Request to reconcile
// getOwnerReconcileRequest looks at object and builds a map of reconcile.Request to reconcile
// owners of object that match e.OwnerType.
func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object) []reconcile.Request {
func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object, result map[reconcile.Request]empty) {
// Iterate through the OwnerReferences looking for a match on Group and Kind against what was requested
// by the user
var result []reconcile.Request
for _, ref := range e.getOwnersReferences(object) {
// Parse the Group out of the OwnerReference to compare it to what was parsed out of the requested OwnerType
refGV, err := schema.ParseGroupVersion(ref.APIVersion)
if err != nil {
log.Error(err, "Could not parse OwnerReference APIVersion",
"api version", ref.APIVersion)
return nil
return
}
// Compare the OwnerReference Group and Kind against the OwnerType Group and Kind specified by the user.
@@ -138,23 +142,20 @@ func (e *EnqueueRequestForOwner) getOwnerReconcileRequest(object metav1.Object)
mapping, err := e.mapper.RESTMapping(e.groupKind, refGV.Version)
if err != nil {
log.Error(err, "Could not retrieve rest mapping", "kind", e.groupKind)
return nil
return
}
if mapping.Scope.Name() != meta.RESTScopeNameRoot {
request.Namespace = object.GetNamespace()
}
result = append(result, request)
result[request] = empty{}
}
}
// Return the matches
return result
}
// getOwnersReferences returns the OwnerReferences for an object as specified by the EnqueueRequestForOwner
// - if IsController is true: only take the Controller OwnerReference (if found)
// - if IsController is false: take all OwnerReferences
// - if IsController is false: take all OwnerReferences.
func (e *EnqueueRequestForOwner) getOwnersReferences(object metav1.Object) []metav1.OwnerReference {
if object == nil {
return nil

View File

@@ -75,28 +75,28 @@ type Funcs struct {
GenericFunc func(event.GenericEvent, workqueue.RateLimitingInterface)
}
// Create implements EventHandler
// Create implements EventHandler.
func (h Funcs) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) {
if h.CreateFunc != nil {
h.CreateFunc(e, q)
}
}
// Delete implements EventHandler
// Delete implements EventHandler.
func (h Funcs) Delete(e event.DeleteEvent, q workqueue.RateLimitingInterface) {
if h.DeleteFunc != nil {
h.DeleteFunc(e, q)
}
}
// Update implements EventHandler
// Update implements EventHandler.
func (h Funcs) Update(e event.UpdateEvent, q workqueue.RateLimitingInterface) {
if h.UpdateFunc != nil {
h.UpdateFunc(e, q)
}
}
// Generic implements EventHandler
// Generic implements EventHandler.
func (h Funcs) Generic(e event.GenericEvent, q workqueue.RateLimitingInterface) {
if h.GenericFunc != nil {
h.GenericFunc(e, q)

View File

@@ -35,7 +35,7 @@ type Handler struct {
Checks map[string]Checker
}
// checkStatus holds the output of a particular check
// checkStatus holds the output of a particular check.
type checkStatus struct {
name string
healthy bool
@@ -173,8 +173,7 @@ type CheckHandler struct {
}
func (h CheckHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
err := h.Checker(req)
if err != nil {
if err := h.Checker(req); err != nil {
http.Error(resp, fmt.Sprintf("internal server error: %v", err), http.StatusInternalServerError)
} else {
fmt.Fprint(resp, "ok")
@@ -184,10 +183,10 @@ func (h CheckHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
// Checker knows how to perform a health check.
type Checker func(req *http.Request) error
// Ping returns true automatically when checked
// Ping returns true automatically when checked.
var Ping Checker = func(_ *http.Request) error { return nil }
// getExcludedChecks extracts the health check names to be excluded from the query param
// getExcludedChecks extracts the health check names to be excluded from the query param.
func getExcludedChecks(r *http.Request) sets.String {
checks, found := r.URL.Query()["exclude"]
if found {

View File

@@ -17,16 +17,18 @@ limitations under the License.
package controller
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/go-logr/logr"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/handler"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/internal/controller/metrics"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/predicate"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
@@ -35,7 +37,7 @@ import (
var _ inject.Injector = &Controller{}
// Controller implements controller.Controller
// Controller implements controller.Controller.
type Controller struct {
// Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required.
Name string
@@ -58,18 +60,25 @@ type Controller struct {
Queue workqueue.RateLimitingInterface
// SetFields is used to inject dependencies into other objects such as Sources, EventHandlers and Predicates
// Deprecated: the caller should handle injected fields itself.
SetFields func(i interface{}) error
// mu is used to synchronize Controller setup
mu sync.Mutex
// JitterPeriod allows tests to reduce the JitterPeriod so they complete faster
JitterPeriod time.Duration
// Started is true if the Controller has been Started
Started bool
// TODO(community): Consider initializing a logger with the Controller Name as the tag
// ctx is the context that was passed to Start() and used when starting watches.
//
// According to the docs, contexts should not be stored in a struct: https://golang.org/pkg/context,
// while we usually always strive to follow best practices, we consider this a legacy case and it should
// undergo a major refactoring and redesign to allow for context to not be stored in a struct.
ctx context.Context
// CacheSyncTimeout refers to the time limit set on waiting for cache to sync
// Defaults to 2 minutes if not set.
CacheSyncTimeout time.Duration
// startWatches maintains a list of sources, handlers, and predicates to start when the controller is started.
startWatches []watchDescription
@@ -85,12 +94,14 @@ type watchDescription struct {
predicates []predicate.Predicate
}
// Reconcile implements reconcile.Reconciler
func (c *Controller) Reconcile(r reconcile.Request) (reconcile.Result, error) {
return c.Do.Reconcile(r)
// Reconcile implements reconcile.Reconciler.
func (c *Controller) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) {
log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace)
ctx = logf.IntoContext(ctx, log)
return c.Do.Reconcile(ctx, req)
}
// Watch implements controller.Controller
// Watch implements controller.Controller.
func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prct ...predicate.Predicate) error {
c.mu.Lock()
defer c.mu.Unlock()
@@ -117,18 +128,30 @@ func (c *Controller) Watch(src source.Source, evthdler handler.EventHandler, prc
}
c.Log.Info("Starting EventSource", "source", src)
return src.Start(evthdler, c.Queue, prct...)
return src.Start(c.ctx, evthdler, c.Queue, prct...)
}
// Start implements controller.Controller
func (c *Controller) Start(stop <-chan struct{}) error {
// Start implements controller.Controller.
func (c *Controller) Start(ctx context.Context) error {
// use an IIFE to get proper lock handling
// but lock outside to get proper handling of the queue shutdown
c.mu.Lock()
if c.Started {
return errors.New("controller was started more than once. This is likely to be caused by being added to a manager multiple times")
}
c.initMetrics()
// Set the internal context.
c.ctx = ctx
c.Queue = c.MakeQueue()
defer c.Queue.ShutDown() // needs to be outside the iife so that we shutdown after the stop channel is closed
go func() {
<-ctx.Done()
c.Queue.ShutDown()
}()
wg := &sync.WaitGroup{}
err := func() error {
defer c.mu.Unlock()
@@ -140,7 +163,8 @@ func (c *Controller) Start(stop <-chan struct{}) error {
// caches.
for _, watch := range c.startWatches {
c.Log.Info("Starting EventSource", "source", watch.src)
if err := watch.src.Start(watch.handler, c.Queue, watch.predicates...); err != nil {
if err := watch.src.Start(ctx, watch.handler, c.Queue, watch.predicates...); err != nil {
return err
}
}
@@ -153,11 +177,22 @@ func (c *Controller) Start(stop <-chan struct{}) error {
if !ok {
continue
}
if err := syncingSource.WaitForSync(stop); err != nil {
// This code is unreachable in case of kube watches since WaitForCacheSync will never return an error
// Leaving it here because that could happen in the future
err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err)
c.Log.Error(err, "Could not wait for Cache to sync")
if err := func() error {
// use a context with timeout for launching sources and syncing caches.
sourceStartCtx, cancel := context.WithTimeout(ctx, c.CacheSyncTimeout)
defer cancel()
// WaitForSync waits for a definitive timeout, and returns if there
// is an error or a timeout
if err := syncingSource.WaitForSync(sourceStartCtx); err != nil {
err := fmt.Errorf("failed to wait for %s caches to sync: %w", c.Name, err)
c.Log.Error(err, "Could not wait for Cache to sync")
return err
}
return nil
}(); err != nil {
return err
}
}
@@ -168,15 +203,17 @@ func (c *Controller) Start(stop <-chan struct{}) error {
// which won't be garbage collected if we hold a reference to it.
c.startWatches = nil
if c.JitterPeriod == 0 {
c.JitterPeriod = 1 * time.Second
}
// Launch workers to process resources
c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles)
wg.Add(c.MaxConcurrentReconciles)
for i := 0; i < c.MaxConcurrentReconciles; i++ {
// Process work items
go wait.Until(c.worker, c.JitterPeriod, stop)
go func() {
defer wg.Done()
// Run a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the reconcileHandler is never invoked concurrently with the same object.
for c.processNextWorkItem(ctx) {
}
}()
}
c.Started = true
@@ -186,21 +223,16 @@ func (c *Controller) Start(stop <-chan struct{}) error {
return err
}
<-stop
c.Log.Info("Stopping workers")
<-ctx.Done()
c.Log.Info("Shutdown signal received, waiting for all workers to finish")
wg.Wait()
c.Log.Info("All workers finished")
return nil
}
// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the reconcileHandler is never invoked concurrently with the same object.
func (c *Controller) worker() {
for c.processNextWorkItem() {
}
}
// processNextWorkItem will read a single work item off the workqueue and
// attempt to process it, by calling the reconcileHandler.
func (c *Controller) processNextWorkItem() bool {
func (c *Controller) processNextWorkItem(ctx context.Context) bool {
obj, shutdown := c.Queue.Get()
if shutdown {
// Stop working
@@ -215,10 +247,31 @@ func (c *Controller) processNextWorkItem() bool {
// period.
defer c.Queue.Done(obj)
return c.reconcileHandler(obj)
ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(1)
defer ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(-1)
c.reconcileHandler(ctx, obj)
return true
}
func (c *Controller) reconcileHandler(obj interface{}) bool {
const (
labelError = "error"
labelRequeueAfter = "requeue_after"
labelRequeue = "requeue"
labelSuccess = "success"
)
func (c *Controller) initMetrics() {
ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Set(0)
ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Add(0)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Add(0)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Add(0)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Add(0)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Add(0)
ctrlmetrics.WorkerCount.WithLabelValues(c.Name).Set(float64(c.MaxConcurrentReconciles))
}
func (c *Controller) reconcileHandler(ctx context.Context, obj interface{}) {
// Update metrics after processing each item
reconcileStartTS := time.Now()
defer func() {
@@ -234,53 +287,52 @@ func (c *Controller) reconcileHandler(obj interface{}) bool {
c.Queue.Forget(obj)
c.Log.Error(nil, "Queue item was not a Request", "type", fmt.Sprintf("%T", obj), "value", obj)
// Return true, don't take a break
return true
return
}
log := c.Log.WithValues("name", req.Name, "namespace", req.Namespace)
ctx = logf.IntoContext(ctx, log)
// RunInformersAndControllers the syncHandler, passing it the namespace/Name string of the
// RunInformersAndControllers the syncHandler, passing it the Namespace/Name string of the
// resource to be synced.
if result, err := c.Do.Reconcile(req); err != nil {
result, err := c.Do.Reconcile(ctx, req)
switch {
case err != nil:
c.Queue.AddRateLimited(req)
log.Error(err, "Reconciler error")
ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "error").Inc()
return false
} else if result.RequeueAfter > 0 {
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelError).Inc()
log.Error(err, "Reconciler error")
case result.RequeueAfter > 0:
// The result.RequeueAfter request will be lost, if it is returned
// along with a non-nil error. But this is intended as
// We need to drive to stable reconcile loops before queuing due
// to result.RequestAfter
c.Queue.Forget(obj)
c.Queue.AddAfter(req, result.RequeueAfter)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue_after").Inc()
return true
} else if result.Requeue {
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeueAfter).Inc()
case result.Requeue:
c.Queue.AddRateLimited(req)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "requeue").Inc()
return true
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelRequeue).Inc()
default:
// Finally, if no error occurs we Forget this item so it does not
// get queued again until another change happens.
c.Queue.Forget(obj)
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, labelSuccess).Inc()
}
// Finally, if no error occurs we Forget this item so it does not
// get queued again until another change happens.
c.Queue.Forget(obj)
// TODO(directxman12): What does 1 mean? Do we want level constants? Do we want levels at all?
log.V(1).Info("Successfully Reconciled")
ctrlmetrics.ReconcileTotal.WithLabelValues(c.Name, "success").Inc()
// Return true, don't take a break
return true
}
// InjectFunc implement SetFields.Injector
// GetLogger returns this controller's logger.
func (c *Controller) GetLogger() logr.Logger {
return c.Log
}
// InjectFunc implement SetFields.Injector.
func (c *Controller) InjectFunc(f inject.Func) error {
c.SetFields = f
return nil
}
// updateMetrics updates prometheus metrics within the controller
// updateMetrics updates prometheus metrics within the controller.
func (c *Controller) updateMetrics(reconcileTime time.Duration) {
ctrlmetrics.ReconcileTime.WithLabelValues(c.Name).Observe(reconcileTime.Seconds())
}

View File

@@ -18,6 +18,7 @@ package metrics
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
"sigs.k8s.io/controller-runtime/pkg/metrics"
)
@@ -25,24 +26,40 @@ var (
// ReconcileTotal is a prometheus counter metrics which holds the total
// number of reconciliations per controller. It has two labels. controller label refers
// to the controller name and result label refers to the reconcile result i.e
// success, error, requeue, requeue_after
// success, error, requeue, requeue_after.
ReconcileTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "controller_runtime_reconcile_total",
Help: "Total number of reconciliations per controller",
}, []string{"controller", "result"})
// ReconcileErrors is a prometheus counter metrics which holds the total
// number of errors from the Reconciler
// number of errors from the Reconciler.
ReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "controller_runtime_reconcile_errors_total",
Help: "Total number of reconciliation errors per controller",
}, []string{"controller"})
// ReconcileTime is a prometheus metric which keeps track of the duration
// of reconciliations
// of reconciliations.
ReconcileTime = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "controller_runtime_reconcile_time_seconds",
Help: "Length of time per reconciliation per controller",
Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0,
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
}, []string{"controller"})
// WorkerCount is a prometheus metric which holds the number of
// concurrent reconciles per controller.
WorkerCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "controller_runtime_max_concurrent_reconciles",
Help: "Maximum number of concurrent reconciles per controller",
}, []string{"controller"})
// ActiveWorkers is a prometheus metric which holds the number
// of active workers per controller.
ActiveWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "controller_runtime_active_workers",
Help: "Number of currently used workers per controller",
}, []string{"controller"})
)
@@ -51,9 +68,11 @@ func init() {
ReconcileTotal,
ReconcileErrors,
ReconcileTime,
WorkerCount,
ActiveWorkers,
// expose process metrics like CPU, Memory, file descriptor usage etc.
prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
// expose Go runtime metrics like GC stats, memory stats etc.
prometheus.NewGoCollector(),
collectors.NewGoCollector(),
)
}

View File

@@ -1,5 +1,5 @@
/*
Copyright 2019 The Kubernetes Authors.
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,13 +14,8 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package apis
import (
"sigs.k8s.io/kubefed/pkg/apis/multiclusterdns/v1alpha1"
)
func init() {
// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme)
}
// Package flock is copied from k8s.io/kubernetes/pkg/util/flock to avoid
// importing k8s.io/kubernetes as a dependency.
//
// Provides file locking functionalities on unix systems.
package flock

View File

@@ -1,5 +1,7 @@
// +build !linux,!darwin,!freebsd,!openbsd,!netbsd,!dragonfly
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,11 +16,9 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package defaultconfig
package flock
const (
namePrefixFieldSpecs = `
namePrefix:
- path: metadata/name
`
)
// Acquire is not implemented on non-unix systems.
func Acquire(path string) error {
return nil
}

View File

@@ -1,5 +1,7 @@
// +build linux darwin freebsd openbsd netbsd dragonfly
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,21 +16,20 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package transformers
package flock
import "sigs.k8s.io/kustomize/pkg/resmap"
import "golang.org/x/sys/unix"
// noOpTransformer contains a no-op transformer.
type noOpTransformer struct{}
// Acquire acquires a lock on a file for the duration of the process. This method
// is reentrant.
func Acquire(path string) error {
fd, err := unix.Open(path, unix.O_CREAT|unix.O_RDWR|unix.O_CLOEXEC, 0600)
if err != nil {
return err
}
var _ Transformer = &noOpTransformer{}
// We don't need to close the fd since we should hold
// it until the process exits.
// NewNoOpTransformer constructs a noOpTransformer.
func NewNoOpTransformer() Transformer {
return &noOpTransformer{}
}
// Transform does nothing.
func (o *noOpTransformer) Transform(_ resmap.ResMap) error {
return nil
return unix.Flock(fd, unix.LOCK_EX)
}

View File

@@ -14,9 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
// Package log contains utilities for fetching a new logger
// when one is not already available.
// Deprecated: use pkg/log
package log
import (

View File

@@ -17,12 +17,17 @@ limitations under the License.
package objectutil
import (
"errors"
"fmt"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
)
// FilterWithLabels returns a copy of the items in objs matching labelSel
// FilterWithLabels returns a copy of the items in objs matching labelSel.
func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtime.Object, error) {
outItems := make([]runtime.Object, 0, len(objs))
for _, obj := range objs {
@@ -40,3 +45,34 @@ func FilterWithLabels(objs []runtime.Object, labelSel labels.Selector) ([]runtim
}
return outItems, nil
}
// IsAPINamespaced returns true if the object is namespace scoped.
// For unstructured objects the gvk is found from the object itself.
func IsAPINamespaced(obj runtime.Object, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) {
gvk, err := apiutil.GVKForObject(obj, scheme)
if err != nil {
return false, err
}
return IsAPINamespacedWithGVK(gvk, scheme, restmapper)
}
// IsAPINamespacedWithGVK returns true if the object having the provided
// GVK is namespace scoped.
func IsAPINamespacedWithGVK(gk schema.GroupVersionKind, scheme *runtime.Scheme, restmapper apimeta.RESTMapper) (bool, error) {
restmapping, err := restmapper.RESTMapping(schema.GroupKind{Group: gk.Group, Kind: gk.Kind})
if err != nil {
return false, fmt.Errorf("failed to get restmapping: %w", err)
}
scope := restmapping.Scope.Name()
if scope == "" {
return false, errors.New("scope cannot be identified, empty scope returned")
}
if scope != apimeta.RESTScopeNameRoot {
return true, nil
}
return false, nil
}

View File

@@ -17,7 +17,9 @@ limitations under the License.
package recorder
import (
"context"
"fmt"
"sync"
"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
@@ -26,35 +28,150 @@ import (
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/recorder"
)
type provider struct {
// EventBroadcasterProducer makes an event broadcaster, returning
// whether or not the broadcaster should be stopped with the Provider,
// or not (e.g. if it's shared, it shouldn't be stopped with the Provider).
type EventBroadcasterProducer func() (caster record.EventBroadcaster, stopWithProvider bool)
// Provider is a recorder.Provider that records events to the k8s API server
// and to a logr Logger.
type Provider struct {
lock sync.RWMutex
stopped bool
// scheme to specify when creating a recorder
scheme *runtime.Scheme
// eventBroadcaster to create new recorder instance
eventBroadcaster record.EventBroadcaster
// logger is the logger to use when logging diagnostic event info
logger logr.Logger
logger logr.Logger
evtClient typedcorev1.EventInterface
makeBroadcaster EventBroadcasterProducer
broadcasterOnce sync.Once
broadcaster record.EventBroadcaster
stopBroadcaster bool
}
// NB(directxman12): this manually implements Stop instead of Being a runnable because we need to
// stop it *after* everything else shuts down, otherwise we'll cause panics as the leader election
// code finishes up and tries to continue emitting events.
// Stop attempts to stop this provider, stopping the underlying broadcaster
// if the broadcaster asked to be stopped. It kinda tries to honor the given
// context, but the underlying broadcaster has an indefinite wait that doesn't
// return until all queued events are flushed, so this may end up just returning
// before the underlying wait has finished instead of cancelling the wait.
// This is Very Frustrating™.
func (p *Provider) Stop(shutdownCtx context.Context) {
doneCh := make(chan struct{})
go func() {
// technically, this could start the broadcaster, but practically, it's
// almost certainly already been started (e.g. by leader election). We
// need to invoke this to ensure that we don't inadvertently race with
// an invocation of getBroadcaster.
broadcaster := p.getBroadcaster()
if p.stopBroadcaster {
p.lock.Lock()
broadcaster.Shutdown()
p.stopped = true
p.lock.Unlock()
}
close(doneCh)
}()
select {
case <-shutdownCtx.Done():
case <-doneCh:
}
}
// getBroadcaster ensures that a broadcaster is started for this
// provider, and returns it. It's threadsafe.
func (p *Provider) getBroadcaster() record.EventBroadcaster {
// NB(directxman12): this can technically still leak if something calls
// "getBroadcaster" (i.e. Emits an Event) but never calls Start, but if we
// create the broadcaster in start, we could race with other things that
// are started at the same time & want to emit events. The alternative is
// silently swallowing events and more locking, but that seems suboptimal.
p.broadcasterOnce.Do(func() {
broadcaster, stop := p.makeBroadcaster()
broadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: p.evtClient})
broadcaster.StartEventWatcher(
func(e *corev1.Event) {
p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message)
})
p.broadcaster = broadcaster
p.stopBroadcaster = stop
})
return p.broadcaster
}
// NewProvider create a new Provider instance.
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, broadcaster record.EventBroadcaster) (recorder.Provider, error) {
func NewProvider(config *rest.Config, scheme *runtime.Scheme, logger logr.Logger, makeBroadcaster EventBroadcasterProducer) (*Provider, error) {
clientSet, err := kubernetes.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to init clientSet: %w", err)
}
p := &provider{scheme: scheme, logger: logger, eventBroadcaster: broadcaster}
p.eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: clientSet.CoreV1().Events("")})
p.eventBroadcaster.StartEventWatcher(
func(e *corev1.Event) {
p.logger.V(1).Info(e.Type, "object", e.InvolvedObject, "reason", e.Reason, "message", e.Message)
})
p := &Provider{scheme: scheme, logger: logger, makeBroadcaster: makeBroadcaster, evtClient: clientSet.CoreV1().Events("")}
return p, nil
}
func (p *provider) GetEventRecorderFor(name string) record.EventRecorder {
return p.eventBroadcaster.NewRecorder(p.scheme, corev1.EventSource{Component: name})
// GetEventRecorderFor returns an event recorder that broadcasts to this provider's
// broadcaster. All events will be associated with a component of the given name.
func (p *Provider) GetEventRecorderFor(name string) record.EventRecorder {
return &lazyRecorder{
prov: p,
name: name,
}
}
// lazyRecorder is a recorder that doesn't actually instantiate any underlying
// recorder until the first event is emitted.
type lazyRecorder struct {
prov *Provider
name string
recOnce sync.Once
rec record.EventRecorder
}
// ensureRecording ensures that a concrete recorder is populated for this recorder.
func (l *lazyRecorder) ensureRecording() {
l.recOnce.Do(func() {
broadcaster := l.prov.getBroadcaster()
l.rec = broadcaster.NewRecorder(l.prov.scheme, corev1.EventSource{Component: l.name})
})
}
func (l *lazyRecorder) Event(object runtime.Object, eventtype, reason, message string) {
l.ensureRecording()
l.prov.lock.RLock()
if !l.prov.stopped {
l.rec.Event(object, eventtype, reason, message)
}
l.prov.lock.RUnlock()
}
func (l *lazyRecorder) Eventf(object runtime.Object, eventtype, reason, messageFmt string, args ...interface{}) {
l.ensureRecording()
l.prov.lock.RLock()
if !l.prov.stopped {
l.rec.Eventf(object, eventtype, reason, messageFmt, args...)
}
l.prov.lock.RUnlock()
}
func (l *lazyRecorder) AnnotatedEventf(object runtime.Object, annotations map[string]string, eventtype, reason, messageFmt string, args ...interface{}) {
l.ensureRecording()
l.prov.lock.RLock()
if !l.prov.stopped {
l.rec.AnnotatedEventf(object, annotations, eventtype, reason, messageFmt, args...)
}
l.prov.lock.RUnlock()
}

View File

@@ -0,0 +1,126 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package addr
import (
"fmt"
"io/fs"
"net"
"os"
"path/filepath"
"strings"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/flock"
)
// TODO(directxman12): interface / release functionality for external port managers
const (
portReserveTime = 10 * time.Minute
portConflictRetry = 100
portFilePrefix = "port-"
)
var (
cacheDir string
)
func init() {
baseDir, err := os.UserCacheDir()
if err != nil {
baseDir = os.TempDir()
}
cacheDir = filepath.Join(baseDir, "kubebuilder-envtest")
if err := os.MkdirAll(cacheDir, 0750); err != nil {
panic(err)
}
}
type portCache struct{}
func (c *portCache) add(port int) (bool, error) {
// Remove outdated ports.
if err := fs.WalkDir(os.DirFS(cacheDir), ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() || !d.Type().IsRegular() || !strings.HasPrefix(path, portFilePrefix) {
return nil
}
info, err := d.Info()
if err != nil {
return err
}
if time.Since(info.ModTime()) > portReserveTime {
if err := os.Remove(filepath.Join(cacheDir, path)); err != nil {
return err
}
}
return nil
}); err != nil {
return false, err
}
// Try allocating new port, by acquiring a file.
if err := flock.Acquire(fmt.Sprintf("%s/%s%d", cacheDir, portFilePrefix, port)); os.IsExist(err) {
return false, nil
} else if err != nil {
return false, err
}
return true, nil
}
var cache = &portCache{}
func suggest(listenHost string) (int, string, error) {
if listenHost == "" {
listenHost = "localhost"
}
addr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(listenHost, "0"))
if err != nil {
return -1, "", err
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return -1, "", err
}
if err := l.Close(); err != nil {
return -1, "", err
}
return l.Addr().(*net.TCPAddr).Port,
addr.IP.String(),
nil
}
// Suggest suggests an address a process can listen on. It returns
// a tuple consisting of a free port and the hostname resolved to its IP.
// It makes sure that new port allocated does not conflict with old ports
// allocated within 1 minute.
func Suggest(listenHost string) (int, string, error) {
for i := 0; i < portConflictRetry; i++ {
port, resolvedHost, err := suggest(listenHost)
if err != nil {
return -1, "", err
}
if ok, err := cache.add(port); ok {
return port, resolvedHost, nil
} else if err != nil {
return -1, "", err
}
}
return -1, "", fmt.Errorf("no free ports found after %d retries", portConflictRetry)
}

View File

@@ -1,4 +1,20 @@
package internal
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package certs
// NB(directxman12): nothing has verified that this has good settings. In fact,
// the setting generated here are probably terrible, but they're fine for integration
@@ -137,15 +153,71 @@ func (c *TinyCA) makeCert(cfg certutil.Config) (CertPair, error) {
}, nil
}
// NewServingCert returns a new CertPair for a serving HTTPS on localhost.
func (c *TinyCA) NewServingCert() (CertPair, error) {
// NewServingCert returns a new CertPair for a serving HTTPS on localhost (or other specified names).
func (c *TinyCA) NewServingCert(names ...string) (CertPair, error) {
if len(names) == 0 {
names = []string{"localhost"}
}
dnsNames, ips, err := resolveNames(names)
if err != nil {
return CertPair{}, err
}
return c.makeCert(certutil.Config{
CommonName: "localhost",
Organization: []string{c.orgName},
AltNames: certutil.AltNames{
DNSNames: []string{"localhost"},
IPs: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
DNSNames: dnsNames,
IPs: ips,
},
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
})
}
// ClientInfo describes some Kubernetes user for the purposes of creating
// client certificates.
type ClientInfo struct {
// Name is the user name (embedded as the cert's CommonName)
Name string
// Groups are the groups to which this user belongs (embedded as the cert's
// Organization)
Groups []string
}
// NewClientCert produces a new CertPair suitable for use with Kubernetes
// client cert auth with an API server validating based on this CA.
func (c *TinyCA) NewClientCert(user ClientInfo) (CertPair, error) {
return c.makeCert(certutil.Config{
CommonName: user.Name,
Organization: user.Groups,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
})
}
func resolveNames(names []string) ([]string, []net.IP, error) {
dnsNames := []string{}
ips := []net.IP{}
for _, name := range names {
if name == "" {
continue
}
ip := net.ParseIP(name)
if ip == nil {
dnsNames = append(dnsNames, name)
// Also resolve to IPs.
nameIPs, err := net.LookupHost(name)
if err != nil {
return nil, nil, err
}
for _, nameIP := range nameIPs {
ip = net.ParseIP(nameIP)
if ip != nil {
ips = append(ips, ip)
}
}
} else {
ips = append(ips, ip)
}
}
return dnsNames, ips, nil
}

View File

@@ -0,0 +1,469 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strconv"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
const (
// saKeyFile is the name of the service account signing private key file.
saKeyFile = "sa-signer.key"
// saKeyFile is the name of the service account signing public key (cert) file.
saCertFile = "sa-signer.crt"
)
// SecureServing provides/configures how the API server serves on the secure port.
type SecureServing struct {
// ListenAddr contains the host & port to serve on.
//
// Configurable. If unset, it will be defaulted.
process.ListenAddr
// CA contains the CA that signed the API server's serving certificates.
//
// Read-only.
CA []byte
// Authn can be used to provision users, and override what type of
// authentication is used to provision users.
//
// Configurable. If unset, it will be defaulted.
Authn
}
// APIServer knows how to run a kubernetes apiserver.
type APIServer struct {
// URL is the address the ApiServer should listen on for client
// connections.
//
// If set, this will configure the *insecure* serving details.
// If unset, it will contain the insecure port if insecure serving is enabled,
// and otherwise will contain the secure port.
//
// If this is not specified, we default to a random free port on localhost.
//
// Deprecated: use InsecureServing (for the insecure URL) or SecureServing, ideally.
URL *url.URL
// SecurePort is the additional secure port that the APIServer should listen on.
//
// If set, this will override SecureServing.Port.
//
// Deprecated: use SecureServing.
SecurePort int
// SecureServing indicates how the API server will serve on the secure port.
//
// Some parts are configurable. Will be defaulted if unset.
SecureServing
// InsecureServing indicates how the API server will serve on the insecure port.
//
// If unset, the insecure port will be disabled. Set to an empty struct to get
// default values.
//
// Deprecated: does not work with Kubernetes versions 1.20 and above. Use secure
// serving instead.
InsecureServing *process.ListenAddr
// Path is the path to the apiserver binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and
// the default test assets directory. See the "Binaries" section above (in
// doc.go) for details.
Path string
// Args is a list of arguments which will passed to the APIServer binary.
// Before they are passed on, they will be evaluated as go-template strings.
// This means you can use fields which are defined and exported on this
// APIServer struct (e.g. "--cert-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the APIServer's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the APIServer will
// be used.
//
// They will be loaded into the same argument set as Configure. Each flag
// will be Append-ed to the configured arguments just before launch.
//
// Deprecated: use Configure instead.
Args []string
// CertDir is a path to a directory containing whatever certificates the
// APIServer will need.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
CertDir string
// EtcdURL is the URL of the Etcd the APIServer should use.
//
// If this is not specified, the Start() method will return an error.
EtcdURL *url.URL
// StartTimeout, StopTimeout specify the time the APIServer is allowed to
// take when starting and stoppping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where APIServer should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *process.State
// args contains the structured arguments to use for running the API server
// Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs()
args *process.Arguments
}
// Configure returns Arguments that may be used to customize the
// flags used to launch the API server. A set of defaults will
// be applied underneath.
func (s *APIServer) Configure() *process.Arguments {
if s.args == nil {
s.args = process.EmptyArguments()
}
return s.args
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (s *APIServer) Start() error {
if err := s.prepare(); err != nil {
return err
}
return s.processState.Start(s.Out, s.Err)
}
func (s *APIServer) prepare() error {
if err := s.setProcessState(); err != nil {
return err
}
return s.Authn.Start()
}
// configurePorts configures the serving ports for this API server.
//
// Most of this method currently deals with making the deprecated fields
// take precedence over the new fields.
func (s *APIServer) configurePorts() error {
// prefer the old fields to the new fields if a user set one,
// otherwise, default the new fields and populate the old ones.
// Insecure: URL, InsecureServing
if s.URL != nil {
s.InsecureServing = &process.ListenAddr{
Address: s.URL.Hostname(),
Port: s.URL.Port(),
}
} else if insec := s.InsecureServing; insec != nil {
if insec.Port == "" || insec.Address == "" {
port, host, err := addr.Suggest("")
if err != nil {
return fmt.Errorf("unable to provision unused insecure port: %w", err)
}
s.InsecureServing.Port = strconv.Itoa(port)
s.InsecureServing.Address = host
}
s.URL = s.InsecureServing.URL("http", "")
}
// Secure: SecurePort, SecureServing
if s.SecurePort != 0 {
s.SecureServing.Port = strconv.Itoa(s.SecurePort)
// if we don't have an address, try the insecure address, and otherwise
// default to loopback.
if s.SecureServing.Address == "" {
if s.InsecureServing != nil {
s.SecureServing.Address = s.InsecureServing.Address
} else {
s.SecureServing.Address = "127.0.0.1"
}
}
} else if s.SecureServing.Port == "" || s.SecureServing.Address == "" {
port, host, err := addr.Suggest("")
if err != nil {
return fmt.Errorf("unable to provision unused secure port: %w", err)
}
s.SecureServing.Port = strconv.Itoa(port)
s.SecureServing.Address = host
s.SecurePort = port
}
return nil
}
func (s *APIServer) setProcessState() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}
var err error
// unconditionally re-set this so we can successfully restart
// TODO(directxman12): we supported this in the past, but do we actually
// want to support re-using an API server object to restart? The loss
// of provisioned users is surprising to say the least.
s.processState = &process.State{
Dir: s.CertDir,
Path: s.Path,
StartTimeout: s.StartTimeout,
StopTimeout: s.StopTimeout,
}
if err := s.processState.Init("kube-apiserver"); err != nil {
return err
}
if err := s.configurePorts(); err != nil {
return err
}
// the secure port will always be on, so use that
s.processState.HealthCheck.URL = *s.SecureServing.URL("https", "/healthz")
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout
if err := s.populateAPIServerCerts(); err != nil {
return err
}
if s.SecureServing.Authn == nil {
authn, err := NewCertAuthn()
if err != nil {
return err
}
s.SecureServing.Authn = authn
}
if err := s.Authn.Configure(s.CertDir, s.Configure()); err != nil {
return err
}
// NB(directxman12): insecure port is a mess:
// - 1.19 and below have the `--insecure-port` flag, and require it to be set to zero to
// disable it, otherwise the default will be used and we'll conflict.
// - 1.20 requires the flag to be unset or set to zero, and yells at you if you configure it
// - 1.24 won't have the flag at all...
//
// In an effort to automatically do the right thing during this mess, we do feature discovery
// on the flags, and hope that we've "parsed" them properly.
//
// TODO(directxman12): once we support 1.20 as the min version (might be when 1.24 comes out,
// might be around 1.25 or 1.26), remove this logic and the corresponding line in API server's
// default args.
if err := s.discoverFlags(); err != nil {
return err
}
s.processState.Args, s.Args, err = process.TemplateAndArguments(s.Args, s.Configure(), process.TemplateDefaults{ //nolint:staticcheck
Data: s,
Defaults: s.defaultArgs(),
MinimalDefaults: map[string][]string{
// as per kubernetes-sigs/controller-runtime#641, we need this (we
// probably need other stuff too, but this is the only thing that was
// previously considered a "minimal default")
"service-cluster-ip-range": {"10.0.0.0/24"},
// we need *some* authorization mode for health checks on the secure port,
// so default to RBAC unless the user set something else (in which case
// this'll be ignored due to SliceToArguments using AppendNoDefaults).
"authorization-mode": {"RBAC"},
},
})
if err != nil {
return err
}
return nil
}
// discoverFlags checks for certain flags that *must* be set in certain
// versions, and *must not* be set in others.
func (s *APIServer) discoverFlags() error {
// Present: <1.24, Absent: >= 1.24
present, err := s.processState.CheckFlag("insecure-port")
if err != nil {
return err
}
if !present {
s.Configure().Disable("insecure-port")
}
return nil
}
func (s *APIServer) defaultArgs() map[string][]string {
args := map[string][]string{
"service-cluster-ip-range": {"10.0.0.0/24"},
"allow-privileged": {"true"},
// we're keeping this disabled because if enabled, default SA is
// missing which would force all tests to create one in normal
// apiserver operation this SA is created by controller, but that is
// not run in integration environment
"disable-admission-plugins": {"ServiceAccount"},
"cert-dir": {s.CertDir},
"authorization-mode": {"RBAC"},
"secure-port": {s.SecureServing.Port},
// NB(directxman12): previously we didn't set the bind address for the secure
// port. It *shouldn't* make a difference unless people are doing something really
// funky, but if you start to get bug reports look here ;-)
"bind-address": {s.SecureServing.Address},
// required on 1.20+, fine to leave on for <1.20
"service-account-issuer": {s.SecureServing.URL("https", "/").String()},
"service-account-key-file": {filepath.Join(s.CertDir, saCertFile)},
"service-account-signing-key-file": {filepath.Join(s.CertDir, saKeyFile)},
}
if s.EtcdURL != nil {
args["etcd-servers"] = []string{s.EtcdURL.String()}
}
if s.URL != nil {
args["insecure-port"] = []string{s.URL.Port()}
args["insecure-bind-address"] = []string{s.URL.Hostname()}
} else {
// TODO(directxman12): remove this once 1.21 is the lowest version we support
// (this might be a while, but this line'll break as of 1.24, so see the comment
// in Start
args["insecure-port"] = []string{"0"}
}
return args
}
func (s *APIServer) populateAPIServerCerts() error {
_, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt"))
if !os.IsNotExist(statErr) {
return statErr
}
ca, err := certs.NewTinyCA()
if err != nil {
return err
}
servingCerts, err := ca.NewServingCert()
if err != nil {
return err
}
certData, keyData, err := servingCerts.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec
return err
}
s.SecureServing.CA = ca.CA.CertBytes()
// service account signing files too
saCA, err := certs.NewTinyCA()
if err != nil {
return err
}
saCert, saKey, err := saCA.CA.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec
return err
}
return ioutil.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
if s.processState.DirNeedsCleaning {
s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
if s.processState != nil {
if err := s.processState.Stop(); err != nil {
return err
}
}
return s.Authn.Stop()
}
// APIServerDefaultArgs exposes the default args for the APIServer so that you
// can use those to append your own additional arguments.
//
// Note that these arguments don't handle newer API servers well to due the more
// complex feature detection neeeded. It's recommended that you switch to .Configure
// as you upgrade API server versions.
//
// Deprecated: use APIServer.Configure().
var APIServerDefaultArgs = []string{
"--advertise-address=127.0.0.1",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{else}}0{{ end }}",
"{{ if .URL }}--insecure-bind-address={{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
// we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one
// in normal apiserver operation this SA is created by controller, but that is not run in integration environment
"--disable-admission-plugins=ServiceAccount",
"--service-cluster-ip-range=10.0.0.0/24",
"--allow-privileged=true",
// NB(directxman12): we also enable RBAC if nothing else was enabled
}
// PrepareAPIServer is an internal-only (NEVER SHOULD BE EXPOSED)
// function that sets up the API server just before starting it,
// without actually starting it. This saves time on tests.
//
// NB(directxman12): do not expose this outside of internal -- it's unsafe to
// use, because things like port allocation could race even more than they
// currently do if you later call start!
func PrepareAPIServer(s *APIServer) error {
return s.prepare()
}
// APIServerArguments is an internal-only (NEVER SHOULD BE EXPOSED)
// function that sets up the API server just before starting it,
// without actually starting it. It's public to make testing easier.
//
// NB(directxman12): do not expose this outside of internal.
func APIServerArguments(s *APIServer) []string {
return s.processState.Args
}

View File

@@ -0,0 +1,142 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"io/ioutil"
"path/filepath"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
// User represents a Kubernetes user.
type User struct {
// Name is the user's Name.
Name string
// Groups are the groups to which the user belongs.
Groups []string
}
// Authn knows how to configure an API server for a particular type of authentication,
// and provision users under that authentication scheme.
//
// The methods must be called in the following order (as presented below in the interface
// for a mnemonic):
//
// 1. Configure
// 2. Start
// 3. AddUsers (0+ calls)
// 4. Stop.
type Authn interface {
// Configure provides the working directory to this authenticator,
// and configures the given API server arguments to make use of this authenticator.
//
// Should be called first.
Configure(workDir string, args *process.Arguments) error
// Start runs this authenticator. Will be called just before API server start.
//
// Must be called after Configure.
Start() error
// AddUser provisions a user, returning a copy of the given base rest.Config
// configured to authenticate as that users.
//
// May only be called while the authenticator is "running".
AddUser(user User, baseCfg *rest.Config) (*rest.Config, error)
// Stop shuts down this authenticator.
Stop() error
}
// CertAuthn is an authenticator (Authn) that makes use of client certificate authn.
type CertAuthn struct {
// ca is the CA used to sign the client certs
ca *certs.TinyCA
// certDir is the directory used to write the CA crt file
// so that the API server can read it.
certDir string
}
// NewCertAuthn creates a new client-cert-based Authn with a new CA.
func NewCertAuthn() (*CertAuthn, error) {
ca, err := certs.NewTinyCA()
if err != nil {
return nil, fmt.Errorf("unable to provision client certificate auth CA: %w", err)
}
return &CertAuthn{
ca: ca,
}, nil
}
// AddUser provisions a new user that's authenticated via certificates, with
// the given uesrname and groups embedded in the certificate as expected by the
// API server.
func (c *CertAuthn) AddUser(user User, baseCfg *rest.Config) (*rest.Config, error) {
certs, err := c.ca.NewClientCert(certs.ClientInfo{
Name: user.Name,
Groups: user.Groups,
})
if err != nil {
return nil, fmt.Errorf("unable to create client certificates for %s: %w", user.Name, err)
}
crt, key, err := certs.AsBytes()
if err != nil {
return nil, fmt.Errorf("unable to serialize client certificates for %s: %w", user.Name, err)
}
cfg := rest.CopyConfig(baseCfg)
cfg.CertData = crt
cfg.KeyData = key
return cfg, nil
}
// caCrtPath returns the path to the on-disk client-cert CA crt file.
func (c *CertAuthn) caCrtPath() string {
return filepath.Join(c.certDir, "client-cert-auth-ca.crt")
}
// Configure provides the working directory to this authenticator,
// and configures the given API server arguments to make use of this authenticator.
func (c *CertAuthn) Configure(workDir string, args *process.Arguments) error {
c.certDir = workDir
args.Set("client-ca-file", c.caCrtPath())
return nil
}
// Start runs this authenticator. Will be called just before API server start.
//
// Must be called after Configure.
func (c *CertAuthn) Start() error {
if len(c.certDir) == 0 {
return fmt.Errorf("start called before configure")
}
caCrt := c.ca.CA.CertBytes()
if err := ioutil.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err)
}
return nil
}
// Stop shuts down this authenticator.
func (c *CertAuthn) Stop() error {
// no-op -- our workdir is cleaned up for us automatically
return nil
}

View File

@@ -0,0 +1,180 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"io"
"net"
"net/url"
"strconv"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
// Etcd knows how to run an etcd server.
type Etcd struct {
// URL is the address the Etcd should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// Path is the path to the etcd binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_ETCD environment variable, and the default
// test assets directory. See the "Binaries" section above (in doc.go) for
// details.
Path string
// Args is a list of arguments which will passed to the Etcd binary. Before
// they are passed on, the`y will be evaluated as go-template strings. This
// means you can use fields which are defined and exported on this Etcd
// struct (e.g. "--data-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the Etcd's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the Etcd will be
// used.
//
// They will be loaded into the same argument set as Configure. Each flag
// will be Append-ed to the configured arguments just before launch.
//
// Deprecated: use Configure instead.
Args []string
// DataDir is a path to a directory in which etcd can store its state.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
DataDir string
// StartTimeout, StopTimeout specify the time the Etcd is allowed to
// take when starting and stopping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where Etcd should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
// processState contains the actual details about this running process
processState *process.State
// args contains the structured arguments to use for running etcd.
// Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs()
args *process.Arguments
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
func (e *Etcd) Start() error {
if err := e.setProcessState(); err != nil {
return err
}
return e.processState.Start(e.Out, e.Err)
}
func (e *Etcd) setProcessState() error {
e.processState = &process.State{
Dir: e.DataDir,
Path: e.Path,
StartTimeout: e.StartTimeout,
StopTimeout: e.StopTimeout,
}
// unconditionally re-set this so we can successfully restart
// TODO(directxman12): we supported this in the past, but do we actually
// want to support re-using an API server object to restart? The loss
// of provisioned users is surprising to say the least.
if err := e.processState.Init("etcd"); err != nil {
return err
}
if e.URL == nil {
port, host, err := addr.Suggest("")
if err != nil {
return err
}
e.URL = &url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
}
// can use /health as of etcd 3.3.0
e.processState.HealthCheck.URL = *e.URL
e.processState.HealthCheck.Path = "/health"
e.DataDir = e.processState.Dir
e.Path = e.processState.Path
e.StartTimeout = e.processState.StartTimeout
e.StopTimeout = e.processState.StopTimeout
var err error
e.processState.Args, e.Args, err = process.TemplateAndArguments(e.Args, e.Configure(), process.TemplateDefaults{ //nolint:staticcheck
Data: e,
Defaults: e.defaultArgs(),
})
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the DataDir if necessary.
func (e *Etcd) Stop() error {
if e.processState.DirNeedsCleaning {
e.DataDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
return e.processState.Stop()
}
func (e *Etcd) defaultArgs() map[string][]string {
args := map[string][]string{
"listen-peer-urls": {"http://localhost:0"},
"data-dir": {e.DataDir},
}
if e.URL != nil {
args["advertise-client-urls"] = []string{e.URL.String()}
args["listen-client-urls"] = []string{e.URL.String()}
}
return args
}
// Configure returns Arguments that may be used to customize the
// flags used to launch etcd. A set of defaults will
// be applied underneath.
func (e *Etcd) Configure() *process.Arguments {
if e.args == nil {
e.args = process.EmptyArguments()
}
return e.args
}
// EtcdDefaultArgs exposes the default args for Etcd so that you
// can use those to append your own additional arguments.
var EtcdDefaultArgs = []string{
"--listen-peer-urls=http://localhost:0",
"--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--data-dir={{ .DataDir }}",
}

View File

@@ -0,0 +1,119 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"bytes"
"fmt"
"io"
"net/url"
"os/exec"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
kcapi "k8s.io/client-go/tools/clientcmd/api"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
const (
envtestName = "envtest"
)
// KubeConfigFromREST reverse-engineers a kubeconfig file from a rest.Config.
// The options are tailored towards the rest.Configs we generate, so they're
// not broadly applicable.
//
// This is not intended to be exposed beyond internal for the above reasons.
func KubeConfigFromREST(cfg *rest.Config) ([]byte, error) {
kubeConfig := kcapi.NewConfig()
protocol := "https"
if !rest.IsConfigTransportTLS(*cfg) {
protocol = "http"
}
// cfg.Host is a URL, so we need to parse it so we can properly append the API path
baseURL, err := url.Parse(cfg.Host)
if err != nil {
return nil, fmt.Errorf("unable to interpret config's host value as a URL: %w", err)
}
kubeConfig.Clusters[envtestName] = &kcapi.Cluster{
// TODO(directxman12): if client-go ever decides to expose defaultServerUrlFor(config),
// we can just use that. Note that this is not the same as the public DefaultServerURL,
// which requires us to pass a bunch of stuff in manually.
Server: (&url.URL{Scheme: protocol, Host: baseURL.Host, Path: cfg.APIPath}).String(),
CertificateAuthorityData: cfg.CAData,
}
kubeConfig.AuthInfos[envtestName] = &kcapi.AuthInfo{
// try to cover all auth strategies that aren't plugins
ClientCertificateData: cfg.CertData,
ClientKeyData: cfg.KeyData,
Token: cfg.BearerToken,
Username: cfg.Username,
Password: cfg.Password,
}
kcCtx := kcapi.NewContext()
kcCtx.Cluster = envtestName
kcCtx.AuthInfo = envtestName
kubeConfig.Contexts[envtestName] = kcCtx
kubeConfig.CurrentContext = envtestName
contents, err := clientcmd.Write(*kubeConfig)
if err != nil {
return nil, fmt.Errorf("unable to serialize kubeconfig file: %w", err)
}
return contents, nil
}
// KubeCtl is a wrapper around the kubectl binary.
type KubeCtl struct {
// Path where the kubectl binary can be found.
//
// If this is left empty, we will attempt to locate a binary, by checking for
// the TEST_ASSET_KUBECTL environment variable, and the default test assets
// directory. See the "Binaries" section above (in doc.go) for details.
Path string
// Opts can be used to configure additional flags which will be used each
// time the wrapped binary is called.
//
// For example, you might want to use this to set the URL of the APIServer to
// connect to.
Opts []string
}
// Run executes the wrapped binary with some preconfigured options and the
// arguments given to this method. It returns Readers for the stdout and
// stderr.
func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) {
if k.Path == "" {
k.Path = process.BinPathFinder("kubectl", "")
}
stdoutBuffer := &bytes.Buffer{}
stderrBuffer := &bytes.Buffer{}
allArgs := append(k.Opts, args...)
cmd := exec.Command(k.Path, allArgs...)
cmd.Stdout = stdoutBuffer
cmd.Stderr = stderrBuffer
err = cmd.Run()
return stdoutBuffer, stderrBuffer, err
}

View File

@@ -0,0 +1,248 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"net/url"
"os"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
)
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
// Don't use this for anything else!
var NewTinyCA = certs.NewTinyCA
// ControlPlane is a struct that knows how to start your test control plane.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in
// future.
type ControlPlane struct {
APIServer *APIServer
Etcd *Etcd
// Kubectl will override the default asset search path for kubectl
KubectlPath string
// for the deprecated methods (Kubectl, etc)
defaultUserCfg *rest.Config
defaultUserKubectl *KubeCtl
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
f.APIServer.EtcdURL = f.Etcd.URL
if err := f.APIServer.Start(); err != nil {
return err
}
// provision the default user -- can be removed when the related
// methods are removed. The default user has admin permissions to
// mimic legacy no-authz setups.
user, err := f.AddUser(User{Name: "default", Groups: []string{"system:masters"}}, &rest.Config{})
if err != nil {
return fmt.Errorf("unable to provision the default (legacy) user: %w", err)
}
kubectl, err := user.Kubectl()
if err != nil {
return fmt.Errorf("unable to provision the default (legacy) kubeconfig: %w", err)
}
f.defaultUserCfg = user.Config()
f.defaultUserKubectl = kubectl
return nil
}
// Stop will stop your control plane processes, and clean up their data.
func (f *ControlPlane) Stop() error {
var errList []error
if f.APIServer != nil {
if err := f.APIServer.Stop(); err != nil {
errList = append(errList, err)
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
errList = append(errList, err)
}
}
return kerrors.NewAggregate(errList)
}
// APIURL returns the URL you should connect to to talk to your API server.
//
// If insecure serving is configured, this will contain the insecure port.
// Otherwise, it will contain the secure port.
//
// Deprecated: use AddUser instead, or APIServer.{Ins|S}ecureServing.URL if
// you really want just the URL.
func (f *ControlPlane) APIURL() *url.URL {
return f.APIServer.URL
}
// KubeCtl returns a pre-configured KubeCtl, ready to connect to this
// ControlPlane.
//
// Deprecated: use AddUser & AuthenticatedUser.Kubectl instead.
func (f *ControlPlane) KubeCtl() *KubeCtl {
return f.defaultUserKubectl
}
// RESTClientConfig returns a pre-configured restconfig, ready to connect to
// this ControlPlane.
//
// Deprecated: use AddUser & AuthenticatedUser.Config instead.
func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) {
return f.defaultUserCfg, nil
}
// AuthenticatedUser contains access information for an provisioned user,
// including REST config, kubeconfig contents, and access to a KubeCtl instance.
//
// It's not "safe" to use the methods on this till after the API server has been
// started (due to certificate initialization and such). The various methods will
// panic if this is done.
type AuthenticatedUser struct {
// cfg is the rest.Config for connecting to the API server. It's lazily initialized.
cfg *rest.Config
// cfgIsComplete indicates the cfg has had late-initialized fields (e.g.
// API server CA data) initialized.
cfgIsComplete bool
// apiServer is a handle to the APIServer that's used when finalizing cfg
// and producing the kubectl instance.
plane *ControlPlane
// kubectl is our existing, provisioned kubectl. We don't provision one
// till someone actually asks for it.
kubectl *KubeCtl
}
// Config returns the REST config that can be used to connect to the API server
// as this user.
//
// Will panic if used before the API server is started.
func (u *AuthenticatedUser) Config() *rest.Config {
// NB(directxman12): we choose to panic here for ergonomics sake, and because there's
// not really much you can do to "handle" this error. This machinery is intended to be
// used in tests anyway, so panicing is not a particularly big deal.
if u.cfgIsComplete {
return u.cfg
}
if len(u.plane.APIServer.SecureServing.CA) == 0 {
panic("the API server has not yet been started, please do that before accessing connection details")
}
u.cfg.CAData = u.plane.APIServer.SecureServing.CA
u.cfg.Host = u.plane.APIServer.SecureServing.URL("https", "/").String()
u.cfgIsComplete = true
return u.cfg
}
// KubeConfig returns a KubeConfig that's roughly equivalent to this user's REST config.
//
// Will panic if used before the API server is started.
func (u AuthenticatedUser) KubeConfig() ([]byte, error) {
// NB(directxman12): we don't return the actual API object to avoid yet another
// piece of kubernetes API in our public API, and also because generally the thing
// you want to do with this is just write it out to a file for external debugging
// purposes, etc.
return KubeConfigFromREST(u.Config())
}
// Kubectl returns a KubeCtl instance for talking to the API server as this user. It uses
// a kubeconfig equivalent to that returned by .KubeConfig.
//
// Will panic if used before the API server is started.
func (u *AuthenticatedUser) Kubectl() (*KubeCtl, error) {
if u.kubectl != nil {
return u.kubectl, nil
}
if len(u.plane.APIServer.CertDir) == 0 {
panic("the API server has not yet been started, please do that before accessing connection details")
}
// cleaning this up is handled when our tmpDir is deleted
out, err := os.CreateTemp(u.plane.APIServer.CertDir, "*.kubecfg")
if err != nil {
return nil, fmt.Errorf("unable to create file for kubeconfig: %w", err)
}
defer out.Close()
contents, err := KubeConfigFromREST(u.Config())
if err != nil {
return nil, err
}
if _, err := out.Write(contents); err != nil {
return nil, fmt.Errorf("unable to write kubeconfig to disk at %s: %w", out.Name(), err)
}
k := &KubeCtl{
Path: u.plane.KubectlPath,
}
k.Opts = append(k.Opts, fmt.Sprintf("--kubeconfig=%s", out.Name()))
u.kubectl = k
return k, nil
}
// AddUser provisions a new user in the cluster. It uses the APIServer's authentication
// strategy -- see APIServer.SecureServing.Authn.
//
// Unlike AddUser, it's safe to pass a nil rest.Config here if you have no
// particular opinions about the config.
//
// The default authentication strategy is not guaranteed to any specific strategy, but it is
// guaranteed to be callable both before and after Start has been called (but, as noted in the
// AuthenticatedUser docs, the given user objects are only valid after Start has been called).
func (f *ControlPlane) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) {
if f.GetAPIServer().SecureServing.Authn == nil {
return nil, fmt.Errorf("no API server authentication is configured yet. The API server defaults one when Start is called, did you mean to use that?")
}
if baseConfig == nil {
baseConfig = &rest.Config{}
}
cfg, err := f.GetAPIServer().SecureServing.AddUser(user, baseConfig)
if err != nil {
return nil, err
}
return &AuthenticatedUser{
cfg: cfg,
plane: f,
}, nil
}
// GetAPIServer returns this ControlPlane's APIServer, initializing it if necessary.
func (f *ControlPlane) GetAPIServer() *APIServer {
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
return f.APIServer
}

View File

@@ -1 +0,0 @@
assets/bin

View File

@@ -1,10 +0,0 @@
# Integration Testing Framework
This package has been moved from [https://github.com/kubernetes-sigs/testing_frameworks/tree/master/integration](https://github.com/kubernetes-sigs/testing_frameworks/tree/master/integration).
A framework for integration testing components of kubernetes. This framework is
intended to work properly both in CI, and on a local dev machine. It therefore
explicitly supports both Linux and Darwin.
For detailed documentation see the
[![GoDoc](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/internal/testing/integration?status.svg)](https://godoc.org/github.com/kubernetes-sigs/controller-runtime/pkg/internal/testing/integration).

View File

@@ -1,74 +0,0 @@
package addr
import (
"fmt"
"net"
"sync"
"time"
)
const (
portReserveTime = 1 * time.Minute
portConflictRetry = 100
)
type portCache struct {
lock sync.Mutex
ports map[int]time.Time
}
func (c *portCache) add(port int) bool {
c.lock.Lock()
defer c.lock.Unlock()
// remove outdated port
for p, t := range c.ports {
if time.Since(t) > portReserveTime {
delete(c.ports, p)
}
}
// try allocating new port
if _, ok := c.ports[port]; ok {
return false
}
c.ports[port] = time.Now()
return true
}
var cache = &portCache{
ports: make(map[int]time.Time),
}
func suggest() (port int, resolvedHost string, err error) {
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
if err != nil {
return
}
l, err := net.ListenTCP("tcp", addr)
if err != nil {
return
}
port = l.Addr().(*net.TCPAddr).Port
defer func() {
err = l.Close()
}()
resolvedHost = addr.IP.String()
return
}
// Suggest suggests an address a process can listen on. It returns
// a tuple consisting of a free port and the hostname resolved to its IP.
// It makes sure that new port allocated does not conflict with old ports
// allocated within 1 minute.
func Suggest() (port int, resolvedHost string, err error) {
for i := 0; i < portConflictRetry; i++ {
port, resolvedHost, err = suggest()
if err != nil {
return
}
if cache.add(port) {
return
}
}
err = fmt.Errorf("no free ports found after %d retries", portConflictRetry)
return
}

View File

@@ -1,177 +0,0 @@
package integration
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// APIServer knows how to run a kubernetes apiserver.
type APIServer struct {
// URL is the address the ApiServer should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// SecurePort is the additional secure port that the APIServer should listen on.
SecurePort int
// Path is the path to the apiserver binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and
// the default test assets directory. See the "Binaries" section above (in
// doc.go) for details.
Path string
// Args is a list of arguments which will passed to the APIServer binary.
// Before they are passed on, they will be evaluated as go-template strings.
// This means you can use fields which are defined and exported on this
// APIServer struct (e.g. "--cert-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the APIServer's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the APIServer will
// be used.
Args []string
// CertDir is a path to a directory containing whatever certificates the
// APIServer will need.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
CertDir string
// EtcdURL is the URL of the Etcd the APIServer should use.
//
// If this is not specified, the Start() method will return an error.
EtcdURL *url.URL
// StartTimeout, StopTimeout specify the time the APIServer is allowed to
// take when starting and stoppping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where APIServer should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (s *APIServer) Start() error {
if s.processState == nil {
if err := s.setProcessState(); err != nil {
return err
}
}
return s.processState.Start(s.Out, s.Err)
}
func (s *APIServer) setProcessState() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}
var err error
s.processState = &internal.ProcessState{}
s.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"kube-apiserver",
s.URL,
s.CertDir,
s.Path,
s.StartTimeout,
s.StopTimeout,
)
if err != nil {
return err
}
// Defaulting the secure port
if s.SecurePort == 0 {
s.SecurePort, _, err = addr.Suggest()
if err != nil {
return err
}
}
s.processState.HealthCheckEndpoint = "/healthz"
s.URL = &s.processState.URL
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout
if err := s.populateAPIServerCerts(); err != nil {
return err
}
s.processState.Args, err = internal.RenderTemplates(
internal.DoAPIServerArgDefaulting(s.Args), s,
)
return err
}
func (s *APIServer) populateAPIServerCerts() error {
_, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt"))
if !os.IsNotExist(statErr) {
return statErr
}
ca, err := internal.NewTinyCA()
if err != nil {
return err
}
certs, err := ca.NewServingCert()
if err != nil {
return err
}
certData, keyData, err := certs.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil {
return err
}
return nil
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
if s.processState != nil {
return s.processState.Stop()
}
return nil
}
// APIServerDefaultArgs exposes the default args for the APIServer so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitly copied here, we don't want to
// allow users to change the internal ones.
var APIServerDefaultArgs = append([]string{}, internal.APIServerDefaultArgs...)

View File

@@ -1,86 +0,0 @@
package integration
import (
"fmt"
"net/url"
"k8s.io/apimachinery/pkg/runtime/serializer"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
// Don't use this for anything else!
var NewTinyCA = internal.NewTinyCA
// ControlPlane is a struct that knows how to start your test control plane.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in
// future.
type ControlPlane struct {
APIServer *APIServer
Etcd *Etcd
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
f.APIServer.EtcdURL = f.Etcd.URL
return f.APIServer.Start()
}
// Stop will stop your control plane processes, and clean up their data.
func (f *ControlPlane) Stop() error {
var errList []error
if f.APIServer != nil {
if err := f.APIServer.Stop(); err != nil {
errList = append(errList, err)
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
errList = append(errList, err)
}
}
return utilerrors.NewAggregate(errList)
}
// APIURL returns the URL you should connect to to talk to your API.
func (f *ControlPlane) APIURL() *url.URL {
return f.APIServer.URL
}
// KubeCtl returns a pre-configured KubeCtl, ready to connect to this
// ControlPlane.
func (f *ControlPlane) KubeCtl() *KubeCtl {
k := &KubeCtl{}
k.Opts = append(k.Opts, fmt.Sprintf("--server=%s", f.APIURL()))
return k
}
// RESTClientConfig returns a pre-configured restconfig, ready to connect to
// this ControlPlane.
func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) {
c := &rest.Config{
Host: f.APIURL().String(),
ContentConfig: rest.ContentConfig{
NegotiatedSerializer: serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs},
},
}
err := rest.SetKubernetesDefaults(c)
return c, err
}

View File

@@ -1,112 +0,0 @@
/*
Package integration implements an integration testing framework for kubernetes.
It provides components for standing up a kubernetes API, against which you can test a
kubernetes client, or other kubernetes components. The lifecycle of the components
needed to provide this API is managed by this framework.
Quickstart
Add something like the following to
your tests:
cp := &integration.ControlPlane{}
cp.Start()
kubeCtl := cp.KubeCtl()
stdout, stderr, err := kubeCtl.Run("get", "pods")
// You can check on err, stdout & stderr and build up
// your tests
cp.Stop()
Components
Currently the framework provides the following components:
ControlPlane: The ControlPlane wraps Etcd & APIServer (see below) and wires
them together correctly. A ControlPlane can be stopped & started and can
provide the URL to connect to the API. The ControlPlane can also be asked for a
KubeCtl which is already correctly configured for this ControlPlane. The
ControlPlane is a good entry point for default setups.
Etcd: Manages an Etcd binary, which can be started, stopped and connected to.
By default Etcd will listen on a random port for http connections and will
create a temporary directory for its data. To configure it differently, see the
Etcd type documentation below.
APIServer: Manages an Kube-APIServer binary, which can be started, stopped and
connected to. By default APIServer will listen on a random port for http
connections and will create a temporary directory to store the (auto-generated)
certificates. To configure it differently, see the APIServer type
documentation below.
KubeCtl: Wraps around a `kubectl` binary and can `Run(...)` arbitrary commands
against a kubernetes control plane.
Binaries
Etcd, APIServer & KubeCtl use the same mechanism to determine which binaries to
use when they get started.
1. If the component is configured with a `Path` the framework tries to run that
binary.
For example:
myEtcd := &Etcd{
Path: "/some/other/etcd",
}
cp := &integration.ControlPlane{
Etcd: myEtcd,
}
cp.Start()
2. If the Path field on APIServer, Etcd or KubeCtl is left unset and an
environment variable named `TEST_ASSET_KUBE_APISERVER`, `TEST_ASSET_ETCD` or
`TEST_ASSET_KUBECTL` is set, its value is used as a path to the binary for the
APIServer, Etcd or KubeCtl.
3. If neither the `Path` field, nor the environment variable is set, the
framework tries to use the binaries `kube-apiserver`, `etcd` or `kubectl` in
the directory `${FRAMEWORK_DIR}/assets/bin/`.
Arguments for Etcd and APIServer
Those components will start without any configuration. However, if you want or
need to, you can override certain configuration -- one of which are the
arguments used when calling the binary.
When you choose to specify your own set of arguments, those won't be appended
to the default set of arguments, it is your responsibility to provide all the
arguments needed for the binary to start successfully.
However, the default arguments for APIServer and Etcd are exported as
`APIServerDefaultArgs` and `EtcdDefaultArgs` from this package. Treat those
variables as read-only constants. Internally we have a set of default
arguments for defaulting, the `APIServerDefaultArgs` and `EtcdDefaultArgs` are
just copies of those. So when you override them you loose access to the actual
internal default arguments, but your override won't affect the defaulting.
All arguments are interpreted as go templates. Those templates have access to
all exported fields of the `APIServer`/`Etcd` struct. It does not matter if
those fields where explicitly set up or if they were defaulted by calling the
`Start()` method, the template evaluation runs just before the binary is
executed and right after the defaulting of all the struct's fields has
happened.
// When you want to append additional arguments ...
etcd := &Etcd{
// Additional custom arguments will appended to the set of default
// arguments
Args: append(EtcdDefaultArgs, "--additional=arg"),
DataDir: "/my/special/data/dir",
}
// When you want to use a custom set of arguments ...
etcd := &Etcd{
// Only custom arguments will be passed to the binary
Args: []string{"--one=1", "--two=2", "--three=3"},
DataDir: "/my/special/data/dir",
}
*/
package integration

View File

@@ -1,114 +0,0 @@
package integration
import (
"io"
"time"
"net/url"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// Etcd knows how to run an etcd server.
type Etcd struct {
// URL is the address the Etcd should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// Path is the path to the etcd binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_ETCD environment variable, and the default
// test assets directory. See the "Binaries" section above (in doc.go) for
// details.
Path string
// Args is a list of arguments which will passed to the Etcd binary. Before
// they are passed on, the`y will be evaluated as go-template strings. This
// means you can use fields which are defined and exported on this Etcd
// struct (e.g. "--data-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the Etcd's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the Etcd will be
// used.
Args []string
// DataDir is a path to a directory in which etcd can store its state.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
DataDir string
// StartTimeout, StopTimeout specify the time the Etcd is allowed to
// take when starting and stopping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where Etcd should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *internal.ProcessState
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
func (e *Etcd) Start() error {
if e.processState == nil {
if err := e.setProcessState(); err != nil {
return err
}
}
return e.processState.Start(e.Out, e.Err)
}
func (e *Etcd) setProcessState() error {
var err error
e.processState = &internal.ProcessState{}
e.processState.DefaultedProcessInput, err = internal.DoDefaulting(
"etcd",
e.URL,
e.DataDir,
e.Path,
e.StartTimeout,
e.StopTimeout,
)
if err != nil {
return err
}
e.processState.StartMessage = internal.GetEtcdStartMessage(e.processState.URL)
e.URL = &e.processState.URL
e.DataDir = e.processState.Dir
e.Path = e.processState.Path
e.StartTimeout = e.processState.StartTimeout
e.StopTimeout = e.processState.StopTimeout
e.processState.Args, err = internal.RenderTemplates(
internal.DoEtcdArgDefaulting(e.Args), e,
)
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the DataDir if necessary.
func (e *Etcd) Stop() error {
return e.processState.Stop()
}
// EtcdDefaultArgs exposes the default args for Etcd so that you
// can use those to append your own additional arguments.
//
// The internal default arguments are explicitly copied here, we don't want to
// allow users to change the internal ones.
var EtcdDefaultArgs = append([]string{}, internal.EtcdDefaultArgs...)

View File

@@ -1,27 +0,0 @@
package internal
// APIServerDefaultArgs allow tests to run offline, by preventing API server from attempting to
// use default route to determine its --advertise-address.
var APIServerDefaultArgs = []string{
"--advertise-address=127.0.0.1",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{ end }}",
"--insecure-bind-address={{ if .URL }}{{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
// we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one
// in normal apiserver operation this SA is created by controller, but that is not run in integration environment
"--disable-admission-plugins=ServiceAccount",
"--service-cluster-ip-range=10.0.0.0/24",
"--allow-privileged=true",
}
// DoAPIServerArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise,
// it will return the same []string arg passed as param.
func DoAPIServerArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return APIServerDefaultArgs
}

View File

@@ -1,29 +0,0 @@
package internal
import (
"bytes"
"html/template"
)
// RenderTemplates returns an []string to render the templates
func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) {
var t *template.Template
for _, arg := range argTemplates {
t, err = template.New(arg).Parse(arg)
if err != nil {
args = nil
return
}
buf := &bytes.Buffer{}
err = t.Execute(buf, data)
if err != nil {
args = nil
return
}
args = append(args, buf.String())
}
return
}

View File

@@ -1,35 +0,0 @@
package internal
import (
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
var assetsPath string
func init() {
_, thisFile, _, ok := runtime.Caller(0)
if !ok {
panic("Could not determine the path of the BinPathFinder")
}
assetsPath = filepath.Join(filepath.Dir(thisFile), "..", "assets", "bin")
}
// BinPathFinder checks the an environment variable, derived from the symbolic name,
// and falls back to a default assets location when this variable is not set
func BinPathFinder(symbolicName string) (binPath string) {
punctuationPattern := regexp.MustCompile("[^A-Z0-9]+")
sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_")
leadingNumberPattern := regexp.MustCompile("^[0-9]+")
sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "")
envVar := "TEST_ASSET_" + sanitizedName
if val, ok := os.LookupEnv(envVar); ok {
return val
}
return filepath.Join(assetsPath, symbolicName)
}

View File

@@ -1,45 +0,0 @@
package internal
import (
"net/url"
)
// EtcdDefaultArgs allow tests to run offline, by preventing API server from attempting to
// use default route to determine its urls.
var EtcdDefaultArgs = []string{
"--listen-peer-urls=http://localhost:0",
"--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--data-dir={{ .DataDir }}",
}
// DoEtcdArgDefaulting will set default values to allow tests to run offline when the args are not informed. Otherwise,
// it will return the same []string arg passed as param.
func DoEtcdArgDefaulting(args []string) []string {
if len(args) != 0 {
return args
}
return EtcdDefaultArgs
}
// isSecureScheme returns false when the schema is insecure.
func isSecureScheme(scheme string) bool {
// https://github.com/coreos/etcd/blob/d9deeff49a080a88c982d328ad9d33f26d1ad7b6/pkg/transport/listener.go#L53
if scheme == "https" || scheme == "unixs" {
return true
}
return false
}
// GetEtcdStartMessage returns an start message to inform if the client is or not insecure.
// It will return true when the URL informed has the scheme == "https" || scheme == "unixs"
func GetEtcdStartMessage(listenURL url.URL) string {
if isSecureScheme(listenURL.Scheme) {
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L167
return "serving client requests on "
}
// https://github.com/coreos/etcd/blob/a7f1fbe00ec216fcb3a1919397a103b41dca8413/embed/serve.go#L124
return "serving insecure client requests on "
}

View File

@@ -1,225 +0,0 @@
package internal
import (
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"strconv"
"time"
"github.com/onsi/gomega/gbytes"
"github.com/onsi/gomega/gexec"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/addr"
)
// ProcessState define the state of the process.
type ProcessState struct {
DefaultedProcessInput
Session *gexec.Session
// Healthcheck Endpoint. If we get http.StatusOK from this endpoint, we
// assume the process is ready to operate. E.g. "/healthz". If this is set,
// we ignore StartMessage.
HealthCheckEndpoint string
// HealthCheckPollInterval is the interval which will be used for polling the
// HealthCheckEndpoint.
// If left empty it will default to 100 Milliseconds.
HealthCheckPollInterval time.Duration
// StartMessage is the message to wait for on stderr. If we receive this
// message, we assume the process is ready to operate. Ignored if
// HealthCheckEndpoint is specified.
//
// The usage of StartMessage is discouraged, favour HealthCheckEndpoint
// instead!
//
// Deprecated: Use HealthCheckEndpoint in favour of StartMessage
StartMessage string
Args []string
// ready holds wether the process is currently in ready state (hit the ready condition) or not.
// It will be set to true on a successful `Start()` and set to false on a successful `Stop()`
ready bool
}
// DefaultedProcessInput defines the default process input required to perform the test.
type DefaultedProcessInput struct {
URL url.URL
Dir string
DirNeedsCleaning bool
Path string
StopTimeout time.Duration
StartTimeout time.Duration
}
// DoDefaulting sets the default configuration according to the data informed and return an DefaultedProcessInput
// and an error if some requirement was not informed.
func DoDefaulting(
name string,
listenURL *url.URL,
dir string,
path string,
startTimeout time.Duration,
stopTimeout time.Duration,
) (DefaultedProcessInput, error) {
defaults := DefaultedProcessInput{
Dir: dir,
Path: path,
StartTimeout: startTimeout,
StopTimeout: stopTimeout,
}
if listenURL == nil {
port, host, err := addr.Suggest()
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.URL = url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
} else {
defaults.URL = *listenURL
}
if dir == "" {
newDir, err := ioutil.TempDir("", "k8s_test_framework_")
if err != nil {
return DefaultedProcessInput{}, err
}
defaults.Dir = newDir
defaults.DirNeedsCleaning = true
}
if path == "" {
if name == "" {
return DefaultedProcessInput{}, fmt.Errorf("must have at least one of name or path")
}
defaults.Path = BinPathFinder(name)
}
if startTimeout == 0 {
defaults.StartTimeout = 20 * time.Second
}
if stopTimeout == 0 {
defaults.StopTimeout = 20 * time.Second
}
return defaults, nil
}
type stopChannel chan struct{}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (ps *ProcessState) Start(stdout, stderr io.Writer) (err error) {
if ps.ready {
return nil
}
command := exec.Command(ps.Path, ps.Args...)
ready := make(chan bool)
timedOut := time.After(ps.StartTimeout)
var pollerStopCh stopChannel
if ps.HealthCheckEndpoint != "" {
healthCheckURL := ps.URL
healthCheckURL.Path = ps.HealthCheckEndpoint
pollerStopCh = make(stopChannel)
go pollURLUntilOK(healthCheckURL, ps.HealthCheckPollInterval, ready, pollerStopCh)
} else {
startDetectStream := gbytes.NewBuffer()
ready = startDetectStream.Detect(ps.StartMessage)
stderr = safeMultiWriter(stderr, startDetectStream)
}
ps.Session, err = gexec.Start(command, stdout, stderr)
if err != nil {
return err
}
select {
case <-ready:
ps.ready = true
return nil
case <-timedOut:
if pollerStopCh != nil {
close(pollerStopCh)
}
if ps.Session != nil {
ps.Session.Terminate()
}
return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path))
}
}
func safeMultiWriter(writers ...io.Writer) io.Writer {
safeWriters := []io.Writer{}
for _, w := range writers {
if w != nil {
safeWriters = append(safeWriters, w)
}
}
return io.MultiWriter(safeWriters...)
}
func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) {
if interval <= 0 {
interval = 100 * time.Millisecond
}
for {
res, err := http.Get(url.String())
if err == nil {
res.Body.Close()
if res.StatusCode == http.StatusOK {
ready <- true
return
}
}
select {
case <-stopCh:
return
default:
time.Sleep(interval)
}
}
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (ps *ProcessState) Stop() error {
if ps.Session == nil {
return nil
}
// gexec's Session methods (Signal, Kill, ...) do not check if the Process is
// nil, so we are doing this here for now.
// This should probably be fixed in gexec.
if ps.Session.Command.Process == nil {
return nil
}
detectedStop := ps.Session.Terminate().Exited
timedOut := time.After(ps.StopTimeout)
select {
case <-detectedStop:
break
case <-timedOut:
return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path))
}
ps.ready = false
if ps.DirNeedsCleaning {
return os.RemoveAll(ps.Dir)
}
return nil
}

View File

@@ -1,47 +0,0 @@
package integration
import (
"bytes"
"io"
"os/exec"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/integration/internal"
)
// KubeCtl is a wrapper around the kubectl binary.
type KubeCtl struct {
// Path where the kubectl binary can be found.
//
// If this is left empty, we will attempt to locate a binary, by checking for
// the TEST_ASSET_KUBECTL environment variable, and the default test assets
// directory. See the "Binaries" section above (in doc.go) for details.
Path string
// Opts can be used to configure additional flags which will be used each
// time the wrapped binary is called.
//
// For example, you might want to use this to set the URL of the APIServer to
// connect to.
Opts []string
}
// Run executes the wrapped binary with some preconfigured options and the
// arguments given to this method. It returns Readers for the stdout and
// stderr.
func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) {
if k.Path == "" {
k.Path = internal.BinPathFinder("kubectl")
}
stdoutBuffer := &bytes.Buffer{}
stderrBuffer := &bytes.Buffer{}
allArgs := append(k.Opts, args...)
cmd := exec.Command(k.Path, allArgs...)
cmd.Stdout = stdoutBuffer
cmd.Stderr = stderrBuffer
err = cmd.Run()
return stdoutBuffer, stderrBuffer, err
}

View File

@@ -0,0 +1,340 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package process
import (
"bytes"
"html/template"
"sort"
"strings"
)
// RenderTemplates returns an []string to render the templates
//
// Deprecated: will be removed in favor of Arguments.
func RenderTemplates(argTemplates []string, data interface{}) (args []string, err error) {
var t *template.Template
for _, arg := range argTemplates {
t, err = template.New(arg).Parse(arg)
if err != nil {
args = nil
return
}
buf := &bytes.Buffer{}
err = t.Execute(buf, data)
if err != nil {
args = nil
return
}
args = append(args, buf.String())
}
return
}
// SliceToArguments converts a slice of arguments to structured arguments,
// appending each argument that starts with `--` and contains an `=` to the
// argument set (ignoring defaults), returning the rest.
//
// Deprecated: will be removed when RenderTemplates is removed.
func SliceToArguments(sliceArgs []string, args *Arguments) []string {
var rest []string
for i, arg := range sliceArgs {
if arg == "--" {
rest = append(rest, sliceArgs[i:]...)
return rest
}
// skip non-flag arguments, skip arguments w/o equals because we
// can't tell if the next argument should take a value
if !strings.HasPrefix(arg, "--") || !strings.Contains(arg, "=") {
rest = append(rest, arg)
continue
}
parts := strings.SplitN(arg[2:], "=", 2)
name := parts[0]
val := parts[1]
args.AppendNoDefaults(name, val)
}
return rest
}
// TemplateDefaults specifies defaults to be used for joining structured arguments with templates.
//
// Deprecated: will be removed when RenderTemplates is removed.
type TemplateDefaults struct {
// Data will be used to render the template.
Data interface{}
// Defaults will be used to default structured arguments if no template is passed.
Defaults map[string][]string
// MinimalDefaults will be used to default structured arguments if a template is passed.
// Use this for flags which *must* be present.
MinimalDefaults map[string][]string // for api server service-cluster-ip-range
}
// TemplateAndArguments joins structured arguments and non-structured arguments, preserving existing
// behavior. Namely:
//
// 1. if templ has len > 0, it will be rendered against data
// 2. the rendered template values that look like `--foo=bar` will be split
// and appended to args, the rest will be kept around
// 3. the given args will be rendered as string form. If a template is given,
// no defaults will be used, otherwise defaults will be used
// 4. a result of [args..., rest...] will be returned
//
// It returns the resulting rendered arguments, plus the arguments that were
// not transferred to `args` during rendering.
//
// Deprecated: will be removed when RenderTemplates is removed.
func TemplateAndArguments(templ []string, args *Arguments, data TemplateDefaults) (allArgs []string, nonFlagishArgs []string, err error) {
if len(templ) == 0 { // 3 & 4 (no template case)
return args.AsStrings(data.Defaults), nil, nil
}
// 1: render the template
rendered, err := RenderTemplates(templ, data.Data)
if err != nil {
return nil, nil, err
}
// 2: filter out structured args and add them to args
rest := SliceToArguments(rendered, args)
// 3 (template case): render structured args, no defaults (matching the
// legacy case where if Args was specified, no defaults were used)
res := args.AsStrings(data.MinimalDefaults)
// 4: return the rendered structured args + all non-structured args
return append(res, rest...), rest, nil
}
// EmptyArguments constructs an empty set of flags with no defaults.
func EmptyArguments() *Arguments {
return &Arguments{
values: make(map[string]Arg),
}
}
// Arguments are structured, overridable arguments.
// Each Arguments object contains some set of default arguments, which may
// be appended to, or overridden.
//
// When ready, you can serialize them to pass to exec.Command and friends using
// AsStrings.
//
// All flag-setting methods return the *same* instance of Arguments so that you
// can chain calls.
type Arguments struct {
// values contains the user-set values for the arguments.
// `values[key] = dontPass` means "don't pass this flag"
// `values[key] = passAsName` means "pass this flag without args like --key`
// `values[key] = []string{a, b, c}` means "--key=a --key=b --key=c`
// any values not explicitly set here will be copied from defaults on final rendering.
values map[string]Arg
}
// Arg is an argument that has one or more values,
// and optionally falls back to default values.
type Arg interface {
// Append adds new values to this argument, returning
// a new instance contain the new value. The intermediate
// argument should generally be assumed to be consumed.
Append(vals ...string) Arg
// Get returns the full set of values, optionally including
// the passed in defaults. If it returns nil, this will be
// skipped. If it returns a non-nil empty slice, it'll be
// assumed that the argument should be passed as name-only.
Get(defaults []string) []string
}
type userArg []string
func (a userArg) Append(vals ...string) Arg {
return userArg(append(a, vals...)) //nolint:unconvert
}
func (a userArg) Get(_ []string) []string {
return []string(a)
}
type defaultedArg []string
func (a defaultedArg) Append(vals ...string) Arg {
return defaultedArg(append(a, vals...)) //nolint:unconvert
}
func (a defaultedArg) Get(defaults []string) []string {
res := append([]string(nil), defaults...)
return append(res, a...)
}
type dontPassArg struct{}
func (a dontPassArg) Append(vals ...string) Arg {
return userArg(vals)
}
func (dontPassArg) Get(_ []string) []string {
return nil
}
type passAsNameArg struct{}
func (a passAsNameArg) Append(_ ...string) Arg {
return passAsNameArg{}
}
func (passAsNameArg) Get(_ []string) []string {
return []string{}
}
var (
// DontPass indicates that the given argument will not actually be
// rendered.
DontPass Arg = dontPassArg{}
// PassAsName indicates that the given flag will be passed as `--key`
// without any value.
PassAsName Arg = passAsNameArg{}
)
// AsStrings serializes this set of arguments to a slice of strings appropriate
// for passing to exec.Command and friends, making use of the given defaults
// as indicated for each particular argument.
//
// - Any flag in defaults that's not in Arguments will be present in the output
// - Any flag that's present in Arguments will be passed the corresponding
// defaults to do with as it will (ignore, append-to, suppress, etc).
func (a *Arguments) AsStrings(defaults map[string][]string) []string {
// sort for deterministic ordering
keysInOrder := make([]string, 0, len(defaults)+len(a.values))
for key := range defaults {
if _, userSet := a.values[key]; userSet {
continue
}
keysInOrder = append(keysInOrder, key)
}
for key := range a.values {
keysInOrder = append(keysInOrder, key)
}
sort.Strings(keysInOrder)
var res []string
for _, key := range keysInOrder {
vals := a.Get(key).Get(defaults[key])
switch {
case vals == nil: // don't pass
continue
case len(vals) == 0: // pass as name
res = append(res, "--"+key)
default:
for _, val := range vals {
res = append(res, "--"+key+"="+val)
}
}
}
return res
}
// Get returns the value of the given flag. If nil,
// it will not be passed in AsString, otherwise:
//
// len == 0 --> `--key`, len > 0 --> `--key=val1 --key=val2 ...`.
func (a *Arguments) Get(key string) Arg {
if vals, ok := a.values[key]; ok {
return vals
}
return defaultedArg(nil)
}
// Enable configures the given key to be passed as a "name-only" flag,
// like, `--key`.
func (a *Arguments) Enable(key string) *Arguments {
a.values[key] = PassAsName
return a
}
// Disable prevents this flag from be passed.
func (a *Arguments) Disable(key string) *Arguments {
a.values[key] = DontPass
return a
}
// Append adds additional values to this flag. If this flag has
// yet to be set, initial values will include defaults. If you want
// to intentionally ignore defaults/start from scratch, call AppendNoDefaults.
//
// Multiple values will look like `--key=value1 --key=value2 ...`.
func (a *Arguments) Append(key string, values ...string) *Arguments {
vals, present := a.values[key]
if !present {
vals = defaultedArg{}
}
a.values[key] = vals.Append(values...)
return a
}
// AppendNoDefaults adds additional values to this flag. However,
// unlike Append, it will *not* copy values from defaults.
func (a *Arguments) AppendNoDefaults(key string, values ...string) *Arguments {
vals, present := a.values[key]
if !present {
vals = userArg{}
}
a.values[key] = vals.Append(values...)
return a
}
// Set resets the given flag to the specified values, ignoring any existing
// values or defaults.
func (a *Arguments) Set(key string, values ...string) *Arguments {
a.values[key] = userArg(values)
return a
}
// SetRaw sets the given flag to the given Arg value directly. Use this if
// you need to do some complicated deferred logic or something.
//
// Otherwise behaves like Set.
func (a *Arguments) SetRaw(key string, val Arg) *Arguments {
a.values[key] = val
return a
}
// FuncArg is a basic implementation of Arg that can be used for custom argument logic,
// like pulling values out of APIServer, or dynamically calculating values just before
// launch.
//
// The given function will be mapped directly to Arg#Get, and will generally be
// used in conjunction with SetRaw. For example, to set `--some-flag` to the
// API server's CertDir, you could do:
//
// server.Configure().SetRaw("--some-flag", FuncArg(func(defaults []string) []string {
// return []string{server.CertDir}
// }))
//
// FuncArg ignores Appends; if you need to support appending values too, consider implementing
// Arg directly.
type FuncArg func([]string) []string
// Append is a no-op for FuncArg, and just returns itself.
func (a FuncArg) Append(vals ...string) Arg { return a }
// Get delegates functionality to the FuncArg function itself.
func (a FuncArg) Get(defaults []string) []string {
return a(defaults)
}

View File

@@ -0,0 +1,70 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package process
import (
"os"
"path/filepath"
"regexp"
"strings"
)
const (
// EnvAssetsPath is the environment variable that stores the global test
// binary location override.
EnvAssetsPath = "KUBEBUILDER_ASSETS"
// EnvAssetOverridePrefix is the environment variable prefix for per-binary
// location overrides.
EnvAssetOverridePrefix = "TEST_ASSET_"
// AssetsDefaultPath is the default location to look for test binaries in,
// if no override was provided.
AssetsDefaultPath = "/usr/local/kubebuilder/bin"
)
// BinPathFinder finds the path to the given named binary, using the following locations
// in order of precedence (highest first). Notice that the various env vars only need
// to be set -- the asset is not checked for existence on the filesystem.
//
// 1. TEST_ASSET_{tr/a-z-/A-Z_/} (if set; asset overrides -- EnvAssetOverridePrefix)
// 1. KUBEBUILDER_ASSETS (if set; global asset path -- EnvAssetsPath)
// 3. assetDirectory (if set; per-config asset directory)
// 4. /usr/local/kubebuilder/bin (AssetsDefaultPath).
func BinPathFinder(symbolicName, assetDirectory string) (binPath string) {
punctuationPattern := regexp.MustCompile("[^A-Z0-9]+")
sanitizedName := punctuationPattern.ReplaceAllString(strings.ToUpper(symbolicName), "_")
leadingNumberPattern := regexp.MustCompile("^[0-9]+")
sanitizedName = leadingNumberPattern.ReplaceAllString(sanitizedName, "")
envVar := EnvAssetOverridePrefix + sanitizedName
// TEST_ASSET_XYZ
if val, ok := os.LookupEnv(envVar); ok {
return val
}
// KUBEBUILDER_ASSETS
if val, ok := os.LookupEnv(EnvAssetsPath); ok {
return filepath.Join(val, symbolicName)
}
// assetDirectory
if assetDirectory != "" {
return filepath.Join(assetDirectory, symbolicName)
}
// default path
return filepath.Join(AssetsDefaultPath, symbolicName)
}

View File

@@ -0,0 +1,277 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package process
import (
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"regexp"
"sync"
"syscall"
"time"
)
// ListenAddr represents some listening address and port.
type ListenAddr struct {
Address string
Port string
}
// URL returns a URL for this address with the given scheme and subpath.
func (l *ListenAddr) URL(scheme string, path string) *url.URL {
return &url.URL{
Scheme: scheme,
Host: l.HostPort(),
Path: path,
}
}
// HostPort returns the joined host-port pair for this address.
func (l *ListenAddr) HostPort() string {
return net.JoinHostPort(l.Address, l.Port)
}
// HealthCheck describes the information needed to health-check a process via
// some health-check URL.
type HealthCheck struct {
url.URL
// HealthCheckPollInterval is the interval which will be used for polling the
// endpoint described by Host, Port, and Path.
//
// If left empty it will default to 100 Milliseconds.
PollInterval time.Duration
}
// State define the state of the process.
type State struct {
Cmd *exec.Cmd
// HealthCheck describes how to check if this process is up. If we get an http.StatusOK,
// we assume the process is ready to operate.
//
// For example, the /healthz endpoint of the k8s API server, or the /health endpoint of etcd.
HealthCheck HealthCheck
Args []string
StopTimeout time.Duration
StartTimeout time.Duration
Dir string
DirNeedsCleaning bool
Path string
// ready holds wether the process is currently in ready state (hit the ready condition) or not.
// It will be set to true on a successful `Start()` and set to false on a successful `Stop()`
ready bool
// waitDone is closed when our call to wait finishes up, and indicates that
// our process has terminated.
waitDone chan struct{}
errMu sync.Mutex
exitErr error
exited bool
}
// Init sets up this process, configuring binary paths if missing, initializing
// temporary directories, etc.
//
// This defaults all defaultable fields.
func (ps *State) Init(name string) error {
if ps.Path == "" {
if name == "" {
return fmt.Errorf("must have at least one of name or path")
}
ps.Path = BinPathFinder(name, "")
}
if ps.Dir == "" {
newDir, err := ioutil.TempDir("", "k8s_test_framework_")
if err != nil {
return err
}
ps.Dir = newDir
ps.DirNeedsCleaning = true
}
if ps.StartTimeout == 0 {
ps.StartTimeout = 20 * time.Second
}
if ps.StopTimeout == 0 {
ps.StopTimeout = 20 * time.Second
}
return nil
}
type stopChannel chan struct{}
// CheckFlag checks the help output of this command for the presence of the given flag, specified
// without the leading `--` (e.g. `CheckFlag("insecure-port")` checks for `--insecure-port`),
// returning true if the flag is present.
func (ps *State) CheckFlag(flag string) (bool, error) {
cmd := exec.Command(ps.Path, "--help")
outContents, err := cmd.CombinedOutput()
if err != nil {
return false, fmt.Errorf("unable to run command %q to check for flag %q: %w", ps.Path, flag, err)
}
pat := `(?m)^\s*--` + flag + `\b` // (m --> multi-line --> ^ matches start of line)
matched, err := regexp.Match(pat, outContents)
if err != nil {
return false, fmt.Errorf("unable to check command %q for flag %q in help output: %w", ps.Path, flag, err)
}
return matched, nil
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (ps *State) Start(stdout, stderr io.Writer) (err error) {
if ps.ready {
return nil
}
ps.Cmd = exec.Command(ps.Path, ps.Args...)
ps.Cmd.Stdout = stdout
ps.Cmd.Stderr = stderr
ready := make(chan bool)
timedOut := time.After(ps.StartTimeout)
pollerStopCh := make(stopChannel)
go pollURLUntilOK(ps.HealthCheck.URL, ps.HealthCheck.PollInterval, ready, pollerStopCh)
ps.waitDone = make(chan struct{})
if err := ps.Cmd.Start(); err != nil {
ps.errMu.Lock()
defer ps.errMu.Unlock()
ps.exited = true
return err
}
go func() {
defer close(ps.waitDone)
err := ps.Cmd.Wait()
ps.errMu.Lock()
defer ps.errMu.Unlock()
ps.exitErr = err
ps.exited = true
}()
select {
case <-ready:
ps.ready = true
return nil
case <-ps.waitDone:
if pollerStopCh != nil {
close(pollerStopCh)
}
return fmt.Errorf("timeout waiting for process %s to start successfully "+
"(it may have failed to start, or stopped unexpectedly before becoming ready)",
path.Base(ps.Path))
case <-timedOut:
if pollerStopCh != nil {
close(pollerStopCh)
}
if ps.Cmd != nil {
// intentionally ignore this -- we might've crashed, failed to start, etc
ps.Cmd.Process.Signal(syscall.SIGTERM) //nolint:errcheck
}
return fmt.Errorf("timeout waiting for process %s to start", path.Base(ps.Path))
}
}
// Exited returns true if the process exited, and may also
// return an error (as per Cmd.Wait) if the process did not
// exit with error code 0.
func (ps *State) Exited() (bool, error) {
ps.errMu.Lock()
defer ps.errMu.Unlock()
return ps.exited, ps.exitErr
}
func pollURLUntilOK(url url.URL, interval time.Duration, ready chan bool, stopCh stopChannel) {
client := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
// there's probably certs *somewhere*,
// but it's fine to just skip validating
// them for health checks during testing
InsecureSkipVerify: true, //nolint:gosec
},
},
}
if interval <= 0 {
interval = 100 * time.Millisecond
}
for {
res, err := client.Get(url.String())
if err == nil {
res.Body.Close()
if res.StatusCode == http.StatusOK {
ready <- true
return
}
}
select {
case <-stopCh:
return
default:
time.Sleep(interval)
}
}
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (ps *State) Stop() error {
// Always clear the directory if we need to.
defer func() {
if ps.DirNeedsCleaning {
_ = os.RemoveAll(ps.Dir)
}
}()
if ps.Cmd == nil {
return nil
}
if done, _ := ps.Exited(); done {
return nil
}
if err := ps.Cmd.Process.Signal(syscall.SIGTERM); err != nil {
return fmt.Errorf("unable to signal for process %s to stop: %w", ps.Path, err)
}
timedOut := time.After(ps.StopTimeout)
select {
case <-ps.waitDone:
break
case <-timedOut:
return fmt.Errorf("timeout waiting for process %s to stop", path.Base(ps.Path))
}
ps.ready = false
return nil
}

View File

@@ -15,7 +15,7 @@ limitations under the License.
*/
/*
Package leaderelection contains a constructors for a leader election resource lock.
Package leaderelection contains a constructor for a leader election resource lock.
This is used to ensure that multiple copies of a controller manager can be run with
only one active set of controllers, for active-passive HA.

View File

@@ -31,28 +31,38 @@ import (
const inClusterNamespacePath = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
// Options provides the required configuration to create a new resource lock
// Options provides the required configuration to create a new resource lock.
type Options struct {
// LeaderElection determines whether or not to use leader election when
// starting the manager.
LeaderElection bool
// LeaderElectionResourceLock determines which resource lock to use for leader election,
// defaults to "configmapsleases".
LeaderElectionResourceLock string
// LeaderElectionNamespace determines the namespace in which the leader
// election configmap will be created.
// election resource will be created.
LeaderElectionNamespace string
// LeaderElectionID determines the name of the configmap that leader election
// LeaderElectionID determines the name of the resource that leader election
// will use for holding the leader lock.
LeaderElectionID string
}
// NewResourceLock creates a new config map resource lock for use in a leader
// election loop
// NewResourceLock creates a new resource lock for use in a leader election loop.
func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, options Options) (resourcelock.Interface, error) {
if !options.LeaderElection {
return nil, nil
}
// Default resource lock to "configmapsleases". We must keep this default until we are sure all controller-runtime
// users have upgraded from the original default ConfigMap lock to a controller-runtime version that has this new
// default. Many users of controller-runtime skip versions, so we should be extremely conservative here.
if options.LeaderElectionResourceLock == "" {
options.LeaderElectionResourceLock = resourcelock.ConfigMapsLeasesResourceLock
}
// LeaderElectionID must be provided to prevent clashes
if options.LeaderElectionID == "" {
return nil, errors.New("LeaderElectionID must be configured")
@@ -80,8 +90,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
return nil, err
}
// TODO(JoelSpeed): switch to leaderelection object in 1.12
return resourcelock.New(resourcelock.ConfigMapsResourceLock,
return resourcelock.New(options.LeaderElectionResourceLock,
options.LeaderElectionNamespace,
options.LeaderElectionID,
client.CoreV1(),
@@ -95,8 +104,7 @@ func NewResourceLock(config *rest.Config, recorderProvider recorder.Provider, op
func getInClusterNamespace() (string, error) {
// Check whether the namespace file exists.
// If not, we are not running in cluster so can't guess the namespace.
_, err := os.Stat(inClusterNamespacePath)
if os.IsNotExist(err) {
if _, err := os.Stat(inClusterNamespacePath); os.IsNotExist(err) {
return "", fmt.Errorf("not running in-cluster, please specify LeaderElectionNamespace")
} else if err != nil {
return "", fmt.Errorf("error checking namespace file: %w", err)

View File

@@ -29,11 +29,11 @@ type loggerPromise struct {
childPromises []*loggerPromise
promisesLock sync.Mutex
name *string
tags []interface{}
name *string
tags []interface{}
level int
}
// WithName provides a new Logger with the name appended
func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromise {
res := &loggerPromise{
logger: l,
@@ -47,7 +47,7 @@ func (p *loggerPromise) WithName(l *DelegatingLogger, name string) *loggerPromis
return res
}
// WithValues provides a new Logger with the tags appended
// WithValues provides a new Logger with the tags appended.
func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *loggerPromise {
res := &loggerPromise{
logger: l,
@@ -61,7 +61,20 @@ func (p *loggerPromise) WithValues(l *DelegatingLogger, tags ...interface{}) *lo
return res
}
// Fulfill instantiates the Logger with the provided logger
func (p *loggerPromise) V(l *DelegatingLogger, level int) *loggerPromise {
res := &loggerPromise{
logger: l,
level: level,
promisesLock: sync.Mutex{},
}
p.promisesLock.Lock()
defer p.promisesLock.Unlock()
p.childPromises = append(p.childPromises, res)
return res
}
// Fulfill instantiates the Logger with the provided logger.
func (p *loggerPromise) Fulfill(parentLogger logr.Logger) {
var logger = parentLogger
if p.name != nil {
@@ -71,9 +84,14 @@ func (p *loggerPromise) Fulfill(parentLogger logr.Logger) {
if p.tags != nil {
logger = logger.WithValues(p.tags...)
}
if p.level != 0 {
logger = logger.V(p.level)
}
p.logger.Logger = logger
p.logger.lock.Lock()
p.logger.logger = logger
p.logger.promise = nil
p.logger.lock.Unlock()
for _, childPromise := range p.childPromises {
childPromise.Fulfill(logger)
@@ -86,30 +104,91 @@ func (p *loggerPromise) Fulfill(parentLogger logr.Logger) {
// logger. It expects to have *some* logr.Logger set at all times (generally
// a no-op logger before the promises are fulfilled).
type DelegatingLogger struct {
logr.Logger
lock sync.RWMutex
logger logr.Logger
promise *loggerPromise
}
// WithName provides a new Logger with the name appended
func (l *DelegatingLogger) WithName(name string) logr.Logger {
// Enabled tests whether this Logger is enabled. For example, commandline
// flags might be used to set the logging verbosity and disable some info
// logs.
func (l *DelegatingLogger) Enabled() bool {
l.lock.RLock()
defer l.lock.RUnlock()
return l.logger.Enabled()
}
// Info logs a non-error message with the given key/value pairs as context.
//
// The msg argument should be used to add some constant description to
// the log line. The key/value pairs can then be used to add additional
// variable information. The key/value pairs should alternate string
// keys and arbitrary values.
func (l *DelegatingLogger) Info(msg string, keysAndValues ...interface{}) {
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Info(msg, keysAndValues...)
}
// Error logs an error, with the given message and key/value pairs as context.
// It functions similarly to calling Info with the "error" named value, but may
// have unique behavior, and should be preferred for logging errors (see the
// package documentations for more information).
//
// The msg field should be used to add context to any underlying error,
// while the err field should be used to attach the actual error that
// triggered this log line, if present.
func (l *DelegatingLogger) Error(err error, msg string, keysAndValues ...interface{}) {
l.lock.RLock()
defer l.lock.RUnlock()
l.logger.Error(err, msg, keysAndValues...)
}
// V returns an Logger value for a specific verbosity level, relative to
// this Logger. In other words, V values are additive. V higher verbosity
// level means a log message is less important. It's illegal to pass a log
// level less than zero.
func (l *DelegatingLogger) V(level int) logr.Logger {
l.lock.RLock()
defer l.lock.RUnlock()
if l.promise == nil {
return l.Logger.WithName(name)
return l.logger.V(level)
}
res := &DelegatingLogger{Logger: l.Logger}
res := &DelegatingLogger{logger: l.logger}
promise := l.promise.V(res, level)
res.promise = promise
return res
}
// WithName provides a new Logger with the name appended.
func (l *DelegatingLogger) WithName(name string) logr.Logger {
l.lock.RLock()
defer l.lock.RUnlock()
if l.promise == nil {
return l.logger.WithName(name)
}
res := &DelegatingLogger{logger: l.logger}
promise := l.promise.WithName(res, name)
res.promise = promise
return res
}
// WithValues provides a new Logger with the tags appended
// WithValues provides a new Logger with the tags appended.
func (l *DelegatingLogger) WithValues(tags ...interface{}) logr.Logger {
l.lock.RLock()
defer l.lock.RUnlock()
if l.promise == nil {
return l.Logger.WithValues(tags...)
return l.logger.WithValues(tags...)
}
res := &DelegatingLogger{Logger: l.Logger}
res := &DelegatingLogger{logger: l.logger}
promise := l.promise.WithValues(res, tags...)
res.promise = promise
@@ -129,7 +208,7 @@ func (l *DelegatingLogger) Fulfill(actual logr.Logger) {
// the given logger before it's promise is fulfilled.
func NewDelegatingLogger(initial logr.Logger) *DelegatingLogger {
l := &DelegatingLogger{
Logger: initial,
logger: initial,
promise: &loggerPromise{promisesLock: sync.Mutex{}},
}
l.promise.logger = l

Some files were not shown because too many files have changed in this diff Show More