update vendor

Signed-off-by: Roland.Ma <rolandma@yunify.com>
This commit is contained in:
Roland.Ma
2021-08-11 07:10:14 +00:00
parent a18f72b565
commit ea8f47c73a
2901 changed files with 269317 additions and 43103 deletions

View File

@@ -0,0 +1,469 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"io"
"io/ioutil"
"net/url"
"os"
"path/filepath"
"strconv"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
const (
// saKeyFile is the name of the service account signing private key file.
saKeyFile = "sa-signer.key"
// saKeyFile is the name of the service account signing public key (cert) file.
saCertFile = "sa-signer.crt"
)
// SecureServing provides/configures how the API server serves on the secure port.
type SecureServing struct {
// ListenAddr contains the host & port to serve on.
//
// Configurable. If unset, it will be defaulted.
process.ListenAddr
// CA contains the CA that signed the API server's serving certificates.
//
// Read-only.
CA []byte
// Authn can be used to provision users, and override what type of
// authentication is used to provision users.
//
// Configurable. If unset, it will be defaulted.
Authn
}
// APIServer knows how to run a kubernetes apiserver.
type APIServer struct {
// URL is the address the ApiServer should listen on for client
// connections.
//
// If set, this will configure the *insecure* serving details.
// If unset, it will contain the insecure port if insecure serving is enabled,
// and otherwise will contain the secure port.
//
// If this is not specified, we default to a random free port on localhost.
//
// Deprecated: use InsecureServing (for the insecure URL) or SecureServing, ideally.
URL *url.URL
// SecurePort is the additional secure port that the APIServer should listen on.
//
// If set, this will override SecureServing.Port.
//
// Deprecated: use SecureServing.
SecurePort int
// SecureServing indicates how the API server will serve on the secure port.
//
// Some parts are configurable. Will be defaulted if unset.
SecureServing
// InsecureServing indicates how the API server will serve on the insecure port.
//
// If unset, the insecure port will be disabled. Set to an empty struct to get
// default values.
//
// Deprecated: does not work with Kubernetes versions 1.20 and above. Use secure
// serving instead.
InsecureServing *process.ListenAddr
// Path is the path to the apiserver binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_KUBE_APISERVER environment variable, and
// the default test assets directory. See the "Binaries" section above (in
// doc.go) for details.
Path string
// Args is a list of arguments which will passed to the APIServer binary.
// Before they are passed on, they will be evaluated as go-template strings.
// This means you can use fields which are defined and exported on this
// APIServer struct (e.g. "--cert-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the APIServer's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the APIServer will
// be used.
//
// They will be loaded into the same argument set as Configure. Each flag
// will be Append-ed to the configured arguments just before launch.
//
// Deprecated: use Configure instead.
Args []string
// CertDir is a path to a directory containing whatever certificates the
// APIServer will need.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
CertDir string
// EtcdURL is the URL of the Etcd the APIServer should use.
//
// If this is not specified, the Start() method will return an error.
EtcdURL *url.URL
// StartTimeout, StopTimeout specify the time the APIServer is allowed to
// take when starting and stoppping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where APIServer should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
processState *process.State
// args contains the structured arguments to use for running the API server
// Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs()
args *process.Arguments
}
// Configure returns Arguments that may be used to customize the
// flags used to launch the API server. A set of defaults will
// be applied underneath.
func (s *APIServer) Configure() *process.Arguments {
if s.args == nil {
s.args = process.EmptyArguments()
}
return s.args
}
// Start starts the apiserver, waits for it to come up, and returns an error,
// if occurred.
func (s *APIServer) Start() error {
if err := s.prepare(); err != nil {
return err
}
return s.processState.Start(s.Out, s.Err)
}
func (s *APIServer) prepare() error {
if err := s.setProcessState(); err != nil {
return err
}
return s.Authn.Start()
}
// configurePorts configures the serving ports for this API server.
//
// Most of this method currently deals with making the deprecated fields
// take precedence over the new fields.
func (s *APIServer) configurePorts() error {
// prefer the old fields to the new fields if a user set one,
// otherwise, default the new fields and populate the old ones.
// Insecure: URL, InsecureServing
if s.URL != nil {
s.InsecureServing = &process.ListenAddr{
Address: s.URL.Hostname(),
Port: s.URL.Port(),
}
} else if insec := s.InsecureServing; insec != nil {
if insec.Port == "" || insec.Address == "" {
port, host, err := addr.Suggest("")
if err != nil {
return fmt.Errorf("unable to provision unused insecure port: %w", err)
}
s.InsecureServing.Port = strconv.Itoa(port)
s.InsecureServing.Address = host
}
s.URL = s.InsecureServing.URL("http", "")
}
// Secure: SecurePort, SecureServing
if s.SecurePort != 0 {
s.SecureServing.Port = strconv.Itoa(s.SecurePort)
// if we don't have an address, try the insecure address, and otherwise
// default to loopback.
if s.SecureServing.Address == "" {
if s.InsecureServing != nil {
s.SecureServing.Address = s.InsecureServing.Address
} else {
s.SecureServing.Address = "127.0.0.1"
}
}
} else if s.SecureServing.Port == "" || s.SecureServing.Address == "" {
port, host, err := addr.Suggest("")
if err != nil {
return fmt.Errorf("unable to provision unused secure port: %w", err)
}
s.SecureServing.Port = strconv.Itoa(port)
s.SecureServing.Address = host
s.SecurePort = port
}
return nil
}
func (s *APIServer) setProcessState() error {
if s.EtcdURL == nil {
return fmt.Errorf("expected EtcdURL to be configured")
}
var err error
// unconditionally re-set this so we can successfully restart
// TODO(directxman12): we supported this in the past, but do we actually
// want to support re-using an API server object to restart? The loss
// of provisioned users is surprising to say the least.
s.processState = &process.State{
Dir: s.CertDir,
Path: s.Path,
StartTimeout: s.StartTimeout,
StopTimeout: s.StopTimeout,
}
if err := s.processState.Init("kube-apiserver"); err != nil {
return err
}
if err := s.configurePorts(); err != nil {
return err
}
// the secure port will always be on, so use that
s.processState.HealthCheck.URL = *s.SecureServing.URL("https", "/healthz")
s.CertDir = s.processState.Dir
s.Path = s.processState.Path
s.StartTimeout = s.processState.StartTimeout
s.StopTimeout = s.processState.StopTimeout
if err := s.populateAPIServerCerts(); err != nil {
return err
}
if s.SecureServing.Authn == nil {
authn, err := NewCertAuthn()
if err != nil {
return err
}
s.SecureServing.Authn = authn
}
if err := s.Authn.Configure(s.CertDir, s.Configure()); err != nil {
return err
}
// NB(directxman12): insecure port is a mess:
// - 1.19 and below have the `--insecure-port` flag, and require it to be set to zero to
// disable it, otherwise the default will be used and we'll conflict.
// - 1.20 requires the flag to be unset or set to zero, and yells at you if you configure it
// - 1.24 won't have the flag at all...
//
// In an effort to automatically do the right thing during this mess, we do feature discovery
// on the flags, and hope that we've "parsed" them properly.
//
// TODO(directxman12): once we support 1.20 as the min version (might be when 1.24 comes out,
// might be around 1.25 or 1.26), remove this logic and the corresponding line in API server's
// default args.
if err := s.discoverFlags(); err != nil {
return err
}
s.processState.Args, s.Args, err = process.TemplateAndArguments(s.Args, s.Configure(), process.TemplateDefaults{ //nolint:staticcheck
Data: s,
Defaults: s.defaultArgs(),
MinimalDefaults: map[string][]string{
// as per kubernetes-sigs/controller-runtime#641, we need this (we
// probably need other stuff too, but this is the only thing that was
// previously considered a "minimal default")
"service-cluster-ip-range": {"10.0.0.0/24"},
// we need *some* authorization mode for health checks on the secure port,
// so default to RBAC unless the user set something else (in which case
// this'll be ignored due to SliceToArguments using AppendNoDefaults).
"authorization-mode": {"RBAC"},
},
})
if err != nil {
return err
}
return nil
}
// discoverFlags checks for certain flags that *must* be set in certain
// versions, and *must not* be set in others.
func (s *APIServer) discoverFlags() error {
// Present: <1.24, Absent: >= 1.24
present, err := s.processState.CheckFlag("insecure-port")
if err != nil {
return err
}
if !present {
s.Configure().Disable("insecure-port")
}
return nil
}
func (s *APIServer) defaultArgs() map[string][]string {
args := map[string][]string{
"service-cluster-ip-range": {"10.0.0.0/24"},
"allow-privileged": {"true"},
// we're keeping this disabled because if enabled, default SA is
// missing which would force all tests to create one in normal
// apiserver operation this SA is created by controller, but that is
// not run in integration environment
"disable-admission-plugins": {"ServiceAccount"},
"cert-dir": {s.CertDir},
"authorization-mode": {"RBAC"},
"secure-port": {s.SecureServing.Port},
// NB(directxman12): previously we didn't set the bind address for the secure
// port. It *shouldn't* make a difference unless people are doing something really
// funky, but if you start to get bug reports look here ;-)
"bind-address": {s.SecureServing.Address},
// required on 1.20+, fine to leave on for <1.20
"service-account-issuer": {s.SecureServing.URL("https", "/").String()},
"service-account-key-file": {filepath.Join(s.CertDir, saCertFile)},
"service-account-signing-key-file": {filepath.Join(s.CertDir, saKeyFile)},
}
if s.EtcdURL != nil {
args["etcd-servers"] = []string{s.EtcdURL.String()}
}
if s.URL != nil {
args["insecure-port"] = []string{s.URL.Port()}
args["insecure-bind-address"] = []string{s.URL.Hostname()}
} else {
// TODO(directxman12): remove this once 1.21 is the lowest version we support
// (this might be a while, but this line'll break as of 1.24, so see the comment
// in Start
args["insecure-port"] = []string{"0"}
}
return args
}
func (s *APIServer) populateAPIServerCerts() error {
_, statErr := os.Stat(filepath.Join(s.CertDir, "apiserver.crt"))
if !os.IsNotExist(statErr) {
return statErr
}
ca, err := certs.NewTinyCA()
if err != nil {
return err
}
servingCerts, err := ca.NewServingCert()
if err != nil {
return err
}
certData, keyData, err := servingCerts.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.crt"), certData, 0640); err != nil { //nolint:gosec
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, "apiserver.key"), keyData, 0640); err != nil { //nolint:gosec
return err
}
s.SecureServing.CA = ca.CA.CertBytes()
// service account signing files too
saCA, err := certs.NewTinyCA()
if err != nil {
return err
}
saCert, saKey, err := saCA.CA.AsBytes()
if err != nil {
return err
}
if err := ioutil.WriteFile(filepath.Join(s.CertDir, saCertFile), saCert, 0640); err != nil { //nolint:gosec
return err
}
return ioutil.WriteFile(filepath.Join(s.CertDir, saKeyFile), saKey, 0640) //nolint:gosec
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the CertDir if necessary.
func (s *APIServer) Stop() error {
if s.processState.DirNeedsCleaning {
s.CertDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
if s.processState != nil {
if err := s.processState.Stop(); err != nil {
return err
}
}
return s.Authn.Stop()
}
// APIServerDefaultArgs exposes the default args for the APIServer so that you
// can use those to append your own additional arguments.
//
// Note that these arguments don't handle newer API servers well to due the more
// complex feature detection neeeded. It's recommended that you switch to .Configure
// as you upgrade API server versions.
//
// Deprecated: use APIServer.Configure().
var APIServerDefaultArgs = []string{
"--advertise-address=127.0.0.1",
"--etcd-servers={{ if .EtcdURL }}{{ .EtcdURL.String }}{{ end }}",
"--cert-dir={{ .CertDir }}",
"--insecure-port={{ if .URL }}{{ .URL.Port }}{{else}}0{{ end }}",
"{{ if .URL }}--insecure-bind-address={{ .URL.Hostname }}{{ end }}",
"--secure-port={{ if .SecurePort }}{{ .SecurePort }}{{ end }}",
// we're keeping this disabled because if enabled, default SA is missing which would force all tests to create one
// in normal apiserver operation this SA is created by controller, but that is not run in integration environment
"--disable-admission-plugins=ServiceAccount",
"--service-cluster-ip-range=10.0.0.0/24",
"--allow-privileged=true",
// NB(directxman12): we also enable RBAC if nothing else was enabled
}
// PrepareAPIServer is an internal-only (NEVER SHOULD BE EXPOSED)
// function that sets up the API server just before starting it,
// without actually starting it. This saves time on tests.
//
// NB(directxman12): do not expose this outside of internal -- it's unsafe to
// use, because things like port allocation could race even more than they
// currently do if you later call start!
func PrepareAPIServer(s *APIServer) error {
return s.prepare()
}
// APIServerArguments is an internal-only (NEVER SHOULD BE EXPOSED)
// function that sets up the API server just before starting it,
// without actually starting it. It's public to make testing easier.
//
// NB(directxman12): do not expose this outside of internal.
func APIServerArguments(s *APIServer) []string {
return s.processState.Args
}

View File

@@ -0,0 +1,142 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"io/ioutil"
"path/filepath"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
// User represents a Kubernetes user.
type User struct {
// Name is the user's Name.
Name string
// Groups are the groups to which the user belongs.
Groups []string
}
// Authn knows how to configure an API server for a particular type of authentication,
// and provision users under that authentication scheme.
//
// The methods must be called in the following order (as presented below in the interface
// for a mnemonic):
//
// 1. Configure
// 2. Start
// 3. AddUsers (0+ calls)
// 4. Stop.
type Authn interface {
// Configure provides the working directory to this authenticator,
// and configures the given API server arguments to make use of this authenticator.
//
// Should be called first.
Configure(workDir string, args *process.Arguments) error
// Start runs this authenticator. Will be called just before API server start.
//
// Must be called after Configure.
Start() error
// AddUser provisions a user, returning a copy of the given base rest.Config
// configured to authenticate as that users.
//
// May only be called while the authenticator is "running".
AddUser(user User, baseCfg *rest.Config) (*rest.Config, error)
// Stop shuts down this authenticator.
Stop() error
}
// CertAuthn is an authenticator (Authn) that makes use of client certificate authn.
type CertAuthn struct {
// ca is the CA used to sign the client certs
ca *certs.TinyCA
// certDir is the directory used to write the CA crt file
// so that the API server can read it.
certDir string
}
// NewCertAuthn creates a new client-cert-based Authn with a new CA.
func NewCertAuthn() (*CertAuthn, error) {
ca, err := certs.NewTinyCA()
if err != nil {
return nil, fmt.Errorf("unable to provision client certificate auth CA: %w", err)
}
return &CertAuthn{
ca: ca,
}, nil
}
// AddUser provisions a new user that's authenticated via certificates, with
// the given uesrname and groups embedded in the certificate as expected by the
// API server.
func (c *CertAuthn) AddUser(user User, baseCfg *rest.Config) (*rest.Config, error) {
certs, err := c.ca.NewClientCert(certs.ClientInfo{
Name: user.Name,
Groups: user.Groups,
})
if err != nil {
return nil, fmt.Errorf("unable to create client certificates for %s: %w", user.Name, err)
}
crt, key, err := certs.AsBytes()
if err != nil {
return nil, fmt.Errorf("unable to serialize client certificates for %s: %w", user.Name, err)
}
cfg := rest.CopyConfig(baseCfg)
cfg.CertData = crt
cfg.KeyData = key
return cfg, nil
}
// caCrtPath returns the path to the on-disk client-cert CA crt file.
func (c *CertAuthn) caCrtPath() string {
return filepath.Join(c.certDir, "client-cert-auth-ca.crt")
}
// Configure provides the working directory to this authenticator,
// and configures the given API server arguments to make use of this authenticator.
func (c *CertAuthn) Configure(workDir string, args *process.Arguments) error {
c.certDir = workDir
args.Set("client-ca-file", c.caCrtPath())
return nil
}
// Start runs this authenticator. Will be called just before API server start.
//
// Must be called after Configure.
func (c *CertAuthn) Start() error {
if len(c.certDir) == 0 {
return fmt.Errorf("start called before configure")
}
caCrt := c.ca.CA.CertBytes()
if err := ioutil.WriteFile(c.caCrtPath(), caCrt, 0640); err != nil { //nolint:gosec
return fmt.Errorf("unable to save the client certificate CA to %s: %w", c.caCrtPath(), err)
}
return nil
}
// Stop shuts down this authenticator.
func (c *CertAuthn) Stop() error {
// no-op -- our workdir is cleaned up for us automatically
return nil
}

View File

@@ -0,0 +1,180 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"io"
"net"
"net/url"
"strconv"
"time"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/addr"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
// Etcd knows how to run an etcd server.
type Etcd struct {
// URL is the address the Etcd should listen on for client connections.
//
// If this is not specified, we default to a random free port on localhost.
URL *url.URL
// Path is the path to the etcd binary.
//
// If this is left as the empty string, we will attempt to locate a binary,
// by checking for the TEST_ASSET_ETCD environment variable, and the default
// test assets directory. See the "Binaries" section above (in doc.go) for
// details.
Path string
// Args is a list of arguments which will passed to the Etcd binary. Before
// they are passed on, the`y will be evaluated as go-template strings. This
// means you can use fields which are defined and exported on this Etcd
// struct (e.g. "--data-dir={{ .Dir }}").
// Those templates will be evaluated after the defaulting of the Etcd's
// fields has already happened and just before the binary actually gets
// started. Thus you have access to calculated fields like `URL` and others.
//
// If not specified, the minimal set of arguments to run the Etcd will be
// used.
//
// They will be loaded into the same argument set as Configure. Each flag
// will be Append-ed to the configured arguments just before launch.
//
// Deprecated: use Configure instead.
Args []string
// DataDir is a path to a directory in which etcd can store its state.
//
// If left unspecified, then the Start() method will create a fresh temporary
// directory, and the Stop() method will clean it up.
DataDir string
// StartTimeout, StopTimeout specify the time the Etcd is allowed to
// take when starting and stopping before an error is emitted.
//
// If not specified, these default to 20 seconds.
StartTimeout time.Duration
StopTimeout time.Duration
// Out, Err specify where Etcd should write its StdOut, StdErr to.
//
// If not specified, the output will be discarded.
Out io.Writer
Err io.Writer
// processState contains the actual details about this running process
processState *process.State
// args contains the structured arguments to use for running etcd.
// Lazily initialized by .Configure(), Defaulted eventually with .defaultArgs()
args *process.Arguments
}
// Start starts the etcd, waits for it to come up, and returns an error, if one
// occoured.
func (e *Etcd) Start() error {
if err := e.setProcessState(); err != nil {
return err
}
return e.processState.Start(e.Out, e.Err)
}
func (e *Etcd) setProcessState() error {
e.processState = &process.State{
Dir: e.DataDir,
Path: e.Path,
StartTimeout: e.StartTimeout,
StopTimeout: e.StopTimeout,
}
// unconditionally re-set this so we can successfully restart
// TODO(directxman12): we supported this in the past, but do we actually
// want to support re-using an API server object to restart? The loss
// of provisioned users is surprising to say the least.
if err := e.processState.Init("etcd"); err != nil {
return err
}
if e.URL == nil {
port, host, err := addr.Suggest("")
if err != nil {
return err
}
e.URL = &url.URL{
Scheme: "http",
Host: net.JoinHostPort(host, strconv.Itoa(port)),
}
}
// can use /health as of etcd 3.3.0
e.processState.HealthCheck.URL = *e.URL
e.processState.HealthCheck.Path = "/health"
e.DataDir = e.processState.Dir
e.Path = e.processState.Path
e.StartTimeout = e.processState.StartTimeout
e.StopTimeout = e.processState.StopTimeout
var err error
e.processState.Args, e.Args, err = process.TemplateAndArguments(e.Args, e.Configure(), process.TemplateDefaults{ //nolint:staticcheck
Data: e,
Defaults: e.defaultArgs(),
})
return err
}
// Stop stops this process gracefully, waits for its termination, and cleans up
// the DataDir if necessary.
func (e *Etcd) Stop() error {
if e.processState.DirNeedsCleaning {
e.DataDir = "" // reset the directory if it was randomly allocated, so that we can safely restart
}
return e.processState.Stop()
}
func (e *Etcd) defaultArgs() map[string][]string {
args := map[string][]string{
"listen-peer-urls": {"http://localhost:0"},
"data-dir": {e.DataDir},
}
if e.URL != nil {
args["advertise-client-urls"] = []string{e.URL.String()}
args["listen-client-urls"] = []string{e.URL.String()}
}
return args
}
// Configure returns Arguments that may be used to customize the
// flags used to launch etcd. A set of defaults will
// be applied underneath.
func (e *Etcd) Configure() *process.Arguments {
if e.args == nil {
e.args = process.EmptyArguments()
}
return e.args
}
// EtcdDefaultArgs exposes the default args for Etcd so that you
// can use those to append your own additional arguments.
var EtcdDefaultArgs = []string{
"--listen-peer-urls=http://localhost:0",
"--advertise-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--listen-client-urls={{ if .URL }}{{ .URL.String }}{{ end }}",
"--data-dir={{ .DataDir }}",
}

View File

@@ -0,0 +1,119 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"bytes"
"fmt"
"io"
"net/url"
"os/exec"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
kcapi "k8s.io/client-go/tools/clientcmd/api"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/process"
)
const (
envtestName = "envtest"
)
// KubeConfigFromREST reverse-engineers a kubeconfig file from a rest.Config.
// The options are tailored towards the rest.Configs we generate, so they're
// not broadly applicable.
//
// This is not intended to be exposed beyond internal for the above reasons.
func KubeConfigFromREST(cfg *rest.Config) ([]byte, error) {
kubeConfig := kcapi.NewConfig()
protocol := "https"
if !rest.IsConfigTransportTLS(*cfg) {
protocol = "http"
}
// cfg.Host is a URL, so we need to parse it so we can properly append the API path
baseURL, err := url.Parse(cfg.Host)
if err != nil {
return nil, fmt.Errorf("unable to interpret config's host value as a URL: %w", err)
}
kubeConfig.Clusters[envtestName] = &kcapi.Cluster{
// TODO(directxman12): if client-go ever decides to expose defaultServerUrlFor(config),
// we can just use that. Note that this is not the same as the public DefaultServerURL,
// which requires us to pass a bunch of stuff in manually.
Server: (&url.URL{Scheme: protocol, Host: baseURL.Host, Path: cfg.APIPath}).String(),
CertificateAuthorityData: cfg.CAData,
}
kubeConfig.AuthInfos[envtestName] = &kcapi.AuthInfo{
// try to cover all auth strategies that aren't plugins
ClientCertificateData: cfg.CertData,
ClientKeyData: cfg.KeyData,
Token: cfg.BearerToken,
Username: cfg.Username,
Password: cfg.Password,
}
kcCtx := kcapi.NewContext()
kcCtx.Cluster = envtestName
kcCtx.AuthInfo = envtestName
kubeConfig.Contexts[envtestName] = kcCtx
kubeConfig.CurrentContext = envtestName
contents, err := clientcmd.Write(*kubeConfig)
if err != nil {
return nil, fmt.Errorf("unable to serialize kubeconfig file: %w", err)
}
return contents, nil
}
// KubeCtl is a wrapper around the kubectl binary.
type KubeCtl struct {
// Path where the kubectl binary can be found.
//
// If this is left empty, we will attempt to locate a binary, by checking for
// the TEST_ASSET_KUBECTL environment variable, and the default test assets
// directory. See the "Binaries" section above (in doc.go) for details.
Path string
// Opts can be used to configure additional flags which will be used each
// time the wrapped binary is called.
//
// For example, you might want to use this to set the URL of the APIServer to
// connect to.
Opts []string
}
// Run executes the wrapped binary with some preconfigured options and the
// arguments given to this method. It returns Readers for the stdout and
// stderr.
func (k *KubeCtl) Run(args ...string) (stdout, stderr io.Reader, err error) {
if k.Path == "" {
k.Path = process.BinPathFinder("kubectl", "")
}
stdoutBuffer := &bytes.Buffer{}
stderrBuffer := &bytes.Buffer{}
allArgs := append(k.Opts, args...)
cmd := exec.Command(k.Path, allArgs...)
cmd.Stdout = stdoutBuffer
cmd.Stderr = stderrBuffer
err = cmd.Run()
return stdoutBuffer, stderrBuffer, err
}

View File

@@ -0,0 +1,248 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controlplane
import (
"fmt"
"net/url"
"os"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/internal/testing/certs"
)
// NewTinyCA creates a new a tiny CA utility for provisioning serving certs and client certs FOR TESTING ONLY.
// Don't use this for anything else!
var NewTinyCA = certs.NewTinyCA
// ControlPlane is a struct that knows how to start your test control plane.
//
// Right now, that means Etcd and your APIServer. This is likely to increase in
// future.
type ControlPlane struct {
APIServer *APIServer
Etcd *Etcd
// Kubectl will override the default asset search path for kubectl
KubectlPath string
// for the deprecated methods (Kubectl, etc)
defaultUserCfg *rest.Config
defaultUserKubectl *KubeCtl
}
// Start will start your control plane processes. To stop them, call Stop().
func (f *ControlPlane) Start() error {
if f.Etcd == nil {
f.Etcd = &Etcd{}
}
if err := f.Etcd.Start(); err != nil {
return err
}
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
f.APIServer.EtcdURL = f.Etcd.URL
if err := f.APIServer.Start(); err != nil {
return err
}
// provision the default user -- can be removed when the related
// methods are removed. The default user has admin permissions to
// mimic legacy no-authz setups.
user, err := f.AddUser(User{Name: "default", Groups: []string{"system:masters"}}, &rest.Config{})
if err != nil {
return fmt.Errorf("unable to provision the default (legacy) user: %w", err)
}
kubectl, err := user.Kubectl()
if err != nil {
return fmt.Errorf("unable to provision the default (legacy) kubeconfig: %w", err)
}
f.defaultUserCfg = user.Config()
f.defaultUserKubectl = kubectl
return nil
}
// Stop will stop your control plane processes, and clean up their data.
func (f *ControlPlane) Stop() error {
var errList []error
if f.APIServer != nil {
if err := f.APIServer.Stop(); err != nil {
errList = append(errList, err)
}
}
if f.Etcd != nil {
if err := f.Etcd.Stop(); err != nil {
errList = append(errList, err)
}
}
return kerrors.NewAggregate(errList)
}
// APIURL returns the URL you should connect to to talk to your API server.
//
// If insecure serving is configured, this will contain the insecure port.
// Otherwise, it will contain the secure port.
//
// Deprecated: use AddUser instead, or APIServer.{Ins|S}ecureServing.URL if
// you really want just the URL.
func (f *ControlPlane) APIURL() *url.URL {
return f.APIServer.URL
}
// KubeCtl returns a pre-configured KubeCtl, ready to connect to this
// ControlPlane.
//
// Deprecated: use AddUser & AuthenticatedUser.Kubectl instead.
func (f *ControlPlane) KubeCtl() *KubeCtl {
return f.defaultUserKubectl
}
// RESTClientConfig returns a pre-configured restconfig, ready to connect to
// this ControlPlane.
//
// Deprecated: use AddUser & AuthenticatedUser.Config instead.
func (f *ControlPlane) RESTClientConfig() (*rest.Config, error) {
return f.defaultUserCfg, nil
}
// AuthenticatedUser contains access information for an provisioned user,
// including REST config, kubeconfig contents, and access to a KubeCtl instance.
//
// It's not "safe" to use the methods on this till after the API server has been
// started (due to certificate initialization and such). The various methods will
// panic if this is done.
type AuthenticatedUser struct {
// cfg is the rest.Config for connecting to the API server. It's lazily initialized.
cfg *rest.Config
// cfgIsComplete indicates the cfg has had late-initialized fields (e.g.
// API server CA data) initialized.
cfgIsComplete bool
// apiServer is a handle to the APIServer that's used when finalizing cfg
// and producing the kubectl instance.
plane *ControlPlane
// kubectl is our existing, provisioned kubectl. We don't provision one
// till someone actually asks for it.
kubectl *KubeCtl
}
// Config returns the REST config that can be used to connect to the API server
// as this user.
//
// Will panic if used before the API server is started.
func (u *AuthenticatedUser) Config() *rest.Config {
// NB(directxman12): we choose to panic here for ergonomics sake, and because there's
// not really much you can do to "handle" this error. This machinery is intended to be
// used in tests anyway, so panicing is not a particularly big deal.
if u.cfgIsComplete {
return u.cfg
}
if len(u.plane.APIServer.SecureServing.CA) == 0 {
panic("the API server has not yet been started, please do that before accessing connection details")
}
u.cfg.CAData = u.plane.APIServer.SecureServing.CA
u.cfg.Host = u.plane.APIServer.SecureServing.URL("https", "/").String()
u.cfgIsComplete = true
return u.cfg
}
// KubeConfig returns a KubeConfig that's roughly equivalent to this user's REST config.
//
// Will panic if used before the API server is started.
func (u AuthenticatedUser) KubeConfig() ([]byte, error) {
// NB(directxman12): we don't return the actual API object to avoid yet another
// piece of kubernetes API in our public API, and also because generally the thing
// you want to do with this is just write it out to a file for external debugging
// purposes, etc.
return KubeConfigFromREST(u.Config())
}
// Kubectl returns a KubeCtl instance for talking to the API server as this user. It uses
// a kubeconfig equivalent to that returned by .KubeConfig.
//
// Will panic if used before the API server is started.
func (u *AuthenticatedUser) Kubectl() (*KubeCtl, error) {
if u.kubectl != nil {
return u.kubectl, nil
}
if len(u.plane.APIServer.CertDir) == 0 {
panic("the API server has not yet been started, please do that before accessing connection details")
}
// cleaning this up is handled when our tmpDir is deleted
out, err := os.CreateTemp(u.plane.APIServer.CertDir, "*.kubecfg")
if err != nil {
return nil, fmt.Errorf("unable to create file for kubeconfig: %w", err)
}
defer out.Close()
contents, err := KubeConfigFromREST(u.Config())
if err != nil {
return nil, err
}
if _, err := out.Write(contents); err != nil {
return nil, fmt.Errorf("unable to write kubeconfig to disk at %s: %w", out.Name(), err)
}
k := &KubeCtl{
Path: u.plane.KubectlPath,
}
k.Opts = append(k.Opts, fmt.Sprintf("--kubeconfig=%s", out.Name()))
u.kubectl = k
return k, nil
}
// AddUser provisions a new user in the cluster. It uses the APIServer's authentication
// strategy -- see APIServer.SecureServing.Authn.
//
// Unlike AddUser, it's safe to pass a nil rest.Config here if you have no
// particular opinions about the config.
//
// The default authentication strategy is not guaranteed to any specific strategy, but it is
// guaranteed to be callable both before and after Start has been called (but, as noted in the
// AuthenticatedUser docs, the given user objects are only valid after Start has been called).
func (f *ControlPlane) AddUser(user User, baseConfig *rest.Config) (*AuthenticatedUser, error) {
if f.GetAPIServer().SecureServing.Authn == nil {
return nil, fmt.Errorf("no API server authentication is configured yet. The API server defaults one when Start is called, did you mean to use that?")
}
if baseConfig == nil {
baseConfig = &rest.Config{}
}
cfg, err := f.GetAPIServer().SecureServing.AddUser(user, baseConfig)
if err != nil {
return nil, err
}
return &AuthenticatedUser{
cfg: cfg,
plane: f,
}, nil
}
// GetAPIServer returns this ControlPlane's APIServer, initializing it if necessary.
func (f *ControlPlane) GetAPIServer() *APIServer {
if f.APIServer == nil {
f.APIServer = &APIServer{}
}
return f.APIServer
}