add service mesh controller

add service mesh metrics

remove unused circle yaml

fix travis misconfiguration

fix travis misconfiguration

fix travis misconfiguration
This commit is contained in:
jeff
2019-03-08 18:22:30 +08:00
committed by Jeff
parent 858facd4b2
commit 4ac20ffc2b
1709 changed files with 344390 additions and 60749 deletions

View File

@@ -17,7 +17,7 @@ limitations under the License.
package types
import (
"github.com/mattbaird/jsonpatch"
"github.com/appscode/jsonpatch"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
"k8s.io/apimachinery/pkg/runtime"

View File

@@ -26,7 +26,7 @@ import (
"strings"
"sync"
"github.com/mattbaird/jsonpatch"
"github.com/appscode/jsonpatch"
admissionv1beta1 "k8s.io/api/admission/v1beta1"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"

View File

@@ -0,0 +1,356 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"errors"
"fmt"
"net"
"net/http"
"net/url"
"path"
"sort"
"strconv"
"k8s.io/api/admissionregistration/v1beta1"
admissionregistration "k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/writer"
"sigs.k8s.io/controller-runtime/pkg/webhook/types"
)
// setDefault does defaulting for the Server.
func (s *Server) setDefault() {
s.setServerDefault()
s.setBootstrappingDefault()
}
// setServerDefault does defaulting for the ServerOptions.
func (s *Server) setServerDefault() {
if len(s.Name) == 0 {
s.Name = "default-k8s-webhook-server"
}
if s.registry == nil {
s.registry = map[string]Webhook{}
}
if s.sMux == nil {
s.sMux = http.DefaultServeMux
}
if s.Port <= 0 {
s.Port = 443
}
if len(s.CertDir) == 0 {
s.CertDir = path.Join("k8s-webhook-server", "cert")
}
if s.DisableWebhookConfigInstaller == nil {
diwc := false
s.DisableWebhookConfigInstaller = &diwc
}
if s.Client == nil {
cfg, err := config.GetConfig()
if err != nil {
s.err = err
return
}
s.Client, err = client.New(cfg, client.Options{})
if err != nil {
s.err = err
return
}
}
}
// setBootstrappingDefault does defaulting for the Server bootstrapping.
func (s *Server) setBootstrappingDefault() {
if s.BootstrapOptions == nil {
s.BootstrapOptions = &BootstrapOptions{}
}
if len(s.MutatingWebhookConfigName) == 0 {
s.MutatingWebhookConfigName = "mutating-webhook-configuration"
}
if len(s.ValidatingWebhookConfigName) == 0 {
s.ValidatingWebhookConfigName = "validating-webhook-configuration"
}
if s.Host == nil && s.Service == nil {
varString := "localhost"
s.Host = &varString
}
var certWriter writer.CertWriter
var err error
if s.Secret != nil {
certWriter, err = writer.NewSecretCertWriter(
writer.SecretCertWriterOptions{
Secret: s.Secret,
Client: s.Client,
})
} else {
certWriter, err = writer.NewFSCertWriter(
writer.FSCertWriterOptions{
Path: s.CertDir,
})
}
if err != nil {
s.err = err
return
}
s.certProvisioner = &cert.Provisioner{
CertWriter: certWriter,
}
}
// InstallWebhookManifests creates the admissionWebhookConfiguration objects and service if any.
// It also provisions the certificate for the admission server.
func (s *Server) InstallWebhookManifests() error {
// do defaulting if necessary
s.once.Do(s.setDefault)
if s.err != nil {
return s.err
}
var err error
s.webhookConfigurations, err = s.whConfigs()
if err != nil {
return err
}
svc := s.service()
objects := append(s.webhookConfigurations, svc)
cc, err := s.getClientConfig()
if err != nil {
return err
}
// Provision the cert by creating new one or refreshing existing one.
_, err = s.certProvisioner.Provision(cert.Options{
ClientConfig: cc,
Objects: s.webhookConfigurations,
})
if err != nil {
return err
}
return batchCreateOrReplace(s.Client, objects...)
}
func (s *Server) getClientConfig() (*admissionregistration.WebhookClientConfig, error) {
if s.Host != nil && s.Service != nil {
return nil, errors.New("URL and Service can't be set at the same time")
}
cc := &admissionregistration.WebhookClientConfig{
CABundle: []byte{},
}
if s.Host != nil {
u := url.URL{
Scheme: "https",
Host: net.JoinHostPort(*s.Host, strconv.Itoa(int(s.Port))),
}
urlString := u.String()
cc.URL = &urlString
}
if s.Service != nil {
cc.Service = &admissionregistration.ServiceReference{
Name: s.Service.Name,
Namespace: s.Service.Namespace,
// Path will be set later
}
}
return cc, nil
}
// getClientConfigWithPath constructs a WebhookClientConfig based on the server options.
// It will use path to the set the path in WebhookClientConfig.
func (s *Server) getClientConfigWithPath(path string) (*admissionregistration.WebhookClientConfig, error) {
cc, err := s.getClientConfig()
if err != nil {
return nil, err
}
return cc, setPath(cc, path)
}
// setPath sets the path in the WebhookClientConfig.
func setPath(cc *admissionregistration.WebhookClientConfig, path string) error {
if cc.URL != nil {
u, err := url.Parse(*cc.URL)
if err != nil {
return err
}
u.Path = path
urlString := u.String()
cc.URL = &urlString
}
if cc.Service != nil {
cc.Service.Path = &path
}
return nil
}
// whConfigs creates a mutatingWebhookConfiguration and(or) a validatingWebhookConfiguration based on registry.
// For the same type of webhook configuration, it generates a webhook entry per endpoint.
func (s *Server) whConfigs() ([]runtime.Object, error) {
objs := []runtime.Object{}
mutatingWH, err := s.mutatingWHConfigs()
if err != nil {
return nil, err
}
if mutatingWH != nil {
objs = append(objs, mutatingWH)
}
validatingWH, err := s.validatingWHConfigs()
if err != nil {
return nil, err
}
if validatingWH != nil {
objs = append(objs, validatingWH)
}
return objs, nil
}
func (s *Server) mutatingWHConfigs() (runtime.Object, error) {
mutatingWebhooks := []v1beta1.Webhook{}
for path, webhook := range s.registry {
if webhook.GetType() != types.WebhookTypeMutating {
continue
}
admissionWebhook := webhook.(*admission.Webhook)
wh, err := s.admissionWebhook(path, admissionWebhook)
if err != nil {
return nil, err
}
mutatingWebhooks = append(mutatingWebhooks, *wh)
}
sort.Slice(mutatingWebhooks, func(i, j int) bool {
return mutatingWebhooks[i].Name < mutatingWebhooks[j].Name
})
if len(mutatingWebhooks) > 0 {
return &admissionregistration.MutatingWebhookConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: fmt.Sprintf("%s/%s", admissionregistration.GroupName, "v1beta1"),
Kind: "MutatingWebhookConfiguration",
},
ObjectMeta: metav1.ObjectMeta{
Name: s.MutatingWebhookConfigName,
},
Webhooks: mutatingWebhooks,
}, nil
}
return nil, nil
}
func (s *Server) validatingWHConfigs() (runtime.Object, error) {
validatingWebhooks := []v1beta1.Webhook{}
for path, webhook := range s.registry {
var admissionWebhook *admission.Webhook
if webhook.GetType() != types.WebhookTypeValidating {
continue
}
admissionWebhook = webhook.(*admission.Webhook)
wh, err := s.admissionWebhook(path, admissionWebhook)
if err != nil {
return nil, err
}
validatingWebhooks = append(validatingWebhooks, *wh)
}
sort.Slice(validatingWebhooks, func(i, j int) bool {
return validatingWebhooks[i].Name < validatingWebhooks[j].Name
})
if len(validatingWebhooks) > 0 {
return &admissionregistration.ValidatingWebhookConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: fmt.Sprintf("%s/%s", admissionregistration.GroupName, "v1beta1"),
Kind: "ValidatingWebhookConfiguration",
},
ObjectMeta: metav1.ObjectMeta{
Name: s.ValidatingWebhookConfigName,
},
Webhooks: validatingWebhooks,
}, nil
}
return nil, nil
}
func (s *Server) admissionWebhook(path string, wh *admission.Webhook) (*admissionregistration.Webhook, error) {
if wh.NamespaceSelector == nil && s.Service != nil && len(s.Service.Namespace) > 0 {
wh.NamespaceSelector = &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: "control-plane",
Operator: metav1.LabelSelectorOpDoesNotExist,
},
},
}
}
webhook := &admissionregistration.Webhook{
Name: wh.GetName(),
Rules: wh.Rules,
FailurePolicy: wh.FailurePolicy,
NamespaceSelector: wh.NamespaceSelector,
ClientConfig: admissionregistration.WebhookClientConfig{
// The reason why we assign an empty byte array to CABundle is that
// CABundle field will be updated by the Provisioner.
CABundle: []byte{},
},
}
cc, err := s.getClientConfigWithPath(path)
if err != nil {
return nil, err
}
webhook.ClientConfig = *cc
return webhook, nil
}
// service creates a corev1.service object fronting the admission server.
func (s *Server) service() runtime.Object {
if s.Service == nil {
return nil
}
svc := &corev1.Service{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Service",
},
ObjectMeta: metav1.ObjectMeta{
Name: s.Service.Name,
Namespace: s.Service.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: s.Service.Selectors,
Ports: []corev1.ServicePort{
{
// When using service, kube-apiserver will send admission request to port 443.
Port: 443,
TargetPort: intstr.IntOrString{Type: intstr.Int, IntVal: s.Port},
},
},
},
}
return svc
}

View File

@@ -0,0 +1,94 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package webhook provides methods to build and bootstrap a webhook server.
Currently, it only supports admission webhooks. It will support CRD conversion webhooks in the near future.
Build webhooks
// mgr is the manager that runs the server.
webhook1, err := NewWebhookBuilder().
Name("foo.k8s.io").
Mutating().
Path("/mutating-pods").
Operations(admissionregistrationv1beta1.Create).
ForType(&corev1.Pod{}).
WithManager(mgr).
Handlers(mutatingHandler1, mutatingHandler2).
Build()
if err != nil {
// handle error
}
webhook2, err := NewWebhookBuilder().
Name("bar.k8s.io").
Validating().
Path("/validating-deployment").
Operations(admissionregistrationv1beta1.Create, admissionregistrationv1beta1.Update).
ForType(&appsv1.Deployment{}).
WithManager(mgr).
Handlers(validatingHandler1).
Build()
if err != nil {
// handle error
}
Create a webhook server.
as, err := NewServer("baz-admission-server", mgr, ServerOptions{
CertDir: "/tmp/cert",
BootstrapOptions: &BootstrapOptions{
Secret: &apitypes.NamespacedName{
Namespace: "default",
Name: "foo-admission-server-secret",
},
Service: &Service{
Namespace: "default",
Name: "foo-admission-server-service",
// Selectors should select the pods that runs this webhook server.
Selectors: map[string]string{
"app": "foo-admission-server",
},
},
},
})
if err != nil {
// handle error
}
Register the webhooks in the server.
err = as.Register(webhook1, webhook2)
if err != nil {
// handle error
}
Start the server by starting the manager
err := mrg.Start(signals.SetupSignalHandler())
if err != nil {
// handle error
}
*/
package webhook
import (
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)
var log = logf.KBLog.WithName("webhook")

View File

@@ -0,0 +1,36 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package cert provides functions to manage certificates for webhookClientConfiguration.
Create a Provisioner with a CertWriter.
provisioner := Provisioner{
CertWriter: admission.NewSecretCertWriter(admission.SecretCertWriterOptions{...}),
}
Provision the certificates for the webhookClientConfig
err := provisioner.Provision(Options{
ClientConfig: webhookClientConfig,
Objects: []runtime.Object{mutatingWebhookConfiguration, validatingWebhookConfiguration}
})
if err != nil {
// handle error
}
*/
package cert

View File

@@ -0,0 +1,38 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generator
// Artifacts hosts a private key, its corresponding serving certificate and
// the CA certificate that signs the serving certificate.
type Artifacts struct {
// PEM encoded private key
Key []byte
// PEM encoded serving certificate
Cert []byte
// PEM encoded CA private key
CAKey []byte
// PEM encoded CA certificate
CACert []byte
}
// CertGenerator is an interface to provision the serving certificate.
type CertGenerator interface {
// Generate returns a Artifacts struct.
Generate(CommonName string) (*Artifacts, error)
// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert.
SetCA(caKey, caCert []byte)
}

View File

@@ -0,0 +1,30 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package generator provides an interface and implementation to provision certificates.
Create an instance of certGenerator.
cg := SelfSignedCertGenerator{}
Generate the certificates.
certs, err := cg.Generate("foo.bar.com")
if err != nil {
// handle error
}
*/
package generator

View File

@@ -0,0 +1,117 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generator
import (
"crypto/rsa"
"crypto/x509"
"fmt"
"time"
"k8s.io/client-go/util/cert"
)
// ServiceToCommonName generates the CommonName for the certificate when using a k8s service.
func ServiceToCommonName(serviceNamespace, serviceName string) string {
return fmt.Sprintf("%s.%s.svc", serviceName, serviceNamespace)
}
// SelfSignedCertGenerator implements the certGenerator interface.
// It provisions self-signed certificates.
type SelfSignedCertGenerator struct {
caKey []byte
caCert []byte
}
var _ CertGenerator = &SelfSignedCertGenerator{}
// SetCA sets the PEM-encoded CA private key and CA cert for signing the generated serving cert.
func (cp *SelfSignedCertGenerator) SetCA(caKey, caCert []byte) {
cp.caKey = caKey
cp.caCert = caCert
}
// Generate creates and returns a CA certificate, certificate and
// key for the server. serverKey and serverCert are used by the server
// to establish trust for clients, CA certificate is used by the
// client to verify the server authentication chain.
// The cert will be valid for 365 days.
func (cp *SelfSignedCertGenerator) Generate(commonName string) (*Artifacts, error) {
var signingKey *rsa.PrivateKey
var signingCert *x509.Certificate
var valid bool
var err error
valid, signingKey, signingCert = cp.validCACert()
if !valid {
signingKey, err = cert.NewPrivateKey()
if err != nil {
return nil, fmt.Errorf("failed to create the CA private key: %v", err)
}
signingCert, err = cert.NewSelfSignedCACert(cert.Config{CommonName: "webhook-cert-ca"}, signingKey)
if err != nil {
return nil, fmt.Errorf("failed to create the CA cert: %v", err)
}
}
key, err := cert.NewPrivateKey()
if err != nil {
return nil, fmt.Errorf("failed to create the private key: %v", err)
}
signedCert, err := cert.NewSignedCert(
cert.Config{
CommonName: commonName,
Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
},
key, signingCert, signingKey,
)
if err != nil {
return nil, fmt.Errorf("failed to create the cert: %v", err)
}
return &Artifacts{
Key: cert.EncodePrivateKeyPEM(key),
Cert: cert.EncodeCertPEM(signedCert),
CAKey: cert.EncodePrivateKeyPEM(signingKey),
CACert: cert.EncodeCertPEM(signingCert),
}, nil
}
func (cp *SelfSignedCertGenerator) validCACert() (bool, *rsa.PrivateKey, *x509.Certificate) {
if !ValidCACert(cp.caKey, cp.caCert, cp.caCert, "",
time.Now().AddDate(1, 0, 0)) {
return false, nil, nil
}
var ok bool
key, err := cert.ParsePrivateKeyPEM(cp.caKey)
if err != nil {
return false, nil, nil
}
privateKey, ok := key.(*rsa.PrivateKey)
if !ok {
return false, nil, nil
}
certs, err := cert.ParseCertsPEM(cp.caCert)
if err != nil {
return false, nil, nil
}
if len(certs) != 1 {
return false, nil, nil
}
return true, privateKey, certs[0]
}

View File

@@ -0,0 +1,61 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package generator
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"time"
)
// ValidCACert think cert and key are valid if they meet the following requirements:
// - key and cert are valid pair
// - caCert is the root ca of cert
// - cert is for dnsName
// - cert won't expire before time
func ValidCACert(key, cert, caCert []byte, dnsName string, time time.Time) bool {
if len(key) == 0 || len(cert) == 0 || len(caCert) == 0 {
return false
}
// Verify key and cert are valid pair
_, err := tls.X509KeyPair(cert, key)
if err != nil {
return false
}
// Verify cert is valid for at least 1 year.
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(caCert) {
return false
}
block, _ := pem.Decode([]byte(cert))
if block == nil {
return false
}
c, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return false
}
ops := x509.VerifyOptions{
DNSName: dnsName,
Roots: pool,
CurrentTime: time,
}
_, err = c.Verify(ops)
return err == nil
}

View File

@@ -0,0 +1,133 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package cert
import (
"bytes"
"errors"
"fmt"
"net"
"net/url"
admissionregistrationv1beta1 "k8s.io/api/admissionregistration/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/generator"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/writer"
)
// Provisioner provisions certificates for webhook configurations and writes them to an output
// destination - such as a Secret or local file. Provisioner can update the CA field of
// certain resources with the CA of the certs.
type Provisioner struct {
// CertWriter knows how to persist the certificate.
CertWriter writer.CertWriter
}
// Options are options for provisioning the certificate.
type Options struct {
// ClientConfig is the WebhookClientCert that contains the information to generate
// the certificate. The CA Certificate will be updated in the ClientConfig.
// The updated ClientConfig will be used to inject into other runtime.Objects,
// e.g. MutatingWebhookConfiguration and ValidatingWebhookConfiguration.
ClientConfig *admissionregistrationv1beta1.WebhookClientConfig
// Objects are the objects that will use the ClientConfig above.
Objects []runtime.Object
}
// Provision provisions certificates for the WebhookClientConfig.
// It ensures the cert and CA are valid and not expiring.
// It updates the CABundle in the webhookClientConfig if necessary.
// It inject the WebhookClientConfig into options.Objects.
func (cp *Provisioner) Provision(options Options) (bool, error) {
if cp.CertWriter == nil {
return false, errors.New("CertWriter need to be set")
}
dnsName, err := dnsNameFromClientConfig(options.ClientConfig)
if err != nil {
return false, err
}
certs, changed, err := cp.CertWriter.EnsureCert(dnsName)
if err != nil {
return false, err
}
caBundle := options.ClientConfig.CABundle
caCert := certs.CACert
// TODO(mengqiy): limit the size of the CABundle by GC the old CA certificate
// this is important since the max record size in etcd is 1MB (latest version is 1.5MB).
if !bytes.Contains(caBundle, caCert) {
// Ensure the CA bundle in the webhook configuration has the signing CA.
options.ClientConfig.CABundle = append(caBundle, caCert...)
changed = true
}
return changed, cp.inject(options.ClientConfig, options.Objects)
}
// Inject the ClientConfig to the objects.
// It supports MutatingWebhookConfiguration and ValidatingWebhookConfiguration.
func (cp *Provisioner) inject(cc *admissionregistrationv1beta1.WebhookClientConfig, objs []runtime.Object) error {
if cc == nil {
return nil
}
for i := range objs {
switch typed := objs[i].(type) {
case *admissionregistrationv1beta1.MutatingWebhookConfiguration:
injectForEachWebhook(cc, typed.Webhooks)
case *admissionregistrationv1beta1.ValidatingWebhookConfiguration:
injectForEachWebhook(cc, typed.Webhooks)
default:
return fmt.Errorf("%#v is not supported for injecting a webhookClientConfig",
objs[i].GetObjectKind().GroupVersionKind())
}
}
return cp.CertWriter.Inject(objs...)
}
func injectForEachWebhook(
cc *admissionregistrationv1beta1.WebhookClientConfig,
webhooks []admissionregistrationv1beta1.Webhook) {
for i := range webhooks {
// only replacing the CA bundle to preserve the path in the WebhookClientConfig
webhooks[i].ClientConfig.CABundle = cc.CABundle
}
}
func dnsNameFromClientConfig(config *admissionregistrationv1beta1.WebhookClientConfig) (string, error) {
if config == nil {
return "", errors.New("clientConfig should not be empty")
}
if config.Service != nil && config.URL != nil {
return "", fmt.Errorf("service and URL can't be set at the same time in a webhook: %v", config)
}
if config.Service == nil && config.URL == nil {
return "", fmt.Errorf("one of service and URL need to be set in a webhook: %v", config)
}
if config.Service != nil {
return generator.ServiceToCommonName(config.Service.Namespace, config.Service.Name), nil
}
u, err := url.Parse(*config.URL)
if err != nil {
return "", err
}
host, _, err := net.SplitHostPort(u.Host)
if err != nil {
return u.Host, nil
}
return host, err
}

View File

@@ -0,0 +1,453 @@
/*
Copyright 2016 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package atomic
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/go-logr/logr"
"k8s.io/apimachinery/pkg/util/sets"
)
const (
maxFileNameLength = 255
maxPathLength = 4096
)
// AtomicWriter handles atomically projecting content for a set of files into
// a target directory.
//
// Note:
//
// 1. AtomicWriter reserves the set of pathnames starting with `..`.
// 2. AtomicWriter offers no concurrency guarantees and must be synchronized
// by the caller.
//
// The visible files in this volume are symlinks to files in the writer's data
// directory. Actual files are stored in a hidden timestamped directory which
// is symlinked to by the data directory. The timestamped directory and
// data directory symlink are created in the writer's target dir.  This scheme
// allows the files to be atomically updated by changing the target of the
// data directory symlink.
//
// Consumers of the target directory can monitor the ..data symlink using
// inotify or fanotify to receive events when the content in the volume is
// updated.
type AtomicWriter struct {
targetDir string
log logr.Logger
}
type FileProjection struct {
Data []byte
Mode int32
}
// NewAtomicWriter creates a new AtomicWriter configured to write to the given
// target directory, or returns an error if the target directory does not exist.
func NewAtomicWriter(targetDir string, log logr.Logger) (*AtomicWriter, error) {
_, err := os.Stat(targetDir)
if os.IsNotExist(err) {
return nil, err
}
return &AtomicWriter{targetDir: targetDir, log: log}, nil
}
const (
dataDirName = "..data"
newDataDirName = "..data_tmp"
)
// Write does an atomic projection of the given payload into the writer's target
// directory. Input paths must not begin with '..'.
//
// The Write algorithm is:
//
// 1. The payload is validated; if the payload is invalid, the function returns
// 2.  The current timestamped directory is detected by reading the data directory
// symlink
// 3. The old version of the volume is walked to determine whether any
// portion of the payload was deleted and is still present on disk.
// 4. The data in the current timestamped directory is compared to the projected
// data to determine if an update is required.
// 5.  A new timestamped dir is created
// 6. The payload is written to the new timestamped directory
// 7.  Symlinks and directory for new user-visible files are created (if needed).
//
// For example, consider the files:
// <target-dir>/podName
// <target-dir>/user/labels
// <target-dir>/k8s/annotations
//
// The user visible files are symbolic links into the internal data directory:
// <target-dir>/podName -> ..data/podName
// <target-dir>/usr -> ..data/usr
// <target-dir>/k8s -> ..data/k8s
//
// The data directory itself is a link to a timestamped directory with
// the real data:
// <target-dir>/..data -> ..2016_02_01_15_04_05.12345678/
// 8.  A symlink to the new timestamped directory ..data_tmp is created that will
// become the new data directory
// 9.  The new data directory symlink is renamed to the data directory; rename is atomic
// 10. Old paths are removed from the user-visible portion of the target directory
// 11.  The previous timestamped directory is removed, if it exists
func (w *AtomicWriter) Write(payload map[string]FileProjection) error {
// (1)
cleanPayload, err := validatePayload(payload)
if err != nil {
w.log.Error(err, "invalid payload")
return err
}
// (2)
dataDirPath := path.Join(w.targetDir, dataDirName)
oldTsDir, err := os.Readlink(dataDirPath)
if err != nil {
if !os.IsNotExist(err) {
w.log.Error(err, "unable to read link for data directory")
return err
}
// although Readlink() returns "" on err, don't be fragile by relying on it (since it's not specified in docs)
// empty oldTsDir indicates that it didn't exist
oldTsDir = ""
}
oldTsPath := path.Join(w.targetDir, oldTsDir)
var pathsToRemove sets.String
// if there was no old version, there's nothing to remove
if len(oldTsDir) != 0 {
// (3)
pathsToRemove, err = w.pathsToRemove(cleanPayload, oldTsPath)
if err != nil {
w.log.Error(err, "unable to determine user-visible files to remove")
return err
}
// (4)
if should, err := shouldWritePayload(cleanPayload, oldTsPath); err != nil {
w.log.Error(err, "unable to determine whether payload should be written to disk")
return err
} else if !should && len(pathsToRemove) == 0 {
w.log.V(1).Info("no update required for target directory", "directory", w.targetDir)
return nil
} else {
w.log.V(1).Info("write required for target directory", "directory", w.targetDir)
}
}
// (5)
tsDir, err := w.newTimestampDir()
if err != nil {
w.log.Error(err, "error creating new ts data directory")
return err
}
tsDirName := filepath.Base(tsDir)
// (6)
if err = w.writePayloadToDir(cleanPayload, tsDir); err != nil {
w.log.Error(err, "unable to write payload to ts data directory", "ts directory", tsDir)
return err
} else {
w.log.V(1).Info("performed write of new data to ts data directory", "ts directory", tsDir)
}
// (7)
if err = w.createUserVisibleFiles(cleanPayload); err != nil {
w.log.Error(err, "unable to create visible symlinks in target directory", "target directory", w.targetDir)
return err
}
// (8)
newDataDirPath := path.Join(w.targetDir, newDataDirName)
if err = os.Symlink(tsDirName, newDataDirPath); err != nil {
os.RemoveAll(tsDir)
w.log.Error(err, "unable to create symbolic link for atomic update")
return err
}
// (9)
if runtime.GOOS == "windows" {
os.Remove(dataDirPath)
err = os.Symlink(tsDirName, dataDirPath)
os.Remove(newDataDirPath)
} else {
err = os.Rename(newDataDirPath, dataDirPath)
}
if err != nil {
os.Remove(newDataDirPath)
os.RemoveAll(tsDir)
w.log.Error(err, "unable to rename symbolic link for data directory", "data directory", newDataDirPath)
return err
}
// (10)
if err = w.removeUserVisiblePaths(pathsToRemove); err != nil {
w.log.Error(err, "unable to remove old visible symlinks")
return err
}
// (11)
if len(oldTsDir) > 0 {
if err = os.RemoveAll(oldTsPath); err != nil {
w.log.Error(err, "unable to remove old data directory", "data directory", oldTsDir)
return err
}
}
return nil
}
// validatePayload returns an error if any path in the payload returns a copy of the payload with the paths cleaned.
func validatePayload(payload map[string]FileProjection) (map[string]FileProjection, error) {
cleanPayload := make(map[string]FileProjection)
for k, content := range payload {
if err := validatePath(k); err != nil {
return nil, err
}
cleanPayload[filepath.Clean(k)] = content
}
return cleanPayload, nil
}
// validatePath validates a single path, returning an error if the path is
// invalid. paths may not:
//
// 1. be absolute
// 2. contain '..' as an element
// 3. start with '..'
// 4. contain filenames larger than 255 characters
// 5. be longer than 4096 characters
func validatePath(targetPath string) error {
// TODO: somehow unify this with the similar api validation,
// validateVolumeSourcePath; the error semantics are just different enough
// from this that it was time-prohibitive trying to find the right
// refactoring to re-use.
if targetPath == "" {
return fmt.Errorf("invalid path: must not be empty: %q", targetPath)
}
if path.IsAbs(targetPath) {
return fmt.Errorf("invalid path: must be relative path: %s", targetPath)
}
if len(targetPath) > maxPathLength {
return fmt.Errorf("invalid path: must be less than or equal to %d characters", maxPathLength)
}
items := strings.Split(targetPath, string(os.PathSeparator))
for _, item := range items {
if item == ".." {
return fmt.Errorf("invalid path: must not contain '..': %s", targetPath)
}
if len(item) > maxFileNameLength {
return fmt.Errorf("invalid path: filenames must be less than or equal to %d characters", maxFileNameLength)
}
}
if strings.HasPrefix(items[0], "..") && len(items[0]) > 2 {
return fmt.Errorf("invalid path: must not start with '..': %s", targetPath)
}
return nil
}
// shouldWritePayload returns whether the payload should be written to disk.
func shouldWritePayload(payload map[string]FileProjection, oldTsDir string) (bool, error) {
for userVisiblePath, fileProjection := range payload {
shouldWrite, err := shouldWriteFile(path.Join(oldTsDir, userVisiblePath), fileProjection.Data)
if err != nil {
return false, err
}
if shouldWrite {
return true, nil
}
}
return false, nil
}
// shouldWriteFile returns whether a new version of a file should be written to disk.
func shouldWriteFile(path string, content []byte) (bool, error) {
_, err := os.Lstat(path)
if os.IsNotExist(err) {
return true, nil
}
contentOnFs, err := ioutil.ReadFile(path)
if err != nil {
return false, err
}
return (bytes.Compare(content, contentOnFs) != 0), nil
}
// pathsToRemove walks the current version of the data directory and
// determines which paths should be removed (if any) after the payload is
// written to the target directory.
func (w *AtomicWriter) pathsToRemove(payload map[string]FileProjection, oldTsDir string) (sets.String, error) {
paths := sets.NewString()
visitor := func(path string, info os.FileInfo, err error) error {
relativePath := strings.TrimPrefix(path, oldTsDir)
relativePath = strings.TrimPrefix(relativePath, string(os.PathSeparator))
if relativePath == "" {
return nil
}
paths.Insert(relativePath)
return nil
}
err := filepath.Walk(oldTsDir, visitor)
if os.IsNotExist(err) {
return nil, nil
} else if err != nil {
return nil, err
}
w.log.V(1).Info("current paths", "target directory", w.targetDir, "paths", paths.List())
newPaths := sets.NewString()
for file := range payload {
// add all subpaths for the payload to the set of new paths
// to avoid attempting to remove non-empty dirs
for subPath := file; subPath != ""; {
newPaths.Insert(subPath)
subPath, _ = filepath.Split(subPath)
subPath = strings.TrimSuffix(subPath, string(os.PathSeparator))
}
}
w.log.V(1).Info("new paths", "target directory", w.targetDir, "paths", newPaths.List())
result := paths.Difference(newPaths)
w.log.V(1).Info("paths to remove", "target directory", w.targetDir, "paths", result)
return result, nil
}
// newTimestampDir creates a new timestamp directory
func (w *AtomicWriter) newTimestampDir() (string, error) {
tsDir, err := ioutil.TempDir(w.targetDir, time.Now().UTC().Format("..2006_01_02_15_04_05."))
if err != nil {
w.log.Error(err, "unable to create new temp directory")
return "", err
}
// 0755 permissions are needed to allow 'group' and 'other' to recurse the
// directory tree. do a chmod here to ensure that permissions are set correctly
// regardless of the process' umask.
err = os.Chmod(tsDir, 0755)
if err != nil {
w.log.Error(err, "unable to set mode on new temp directory")
return "", err
}
return tsDir, nil
}
// writePayloadToDir writes the given payload to the given directory. The
// directory must exist.
func (w *AtomicWriter) writePayloadToDir(payload map[string]FileProjection, dir string) error {
for userVisiblePath, fileProjection := range payload {
content := fileProjection.Data
mode := os.FileMode(fileProjection.Mode)
fullPath := path.Join(dir, userVisiblePath)
baseDir, _ := filepath.Split(fullPath)
err := os.MkdirAll(baseDir, os.ModePerm)
if err != nil {
w.log.Error(err, "unable to create directory", "directory", baseDir)
return err
}
err = ioutil.WriteFile(fullPath, content, mode)
if err != nil {
w.log.Error(err, "unable to write file", "file", fullPath, "mode", mode)
return err
}
// Chmod is needed because ioutil.WriteFile() ends up calling
// open(2) to create the file, so the final mode used is "mode &
// ~umask". But we want to make sure the specified mode is used
// in the file no matter what the umask is.
err = os.Chmod(fullPath, mode)
if err != nil {
w.log.Error(err, "unable to write file", "file", fullPath, "mode", mode)
}
}
return nil
}
// createUserVisibleFiles creates the relative symlinks for all the
// files configured in the payload. If the directory in a file path does not
// exist, it is created.
//
// Viz:
// For files: "bar", "foo/bar", "baz/bar", "foo/baz/blah"
// the following symlinks are created:
// bar -> ..data/bar
// foo -> ..data/foo
// baz -> ..data/baz
func (w *AtomicWriter) createUserVisibleFiles(payload map[string]FileProjection) error {
for userVisiblePath := range payload {
slashpos := strings.Index(userVisiblePath, string(os.PathSeparator))
if slashpos == -1 {
slashpos = len(userVisiblePath)
}
linkname := userVisiblePath[:slashpos]
_, err := os.Readlink(path.Join(w.targetDir, linkname))
if err != nil && os.IsNotExist(err) {
// The link into the data directory for this path doesn't exist; create it
visibleFile := path.Join(w.targetDir, linkname)
dataDirFile := path.Join(dataDirName, linkname)
err = os.Symlink(dataDirFile, visibleFile)
if err != nil {
return err
}
}
}
return nil
}
// removeUserVisiblePaths removes the set of paths from the user-visible
// portion of the writer's target directory.
func (w *AtomicWriter) removeUserVisiblePaths(paths sets.String) error {
ps := string(os.PathSeparator)
var lasterr error
for p := range paths {
// only remove symlinks from the volume root directory (i.e. items that don't contain '/')
if strings.Contains(p, ps) {
continue
}
if err := os.Remove(path.Join(w.targetDir, p)); err != nil {
w.log.Error(err, "unable to prune old user-visible path", "path", p)
lasterr = err
}
}
return lasterr
}

View File

@@ -0,0 +1,137 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package writer
import (
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"time"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/generator"
)
const (
// CAKeyName is the name of the CA private key
CAKeyName = "ca-key.pem"
// CACertName is the name of the CA certificate
CACertName = "ca-cert.pem"
// ServerKeyName is the name of the server private key
ServerKeyName = "key.pem"
// ServerCertName is the name of the serving certificate
ServerCertName = "cert.pem"
)
// CertWriter provides method to handle webhooks.
type CertWriter interface {
// EnsureCert provisions the cert for the webhookClientConfig.
EnsureCert(dnsName string) (*generator.Artifacts, bool, error)
// Inject injects the necessary information given the objects.
// It supports MutatingWebhookConfiguration and ValidatingWebhookConfiguration.
Inject(objs ...runtime.Object) error
}
// handleCommon ensures the given webhook has a proper certificate.
// It uses the given certReadWriter to read and (or) write the certificate.
func handleCommon(dnsName string, ch certReadWriter) (*generator.Artifacts, bool, error) {
if len(dnsName) == 0 {
return nil, false, errors.New("dnsName should not be empty")
}
if ch == nil {
return nil, false, errors.New("certReaderWriter should not be nil")
}
certs, changed, err := createIfNotExists(ch)
if err != nil {
return nil, changed, err
}
// Recreate the cert if it's invalid.
valid := validCert(certs, dnsName)
if !valid {
log.Info("cert is invalid or expiring, regenerating a new one")
certs, err = ch.overwrite()
if err != nil {
return nil, false, err
}
changed = true
}
return certs, changed, nil
}
func createIfNotExists(ch certReadWriter) (*generator.Artifacts, bool, error) {
// Try to read first
certs, err := ch.read()
if isNotFound(err) {
// Create if not exists
certs, err = ch.write()
switch {
// This may happen if there is another racer.
case isAlreadyExists(err):
certs, err = ch.read()
return certs, true, err
default:
return certs, true, err
}
}
return certs, false, err
}
// certReadWriter provides methods for reading and writing certificates.
type certReadWriter interface {
// read reads a webhook name and returns the certs for it.
read() (*generator.Artifacts, error)
// write writes the certs and return the certs it wrote.
write() (*generator.Artifacts, error)
// overwrite overwrites the existing certs and return the certs it wrote.
overwrite() (*generator.Artifacts, error)
}
func validCert(certs *generator.Artifacts, dnsName string) bool {
if certs == nil {
return false
}
// Verify key and cert are valid pair
_, err := tls.X509KeyPair(certs.Cert, certs.Key)
if err != nil {
return false
}
// Verify cert is good for desired DNS name and signed by CA and will be valid for desired period of time.
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(certs.CACert) {
return false
}
block, _ := pem.Decode([]byte(certs.Cert))
if block == nil {
return false
}
cert, err := x509.ParseCertificate(block.Bytes)
if err != nil {
return false
}
ops := x509.VerifyOptions{
DNSName: dnsName,
Roots: pool,
CurrentTime: time.Now().AddDate(0, 6, 0),
}
_, err = cert.Verify(ops)
return err == nil
}

View File

@@ -0,0 +1,64 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
/*
Package writer provides method to provision and persist the certificates.
It will create the certificates if they don't exist.
It will ensure the certificates are valid and not expiring. If not, it will recreate them.
Create a CertWriter that can write the certificate to secret
writer, err := NewSecretCertWriter(SecretCertWriterOptions{
Secret: types.NamespacedName{Namespace: "foo", Name: "bar"},
Client: client,
})
if err != nil {
// handler error
}
Create a CertWriter that can write the certificate to the filesystem.
writer, err := NewFSCertWriter(FSCertWriterOptions{
Path: "path/to/cert/",
})
if err != nil {
// handler error
}
Provision the certificates using the CertWriter. The certificate will be available in the desired secret or
the desired path.
// writer can be either one of the CertWriters created above
certs, changed, err := writer.EnsureCerts("admissionwebhook.k8s.io", false)
if err != nil {
// handler error
}
Inject necessary information given the objects.
err = writer.Inject(objs...)
if err != nil {
// handler error
}
*/
package writer
import (
logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
)
var log = logf.KBLog.WithName("admission").WithName("cert").WithName("writer")

View File

@@ -0,0 +1,43 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package writer
type notFoundError struct {
err error
}
func (e notFoundError) Error() string {
return e.err.Error()
}
func isNotFound(err error) bool {
_, ok := err.(notFoundError)
return ok
}
type alreadyExistError struct {
err error
}
func (e alreadyExistError) Error() string {
return e.err.Error()
}
func isAlreadyExists(err error) bool {
_, ok := err.(alreadyExistError)
return ok
}

View File

@@ -0,0 +1,216 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package writer
import (
"errors"
"fmt"
"io/ioutil"
"os"
"path"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/generator"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/writer/atomic"
)
// fsCertWriter provisions the certificate by reading and writing to the filesystem.
type fsCertWriter struct {
// dnsName is the DNS name that the certificate is for.
dnsName string
*FSCertWriterOptions
}
// FSCertWriterOptions are options for constructing a FSCertWriter.
type FSCertWriterOptions struct {
// certGenerator generates the certificates.
CertGenerator generator.CertGenerator
// path is the directory that the certificate and private key and CA certificate will be written.
Path string
}
var _ CertWriter = &fsCertWriter{}
func (ops *FSCertWriterOptions) setDefaults() {
if ops.CertGenerator == nil {
ops.CertGenerator = &generator.SelfSignedCertGenerator{}
}
}
func (ops *FSCertWriterOptions) validate() error {
if len(ops.Path) == 0 {
return errors.New("path must be set in FSCertWriterOptions")
}
return nil
}
// NewFSCertWriter constructs a CertWriter that persists the certificate on filesystem.
func NewFSCertWriter(ops FSCertWriterOptions) (CertWriter, error) {
ops.setDefaults()
err := ops.validate()
if err != nil {
return nil, err
}
return &fsCertWriter{
FSCertWriterOptions: &ops,
}, nil
}
// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates in the filesystem.
func (f *fsCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) {
// create or refresh cert and write it to fs
f.dnsName = dnsName
return handleCommon(f.dnsName, f)
}
func (f *fsCertWriter) write() (*generator.Artifacts, error) {
return f.doWrite()
}
func (f *fsCertWriter) overwrite() (*generator.Artifacts, error) {
return f.doWrite()
}
func (f *fsCertWriter) doWrite() (*generator.Artifacts, error) {
certs, err := f.CertGenerator.Generate(f.dnsName)
if err != nil {
return nil, err
}
// AtomicWriter's algorithm only manages files using symbolic link.
// If a file is not a symbolic link, will ignore the update for it.
// We want to cleanup for AtomicWriter by removing old files that are not symbolic links.
err = prepareToWrite(f.Path)
if err != nil {
return nil, err
}
aw, err := atomic.NewAtomicWriter(f.Path, log.WithName("atomic-writer").
WithValues("task", "processing webhook"))
if err != nil {
return nil, err
}
err = aw.Write(certToProjectionMap(certs))
return certs, err
}
// prepareToWrite ensures it directory is compatible with the atomic.Writer library.
func prepareToWrite(dir string) error {
_, err := os.Stat(dir)
switch {
case os.IsNotExist(err):
log.Info("cert directory doesn't exist, creating", "directory", dir)
// TODO: figure out if we can reduce the permission. (Now it's 0777)
err = os.MkdirAll(dir, 0777)
if err != nil {
return fmt.Errorf("can't create dir: %v", dir)
}
case err != nil:
return err
}
filenames := []string{CAKeyName, CACertName, ServerCertName, ServerKeyName}
for _, f := range filenames {
abspath := path.Join(dir, f)
_, err := os.Stat(abspath)
if os.IsNotExist(err) {
continue
} else if err != nil {
log.Error(err, "unable to stat file", "file", abspath)
}
_, err = os.Readlink(abspath)
// if it's not a symbolic link
if err != nil {
err = os.Remove(abspath)
if err != nil {
log.Error(err, "unable to remove old file", "file", abspath)
}
}
}
return nil
}
func (f *fsCertWriter) read() (*generator.Artifacts, error) {
if err := ensureExist(f.Path); err != nil {
return nil, err
}
caKeyBytes, err := ioutil.ReadFile(path.Join(f.Path, CAKeyName))
if err != nil {
return nil, err
}
caCertBytes, err := ioutil.ReadFile(path.Join(f.Path, CACertName))
if err != nil {
return nil, err
}
certBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerCertName))
if err != nil {
return nil, err
}
keyBytes, err := ioutil.ReadFile(path.Join(f.Path, ServerKeyName))
if err != nil {
return nil, err
}
return &generator.Artifacts{
CAKey: caKeyBytes,
CACert: caCertBytes,
Cert: certBytes,
Key: keyBytes,
}, nil
}
func ensureExist(dir string) error {
filenames := []string{CAKeyName, CACertName, ServerCertName, ServerKeyName}
for _, filename := range filenames {
_, err := os.Stat(path.Join(dir, filename))
switch {
case err == nil:
continue
case os.IsNotExist(err):
return notFoundError{err}
default:
return err
}
}
return nil
}
func certToProjectionMap(cert *generator.Artifacts) map[string]atomic.FileProjection {
// TODO: figure out if we can reduce the permission. (Now it's 0666)
return map[string]atomic.FileProjection{
CAKeyName: {
Data: cert.CAKey,
Mode: 0666,
},
CACertName: {
Data: cert.CACert,
Mode: 0666,
},
ServerCertName: {
Data: cert.Cert,
Mode: 0666,
},
ServerKeyName: {
Data: cert.Key,
Mode: 0666,
},
}
}
func (f *fsCertWriter) Inject(objs ...runtime.Object) error {
return nil
}

View File

@@ -0,0 +1,184 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package writer
import (
"errors"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/generator"
)
// secretCertWriter provisions the certificate by reading and writing to the k8s secrets.
type secretCertWriter struct {
*SecretCertWriterOptions
// dnsName is the DNS name that the certificate is for.
dnsName string
}
// SecretCertWriterOptions is options for constructing a secretCertWriter.
type SecretCertWriterOptions struct {
// client talks to a kubernetes cluster for creating the secret.
Client client.Client
// certGenerator generates the certificates.
CertGenerator generator.CertGenerator
// secret points the secret that contains certificates that written by the CertWriter.
Secret *types.NamespacedName
}
var _ CertWriter = &secretCertWriter{}
func (ops *SecretCertWriterOptions) setDefaults() {
if ops.CertGenerator == nil {
ops.CertGenerator = &generator.SelfSignedCertGenerator{}
}
}
func (ops *SecretCertWriterOptions) validate() error {
if ops.Client == nil {
return errors.New("client must be set in SecretCertWriterOptions")
}
if ops.Secret == nil {
return errors.New("secret must be set in SecretCertWriterOptions")
}
return nil
}
// NewSecretCertWriter constructs a CertWriter that persists the certificate in a k8s secret.
func NewSecretCertWriter(ops SecretCertWriterOptions) (CertWriter, error) {
ops.setDefaults()
err := ops.validate()
if err != nil {
return nil, err
}
return &secretCertWriter{
SecretCertWriterOptions: &ops,
}, nil
}
// EnsureCert provisions certificates for a webhookClientConfig by writing the certificates to a k8s secret.
func (s *secretCertWriter) EnsureCert(dnsName string) (*generator.Artifacts, bool, error) {
// Create or refresh the certs based on clientConfig
s.dnsName = dnsName
return handleCommon(s.dnsName, s)
}
var _ certReadWriter = &secretCertWriter{}
func (s *secretCertWriter) buildSecret() (*corev1.Secret, *generator.Artifacts, error) {
certs, err := s.CertGenerator.Generate(s.dnsName)
if err != nil {
return nil, nil, err
}
secret := certsToSecret(certs, *s.Secret)
return secret, certs, err
}
func (s *secretCertWriter) write() (*generator.Artifacts, error) {
secret, certs, err := s.buildSecret()
if err != nil {
return nil, err
}
err = s.Client.Create(nil, secret)
if apierrors.IsAlreadyExists(err) {
return nil, alreadyExistError{err}
}
return certs, err
}
func (s *secretCertWriter) overwrite() (
*generator.Artifacts, error) {
secret, certs, err := s.buildSecret()
if err != nil {
return nil, err
}
err = s.Client.Update(nil, secret)
return certs, err
}
func (s *secretCertWriter) read() (*generator.Artifacts, error) {
secret := &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
}
err := s.Client.Get(nil, *s.Secret, secret)
if apierrors.IsNotFound(err) {
return nil, notFoundError{err}
}
certs := secretToCerts(secret)
if certs != nil {
// Store the CA for next usage.
s.CertGenerator.SetCA(certs.CAKey, certs.CACert)
}
return certs, nil
}
func secretToCerts(secret *corev1.Secret) *generator.Artifacts {
if secret.Data == nil {
return nil
}
return &generator.Artifacts{
CAKey: secret.Data[CAKeyName],
CACert: secret.Data[CACertName],
Cert: secret.Data[ServerCertName],
Key: secret.Data[ServerKeyName],
}
}
func certsToSecret(certs *generator.Artifacts, sec types.NamespacedName) *corev1.Secret {
return &corev1.Secret{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Secret",
},
ObjectMeta: metav1.ObjectMeta{
Namespace: sec.Namespace,
Name: sec.Name,
},
Data: map[string][]byte{
CAKeyName: certs.CAKey,
CACertName: certs.CACert,
ServerKeyName: certs.Key,
ServerCertName: certs.Cert,
},
}
}
// Inject sets the ownerReference in the secret.
func (s *secretCertWriter) Inject(objs ...runtime.Object) error {
// TODO: figure out how to get the UID
//for i := range objs {
// accessor, err := meta.Accessor(objs[i])
// if err != nil {
// return err
// }
// err = controllerutil.SetControllerReference(accessor, s.sec, scheme.Scheme)
// if err != nil {
// return err
// }
//}
//return s.client.Update(context.Background(), s.sec)
return nil
}

View File

@@ -0,0 +1,308 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
"fmt"
"io"
"net/http"
"path"
"sync"
"time"
"k8s.io/apimachinery/pkg/runtime"
apitypes "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/runtime/inject"
atypes "sigs.k8s.io/controller-runtime/pkg/webhook/admission/types"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert"
"sigs.k8s.io/controller-runtime/pkg/webhook/internal/cert/writer"
"sigs.k8s.io/controller-runtime/pkg/webhook/types"
)
// default interval for checking cert is 90 days (~3 months)
var defaultCertRefreshInterval = 3 * 30 * 24 * time.Hour
// ServerOptions are options for configuring an admission webhook server.
type ServerOptions struct {
// Port is the port number that the server will serve.
// It will be defaulted to 443 if unspecified.
Port int32
// CertDir is the directory that contains the server key and certificate.
// If using FSCertWriter in Provisioner, the server itself will provision the certificate and
// store it in this directory.
// If using SecretCertWriter in Provisioner, the server will provision the certificate in a secret,
// the user is responsible to mount the secret to the this location for the server to consume.
CertDir string
// Client is a client defined in controller-runtime instead of a client-go client.
// It knows how to talk to a kubernetes cluster.
// Client will be injected by the manager if not set.
Client client.Client
// DisableWebhookConfigInstaller controls if the server will automatically create webhook related objects
// during bootstrapping. e.g. webhookConfiguration, service and secret.
// If false, the server will install the webhook config objects. It is defaulted to false.
DisableWebhookConfigInstaller *bool
// BootstrapOptions contains the options for bootstrapping the admission server.
*BootstrapOptions
}
// BootstrapOptions are options for bootstrapping an admission webhook server.
type BootstrapOptions struct {
// MutatingWebhookConfigName is the name that used for creating the MutatingWebhookConfiguration object.
MutatingWebhookConfigName string
// ValidatingWebhookConfigName is the name that used for creating the ValidatingWebhookConfiguration object.
ValidatingWebhookConfigName string
// Secret is the location for storing the certificate for the admission server.
// The server should have permission to create a secret in the namespace.
// This is optional. If unspecified, it will write to the filesystem.
// It the secret already exists and is different from the desired, it will be replaced.
Secret *apitypes.NamespacedName
// Deprecated: Writer will not be used anywhere.
Writer io.Writer
// Service is k8s service fronting the webhook server pod(s).
// This field is optional. But one and only one of Service and Host need to be set.
// This maps to field .webhooks.getClientConfig.service
// https://github.com/kubernetes/api/blob/183f3326a9353bd6d41430fc80f96259331d029c/admissionregistration/v1beta1/types.go#L260
Service *Service
// Host is the host name of .webhooks.clientConfig.url
// https://github.com/kubernetes/api/blob/183f3326a9353bd6d41430fc80f96259331d029c/admissionregistration/v1beta1/types.go#L250
// This field is optional. But one and only one of Service and Host need to be set.
// If neither Service nor Host is unspecified, Host will be defaulted to "localhost".
Host *string
// certProvisioner is constructed using certGenerator and certWriter
certProvisioner *cert.Provisioner // nolint: structcheck
// err will be non-nil if there is an error occur during initialization.
err error // nolint: structcheck
}
// Service contains information for creating a service
type Service struct {
// Name of the service
Name string
// Namespace of the service
Namespace string
// Selectors is the selector of the service.
// This must select the pods that runs this webhook server.
Selectors map[string]string
}
// Server is an admission webhook server that can serve traffic and
// generates related k8s resources for deploying.
type Server struct {
// Name is the name of server
Name string
// ServerOptions contains options for configuring the admission server.
ServerOptions
sMux *http.ServeMux
// registry maps a path to a http.Handler.
registry map[string]Webhook
// mutatingWebhookConfiguration and validatingWebhookConfiguration are populated during server bootstrapping.
// They can be nil, if there is no webhook registered under it.
webhookConfigurations []runtime.Object
// manager is the manager that this webhook server will be registered.
manager manager.Manager
// httpServer is the actual server that serves the traffic.
httpServer *http.Server
once sync.Once
}
// Webhook defines the basics that a webhook should support.
type Webhook interface {
// GetName returns the name of the webhook.
GetName() string
// GetPath returns the path that the webhook registered.
GetPath() string
// GetType returns the Type of the webhook.
// e.g. mutating or validating
GetType() types.WebhookType
// Handler returns a http.Handler for the webhook.
Handler() http.Handler
// Validate validates if the webhook itself is valid.
// If invalid, a non-nil error will be returned.
Validate() error
}
// NewServer creates a new admission webhook server.
func NewServer(name string, mgr manager.Manager, options ServerOptions) (*Server, error) {
as := &Server{
Name: name,
sMux: http.NewServeMux(),
registry: map[string]Webhook{},
ServerOptions: options,
manager: mgr,
}
return as, nil
}
// Register validates and registers webhook(s) in the server
func (s *Server) Register(webhooks ...Webhook) error {
for i, webhook := range webhooks {
// validate the webhook before registering it.
err := webhook.Validate()
if err != nil {
return err
}
_, found := s.registry[webhook.GetPath()]
if found {
return fmt.Errorf("can't register duplicate path: %v", webhook.GetPath())
}
s.registry[webhook.GetPath()] = webhooks[i]
s.sMux.Handle(webhook.GetPath(), webhook.Handler())
}
// Lazily add Server to manager.
// Because the all webhook handlers to be in place, so we can inject the things they need.
return s.manager.Add(s)
}
// Handle registers a http.Handler for the given pattern.
func (s *Server) Handle(pattern string, handler http.Handler) {
s.sMux.Handle(pattern, handler)
}
var _ manager.Runnable = &Server{}
// Start runs the server.
// It will install the webhook related resources depend on the server configuration.
func (s *Server) Start(stop <-chan struct{}) error {
s.once.Do(s.setDefault)
if s.err != nil {
return s.err
}
if s.DisableWebhookConfigInstaller != nil && !*s.DisableWebhookConfigInstaller {
log.Info("installing webhook configuration in cluster")
err := s.InstallWebhookManifests()
if err != nil {
return err
}
} else {
log.Info("webhook installer is disabled")
}
return s.run(stop)
}
func (s *Server) run(stop <-chan struct{}) error { // nolint: gocyclo
errCh := make(chan error)
serveFn := func() {
s.httpServer = &http.Server{
Addr: fmt.Sprintf(":%v", s.Port),
Handler: s.sMux,
}
log.Info("starting the webhook server.")
errCh <- s.httpServer.ListenAndServeTLS(path.Join(s.CertDir, writer.ServerCertName), path.Join(s.CertDir, writer.ServerKeyName))
}
shutdownHappend := false
timer := time.Tick(wait.Jitter(defaultCertRefreshInterval, 0.1))
go serveFn()
for {
select {
case <-timer:
changed, err := s.RefreshCert()
if err != nil {
log.Error(err, "encountering error when refreshing the certificate")
return err
}
if !changed {
log.Info("no need to reload the certificates.")
continue
}
log.Info("server is shutting down to reload the certificates.")
shutdownHappend = true
err = s.httpServer.Shutdown(context.Background())
if err != nil {
log.Error(err, "encountering error when shutting down")
return err
}
timer = time.Tick(wait.Jitter(defaultCertRefreshInterval, 0.1))
go serveFn()
case <-stop:
return s.httpServer.Shutdown(context.Background())
case e := <-errCh:
// Don't exit when getting an http.ErrServerClosed error due to restarting the server.
if shutdownHappend && e == http.ErrServerClosed {
shutdownHappend = false
} else if e != nil {
log.Error(e, "server returns an unexpected error")
return e
}
}
}
}
// RefreshCert refreshes the certificate using Server's Provisioner if the certificate is expiring.
func (s *Server) RefreshCert() (bool, error) {
cc, err := s.getClientConfig()
if err != nil {
return false, err
}
changed, err := s.certProvisioner.Provision(cert.Options{
ClientConfig: cc,
Objects: s.webhookConfigurations,
})
if err != nil {
return false, err
}
return changed, batchCreateOrReplace(s.Client, s.webhookConfigurations...)
}
var _ inject.Client = &Server{}
// InjectClient injects the client into the server
func (s *Server) InjectClient(c client.Client) error {
s.Client = c
for _, wh := range s.registry {
if _, err := inject.ClientInto(c, wh.Handler()); err != nil {
return err
}
}
return nil
}
var _ inject.Decoder = &Server{}
// InjectDecoder injects the client into the server
func (s *Server) InjectDecoder(d atypes.Decoder) error {
for _, wh := range s.registry {
if _, err := inject.DecoderInto(d, wh.Handler()); err != nil {
return err
}
}
return nil
}

View File

@@ -0,0 +1,115 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package webhook
import (
"context"
admissionregistration "k8s.io/api/admissionregistration/v1beta1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
type mutateFn func(current, desired *runtime.Object) error
var serviceFn = func(current, desired *runtime.Object) error {
typedC := (*current).(*corev1.Service)
typedD := (*desired).(*corev1.Service)
typedC.Spec.Selector = typedD.Spec.Selector
return nil
}
var mutatingWebhookConfigFn = func(current, desired *runtime.Object) error {
typedC := (*current).(*admissionregistration.MutatingWebhookConfiguration)
typedD := (*desired).(*admissionregistration.MutatingWebhookConfiguration)
typedC.Webhooks = typedD.Webhooks
return nil
}
var validatingWebhookConfigFn = func(current, desired *runtime.Object) error {
typedC := (*current).(*admissionregistration.ValidatingWebhookConfiguration)
typedD := (*desired).(*admissionregistration.ValidatingWebhookConfiguration)
typedC.Webhooks = typedD.Webhooks
return nil
}
var genericFn = func(current, desired *runtime.Object) error {
*current = *desired
return nil
}
// createOrReplaceHelper creates the object if it doesn't exist;
// otherwise, it will replace it.
// When replacing, fn should know how to preserve existing fields in the object GET from the APIServer.
// TODO: use the helper in #98 when it merges.
func createOrReplaceHelper(c client.Client, obj runtime.Object, fn mutateFn) error {
if obj == nil {
return nil
}
err := c.Create(context.Background(), obj)
if apierrors.IsAlreadyExists(err) {
// TODO: retry mutiple times with backoff if necessary.
existing := obj.DeepCopyObject()
objectKey, err := client.ObjectKeyFromObject(obj)
if err != nil {
return err
}
err = c.Get(context.Background(), objectKey, existing)
if err != nil {
return err
}
err = fn(&existing, &obj)
if err != nil {
return err
}
return c.Update(context.Background(), existing)
}
return err
}
// createOrReplace creates the object if it doesn't exist;
// otherwise, it will replace it.
// When replacing, it knows how to preserve existing fields in the object GET from the APIServer.
// It currently only support MutatingWebhookConfiguration, ValidatingWebhookConfiguration and Service.
// For other kinds, it uses genericFn to replace the whole object.
func createOrReplace(c client.Client, obj runtime.Object) error {
if obj == nil {
return nil
}
switch obj.(type) {
case *admissionregistration.MutatingWebhookConfiguration:
return createOrReplaceHelper(c, obj, mutatingWebhookConfigFn)
case *admissionregistration.ValidatingWebhookConfiguration:
return createOrReplaceHelper(c, obj, validatingWebhookConfigFn)
case *corev1.Service:
return createOrReplaceHelper(c, obj, serviceFn)
default:
return createOrReplaceHelper(c, obj, genericFn)
}
}
func batchCreateOrReplace(c client.Client, objs ...runtime.Object) error {
for i := range objs {
err := createOrReplace(c, objs[i])
if err != nil {
return err
}
}
return nil
}