From b6b6f14fb6cd19837988190d6872177cae8b4f92 Mon Sep 17 00:00:00 2001 From: hongming Date: Tue, 14 Jan 2020 15:07:13 +0800 Subject: [PATCH] use CSR generate user client certificate Signed-off-by: hongming --- go.mod | 1 + pkg/models/kubeconfig/internal/pki_helpers.go | 64 +++ pkg/models/kubeconfig/kubeconfig.go | 411 ++++++++---------- 3 files changed, 236 insertions(+), 240 deletions(-) create mode 100644 pkg/models/kubeconfig/internal/pki_helpers.go diff --git a/go.mod b/go.mod index f121df41c..6724572a0 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/openshift/api v3.9.0+incompatible // indirect github.com/pborman/uuid v0.0.0-20180906182336-adf5a7427709 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect + github.com/pkg/errors v0.8.1 github.com/projectcalico/go-json v0.0.0-20161128004156-6219dc7339ba // indirect github.com/projectcalico/go-yaml v0.0.0-20161201183616-955bc3e451ef // indirect github.com/projectcalico/go-yaml-wrapper v0.0.0-20161127220527-598e54215bee // indirect diff --git a/pkg/models/kubeconfig/internal/pki_helpers.go b/pkg/models/kubeconfig/internal/pki_helpers.go new file mode 100644 index 000000000..16baa0579 --- /dev/null +++ b/pkg/models/kubeconfig/internal/pki_helpers.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package pkiutil + +import ( + "crypto" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "github.com/pkg/errors" + certutil "k8s.io/client-go/util/cert" +) + +// NewCSRAndKey generates a new key and CSR and that could be signed to create the given certificate +func NewCSRAndKey(config *certutil.Config) (*x509.CertificateRequest, *rsa.PrivateKey, error) { + key, err := certutil.NewPrivateKey() + if err != nil { + return nil, nil, errors.Wrap(err, "unable to create private key") + } + + csr, err := NewCSR(*config, key) + if err != nil { + return nil, nil, errors.Wrap(err, "unable to generate CSR") + } + + return csr, key, nil +} + +// NewCSR creates a new CSR +func NewCSR(cfg certutil.Config, key crypto.Signer) (*x509.CertificateRequest, error) { + template := &x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: cfg.CommonName, + Organization: cfg.Organization, + }, + DNSNames: cfg.AltNames.DNSNames, + IPAddresses: cfg.AltNames.IPs, + } + + csrBytes, err := x509.CreateCertificateRequest(cryptorand.Reader, template, key) + + if err != nil { + return nil, errors.Wrap(err, "failed to create a CSR") + } + + return x509.ParseCertificateRequest(csrBytes) +} diff --git a/pkg/models/kubeconfig/kubeconfig.go b/pkg/models/kubeconfig/kubeconfig.go index e1a2bd8ed..34fa68db3 100644 --- a/pkg/models/kubeconfig/kubeconfig.go +++ b/pkg/models/kubeconfig/kubeconfig.go @@ -21,294 +21,225 @@ package kubeconfig import ( "bytes" "crypto/rand" - "crypto/rsa" "crypto/x509" - "crypto/x509/pkix" - "encoding/base64" "encoding/pem" "fmt" - "gopkg.in/yaml.v2" - "io/ioutil" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/simple/client" - "math/big" - rd "math/rand" - "time" - - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - - "k8s.io/api/core/v1" - + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + certutil "k8s.io/client-go/util/cert" + "k8s.io/klog" "kubesphere.io/kubesphere/pkg/constants" + pkiutil "kubesphere.io/kubesphere/pkg/models/kubeconfig/internal" + "kubesphere.io/kubesphere/pkg/simple/client" + "time" ) const ( - caPath = "/etc/kubernetes/pki/ca.crt" - keyPath = "/etc/kubernetes/pki/ca.key" - clusterName = "kubernetes" - kubectlConfigKey = "config" - defaultNamespace = "default" + kubeconfigNameFormat = "kubeconfig-%s" + defaultClusterName = "local" + defaultNamespace = "default" + fileName = "config" + configMapKind = "ConfigMap" + configMapAPIVersion = "v1" ) -type clusterInfo struct { - CertificateAuthorityData string `yaml:"certificate-authority-data"` - Server string `yaml:"server"` -} - -type cluster struct { - Cluster clusterInfo `yaml:"cluster"` - Name string `yaml:"name"` -} - -type contextInfo struct { - Cluster string `yaml:"cluster"` - User string `yaml:"user"` - NameSpace string `yaml:"namespace"` -} - -type contextObject struct { - Context contextInfo `yaml:"context"` - Name string `yaml:"name"` -} - -type userInfo struct { - CaData string `yaml:"client-certificate-data"` - KeyData string `yaml:"client-key-data"` -} - -type user struct { - Name string `yaml:"name"` - User userInfo `yaml:"user"` -} - -type kubeConfig struct { - ApiVersion string `yaml:"apiVersion"` - Clusters []cluster `yaml:"clusters"` - Contexts []contextObject `yaml:"contexts"` - CurrentContext string `yaml:"current-context"` - Kind string `yaml:"kind"` - Preferences map[string]string `yaml:"preferences"` - Users []user `yaml:"users"` -} - -type CertInformation struct { - Country []string - Organization []string - OrganizationalUnit []string - EmailAddress []string - Province []string - Locality []string - CommonName string - CrtName, KeyName string - IsCA bool - Names []pkix.AttributeTypeAndValue -} - -func createCRT(RootCa *x509.Certificate, RootKey *rsa.PrivateKey, info CertInformation) ([]byte, []byte, error) { - var cert, key bytes.Buffer - Crt := newCertificate(info) - Key, err := rsa.GenerateKey(rand.Reader, 2048) - if err != nil { - klog.Error(err) - return nil, nil, err - } - - var buf []byte - - buf, err = x509.CreateCertificate(rand.Reader, Crt, RootCa, &Key.PublicKey, RootKey) - - if err != nil { - klog.Error(err) - return nil, nil, err - } - pem.Encode(&cert, &pem.Block{Type: "CERTIFICATE", Bytes: buf}) - - if err != nil { - klog.Error(err) - return nil, nil, err - } - - buf = x509.MarshalPKCS1PrivateKey(Key) - pem.Encode(&key, &pem.Block{Type: "PRIVATE KEY", Bytes: buf}) - - return cert.Bytes(), key.Bytes(), nil -} - -func Parse(crtPath, keyPath string) (rootcertificate *x509.Certificate, rootPrivateKey *rsa.PrivateKey, err error) { - rootcertificate, err = parseCrt(crtPath) - if err != nil { - klog.Error(err) - return nil, nil, err - } - rootPrivateKey, err = parseKey(keyPath) - return rootcertificate, rootPrivateKey, nil -} - -func parseCrt(path string) (*x509.Certificate, error) { - buf, err := ioutil.ReadFile(path) - if err != nil { - klog.Error(err) - return nil, err - } - p := &pem.Block{} - p, buf = pem.Decode(buf) - return x509.ParseCertificate(p.Bytes) -} - -func parseKey(path string) (*rsa.PrivateKey, error) { - buf, err := ioutil.ReadFile(path) - if err != nil { - klog.Error(err) - return nil, err - } - p, buf := pem.Decode(buf) - return x509.ParsePKCS1PrivateKey(p.Bytes) -} - -func newCertificate(info CertInformation) *x509.Certificate { - rd.Seed(time.Now().UnixNano()) - return &x509.Certificate{ - SerialNumber: big.NewInt(rd.Int63()), - Subject: pkix.Name{ - Country: info.Country, - Organization: info.Organization, - OrganizationalUnit: info.OrganizationalUnit, - Province: info.Province, - CommonName: info.CommonName, - Locality: info.Locality, - ExtraNames: info.Names, - }, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(20, 0, 0), - BasicConstraintsValid: true, - IsCA: info.IsCA, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, - KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - EmailAddresses: info.EmailAddress, - } -} - -func generateCaAndKey(user, caPath, keyPath string) (string, string, error) { - crtInfo := CertInformation{CommonName: user, IsCA: false} - - crt, pri, err := Parse(caPath, keyPath) - if err != nil { - klog.Error(err) - return "", "", err - } - cert, key, err := createCRT(crt, pri, crtInfo) - if err != nil { - klog.Error(err) - return "", "", err - } - - base64Cert := base64.StdEncoding.EncodeToString(cert) - base64Key := base64.StdEncoding.EncodeToString(key) - return base64Cert, base64Key, nil -} - -func createKubeConfig(username string) (string, error) { - tmpKubeConfig := kubeConfig{ApiVersion: "v1", Kind: "Config"} - serverCa, err := ioutil.ReadFile(caPath) - if err != nil { - klog.Errorln(err) - return "", err - } - base64ServerCa := base64.StdEncoding.EncodeToString(serverCa) - tmpClusterInfo := clusterInfo{CertificateAuthorityData: base64ServerCa, Server: client.ClientSets().K8s().Master()} - tmpCluster := cluster{Cluster: tmpClusterInfo, Name: clusterName} - tmpKubeConfig.Clusters = append(tmpKubeConfig.Clusters, tmpCluster) - - contextName := username + "@" + clusterName - tmpContext := contextObject{Context: contextInfo{User: username, Cluster: clusterName, NameSpace: defaultNamespace}, Name: contextName} - tmpKubeConfig.Contexts = append(tmpKubeConfig.Contexts, tmpContext) - - cert, key, err := generateCaAndKey(username, caPath, keyPath) - - if err != nil { - return "", err - } - - tmpUser := user{User: userInfo{CaData: cert, KeyData: key}, Name: username} - tmpKubeConfig.Users = append(tmpKubeConfig.Users, tmpUser) - tmpKubeConfig.CurrentContext = contextName - - config, err := yaml.Marshal(tmpKubeConfig) - if err != nil { - return "", err - } - - return string(config), nil -} - func CreateKubeConfig(username string) error { + k8sClient := client.ClientSets().K8s().Kubernetes() - configName := fmt.Sprintf("kubeconfig-%s", username) - _, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) + + configName := fmt.Sprintf(kubeconfigNameFormat, username) + _, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metav1.GetOptions{}) if errors.IsNotFound(err) { - config, err := createKubeConfig(username) + kubeconfig, err := createKubeConfig(username) if err != nil { klog.Errorln(err) return err } - - data := map[string]string{"config": config} - configMap := v1.ConfigMap{TypeMeta: metaV1.TypeMeta{Kind: "Configmap", APIVersion: "v1"}, ObjectMeta: metaV1.ObjectMeta{Name: configName}, Data: data} - _, err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Create(&configMap) + data := map[string]string{fileName: string(kubeconfig)} + cm := &corev1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: configMapKind, APIVersion: configMapAPIVersion}, ObjectMeta: metav1.ObjectMeta{Name: configName}, Data: data} + _, err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Create(cm) if err != nil && !errors.IsAlreadyExists(err) { - klog.Errorf("create username %s's kubeConfig failed, reason: %v", username, err) + klog.Errorln(err) return err } } return nil +} +func createKubeConfig(username string) ([]byte, error) { + k8sClient := client.ClientSets().K8s().Kubernetes() + kubeconfig := client.ClientSets().K8s().Config() + + ca := kubeconfig.CAData + + csrConfig := &certutil.Config{ + CommonName: username, + Organization: nil, + AltNames: certutil.AltNames{}, + Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + } + x509csr, x509key, err := pkiutil.NewCSRAndKey(csrConfig) + if err != nil { + klog.Errorln(err) + return nil, err + } + + var csrBuffer, keyBuffer bytes.Buffer + pem.Encode(&keyBuffer, &pem.Block{Type: "PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(x509key)}) + + csrBytes, _ := x509.CreateCertificateRequest(rand.Reader, x509csr, x509key) + pem.Encode(&csrBuffer, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes}) + + csr := csrBuffer.Bytes() + key := keyBuffer.Bytes() + + csrName := fmt.Sprintf("%s-csr-%d", username, time.Now().Unix()) + + k8sCSR := &certificatesv1beta1.CertificateSigningRequest{ + TypeMeta: metav1.TypeMeta{ + Kind: "CertificateSigningRequest", + APIVersion: "certificates.k8s.io/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: csrName, + }, + Spec: certificatesv1beta1.CertificateSigningRequestSpec{ + Request: csr, + Usages: []certificatesv1beta1.KeyUsage{certificatesv1beta1.UsageServerAuth, certificatesv1beta1.UsageKeyEncipherment, certificatesv1beta1.UsageClientAuth, certificatesv1beta1.UsageDigitalSignature}, + Username: username, + Groups: []string{"system:authenticated"}, + }, + } + + // create csr + k8sCSR, err = k8sClient.CertificatesV1beta1().CertificateSigningRequests().Create(k8sCSR) + + if err != nil { + klog.Errorln(err) + return nil, err + } + + // release csr, if it fails need to delete it manually + defer func() { + err := k8sClient.CertificatesV1beta1().CertificateSigningRequests().Delete(csrName, &metav1.DeleteOptions{}) + if err != nil { + klog.Errorln(err) + } + }() + + k8sCSR.Status = certificatesv1beta1.CertificateSigningRequestStatus{ + Conditions: []certificatesv1beta1.CertificateSigningRequestCondition{{ + Type: "Approved", + Reason: "KubeSphereApprove", + Message: "This CSR was approved by KubeSphere certificate approve.", + LastUpdateTime: metav1.Time{ + Time: time.Now(), + }, + }}, + } + + // approve csr + k8sCSR, err = k8sClient.CertificatesV1beta1().CertificateSigningRequests().UpdateApproval(k8sCSR) + + if err != nil { + klog.Errorln(err) + return nil, err + } + + // get client cert + var cert []byte + maxRetries := 3 + for i := 0; i < maxRetries; i++ { + + k8sCSR, err = k8sClient.CertificatesV1beta1().CertificateSigningRequests().Get(csrName, metav1.GetOptions{}) + + if k8sCSR != nil && k8sCSR.Status.Certificate != nil { + cert = k8sCSR.Status.Certificate + break + } + + // sleep 0/200/400 millisecond + time.Sleep(200 * time.Millisecond * time.Duration(i)) + } + + if cert == nil { + return nil, fmt.Errorf("create client certificate failed: %v", err) + } + + currentContext := fmt.Sprintf("%s@%s", username, defaultClusterName) + + config := clientcmdapi.Config{ + Kind: configMapKind, + APIVersion: configMapAPIVersion, + Preferences: clientcmdapi.Preferences{}, + Clusters: map[string]*clientcmdapi.Cluster{defaultClusterName: { + Server: kubeconfig.Host, + InsecureSkipTLSVerify: false, + CertificateAuthorityData: ca, + }}, + AuthInfos: map[string]*clientcmdapi.AuthInfo{username: { + ClientCertificateData: cert, + ClientKeyData: key, + }}, + Contexts: map[string]*clientcmdapi.Context{currentContext: { + Cluster: defaultClusterName, + AuthInfo: username, + Namespace: defaultNamespace, + }}, + CurrentContext: currentContext, + } + + return clientcmd.Write(config) } func GetKubeConfig(username string) (string, error) { k8sClient := client.ClientSets().K8s().Kubernetes() - configName := fmt.Sprintf("kubeconfig-%s", username) - configMap, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) + configName := fmt.Sprintf(kubeconfigNameFormat, username) + configMap, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metav1.GetOptions{}) if err != nil { - klog.Errorf("cannot get username %s's kubeConfig, reason: %v", username, err) + klog.Errorln(err) return "", err } - str := configMap.Data[kubectlConfigKey] - var kubeConfig kubeConfig - err = yaml.Unmarshal([]byte(str), &kubeConfig) + data := []byte(configMap.Data[fileName]) + + kubeconfig, err := clientcmd.Load(data) + if err != nil { - klog.Error(err) + klog.Errorln(err) return "", err } + masterURL := client.ClientSets().K8s().Master() - for i, cluster := range kubeConfig.Clusters { - cluster.Cluster.Server = masterURL - kubeConfig.Clusters[i] = cluster + + if cluster := kubeconfig.Clusters[defaultClusterName]; cluster != nil { + cluster.Server = masterURL } - data, err := yaml.Marshal(kubeConfig) + + data, err = clientcmd.Write(*kubeconfig) + if err != nil { - klog.Error(err) + klog.Errorln(err) return "", err } + return string(data), nil } func DelKubeConfig(username string) error { k8sClient := client.ClientSets().K8s().Kubernetes() - configName := fmt.Sprintf("kubeconfig-%s", username) - _, err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName, metaV1.GetOptions{}) - if errors.IsNotFound(err) { - return nil - } + configName := fmt.Sprintf(kubeconfigNameFormat, username) - deletePolicy := metaV1.DeletePropagationBackground - err = k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(configName, &metaV1.DeleteOptions{PropagationPolicy: &deletePolicy}) + deletePolicy := metav1.DeletePropagationBackground + err := k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Delete(configName, &metav1.DeleteOptions{PropagationPolicy: &deletePolicy}) if err != nil { - klog.Errorf("delete username %s's kubeConfig failed, reason: %v", username, err) + klog.Errorln(err) return err } return nil