/* Copyright 2019 The KubeSphere Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package kubeconfig import ( "bytes" "context" "crypto/rand" "crypto/x509" "encoding/pem" "fmt" "io/ioutil" "time" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/user" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" certutil "k8s.io/client-go/util/cert" "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" iamv1alpha2 "kubesphere.io/api/iam/v1alpha2" "kubesphere.io/kubesphere/pkg/client/clientset/versioned/scheme" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/utils/pkiutil" ) const ( inClusterCAFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" configMapPrefix = "kubeconfig-" kubeconfigNameFormat = configMapPrefix + "%s" defaultClusterName = "local" defaultNamespace = "default" kubeconfigFileName = "config" configMapKind = "ConfigMap" configMapAPIVersion = "v1" privateKeyAnnotation = "kubesphere.io/private-key" residual = 72 * time.Hour ) type Interface interface { GetKubeConfig(username string) (string, error) CreateKubeConfig(user *iamv1alpha2.User) error UpdateKubeconfig(username string, csr *certificatesv1beta1.CertificateSigningRequest) error } type operator struct { k8sClient kubernetes.Interface configMapInformer corev1informers.ConfigMapInformer config *rest.Config masterURL string } func NewOperator(k8sClient kubernetes.Interface, configMapInformer corev1informers.ConfigMapInformer, config *rest.Config) Interface { return &operator{k8sClient: k8sClient, configMapInformer: configMapInformer, config: config} } func NewReadOnlyOperator(configMapInformer corev1informers.ConfigMapInformer, masterURL string) Interface { return &operator{configMapInformer: configMapInformer, masterURL: masterURL} } func (o *operator) CreateKubeConfig(user *iamv1alpha2.User) error { configName := fmt.Sprintf(kubeconfigNameFormat, user.Name) cm, err := o.configMapInformer.Lister().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName) // already exist and cert will not expire in 3 days if err == nil && !isExpirated(cm, user.Name) { return nil } // internal error if err != nil && !errors.IsNotFound(err) { klog.Error(err) return err } // create a new CSR var ca []byte if len(o.config.CAData) > 0 { ca = o.config.CAData } else { ca, err = ioutil.ReadFile(inClusterCAFilePath) if err != nil { klog.Errorln(err) return err } } if err = o.createCSR(user.Name); err != nil { klog.Errorln(err) return err } currentContext := fmt.Sprintf("%s@%s", user.Name, defaultClusterName) config := clientcmdapi.Config{ Kind: configMapKind, APIVersion: configMapAPIVersion, Preferences: clientcmdapi.Preferences{}, Clusters: map[string]*clientcmdapi.Cluster{defaultClusterName: { Server: o.config.Host, InsecureSkipTLSVerify: false, CertificateAuthorityData: ca, }}, Contexts: map[string]*clientcmdapi.Context{currentContext: { Cluster: defaultClusterName, AuthInfo: user.Name, Namespace: defaultNamespace, }}, CurrentContext: currentContext, } kubeconfig, err := clientcmd.Write(config) if err != nil { klog.Error(err) return err } // update configmap if it already exist. if cm != nil { cm.Data = map[string]string{kubeconfigFileName: string(kubeconfig)} if _, err = o.k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Update(context.Background(), cm, metav1.UpdateOptions{}); err != nil { klog.Errorln(err) return err } return nil } // create a new config cm = &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ Kind: configMapKind, APIVersion: configMapAPIVersion, }, ObjectMeta: metav1.ObjectMeta{ Name: configName, Labels: map[string]string{constants.UsernameLabelKey: user.Name}, }, Data: map[string]string{kubeconfigFileName: string(kubeconfig)}, } if err = controllerutil.SetControllerReference(user, cm, scheme.Scheme); err != nil { klog.Errorln(err) return err } if _, err = o.k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Create(context.Background(), cm, metav1.CreateOptions{}); err != nil { klog.Errorln(err) return err } return nil } func (o *operator) GetKubeConfig(username string) (string, error) { configName := fmt.Sprintf(kubeconfigNameFormat, username) configMap, err := o.configMapInformer.Lister().ConfigMaps(constants.KubeSphereControlNamespace).Get(configName) if err != nil { klog.Errorln(err) return "", err } data := []byte(configMap.Data[kubeconfigFileName]) kubeconfig, err := clientcmd.Load(data) if err != nil { klog.Errorln(err) return "", err } masterURL := o.masterURL // server host override if cluster := kubeconfig.Clusters[defaultClusterName]; cluster != nil && masterURL != "" { cluster.Server = masterURL } data, err = clientcmd.Write(*kubeconfig) if err != nil { klog.Errorln(err) return "", err } return string(data), nil } func (o *operator) createCSR(username string) error { csrConfig := &certutil.Config{ CommonName: username, Organization: nil, AltNames: certutil.AltNames{}, Usages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, } x509csr, x509key, err := pkiutil.NewCSRAndKey(csrConfig) if err != nil { klog.Errorln(err) return err } var csrBuffer, keyBuffer bytes.Buffer if err = pem.Encode(&keyBuffer, &pem.Block{Type: "PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(x509key)}); err != nil { klog.Errorln(err) return err } var csrBytes []byte if csrBytes, err = x509.CreateCertificateRequest(rand.Reader, x509csr, x509key); err != nil { klog.Errorln(err) return err } if err = pem.Encode(&csrBuffer, &pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes}); err != nil { klog.Errorln(err) return err } csr := csrBuffer.Bytes() key := keyBuffer.Bytes() csrName := fmt.Sprintf("%s-csr-%d", username, time.Now().Unix()) k8sCSR := &certificatesv1beta1.CertificateSigningRequest{ TypeMeta: metav1.TypeMeta{ Kind: "CertificateSigningRequest", APIVersion: "certificates.k8s.io/v1beta1", }, ObjectMeta: metav1.ObjectMeta{ Name: csrName, Labels: map[string]string{constants.UsernameLabelKey: username}, Annotations: map[string]string{privateKeyAnnotation: string(key)}, }, Spec: certificatesv1beta1.CertificateSigningRequestSpec{ Request: csr, Usages: []certificatesv1beta1.KeyUsage{certificatesv1beta1.UsageKeyEncipherment, certificatesv1beta1.UsageClientAuth, certificatesv1beta1.UsageDigitalSignature}, Username: username, Groups: []string{user.AllAuthenticated}, }, } // create csr if _, err = o.k8sClient.CertificatesV1beta1().CertificateSigningRequests().Create(context.Background(), k8sCSR, metav1.CreateOptions{}); err != nil { klog.Errorln(err) return err } return nil } // Update client key and client certificate after CertificateSigningRequest has been approved func (o *operator) UpdateKubeconfig(username string, csr *certificatesv1beta1.CertificateSigningRequest) error { configName := fmt.Sprintf(kubeconfigNameFormat, username) configMap, err := o.k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Get(context.Background(), configName, metav1.GetOptions{}) if err != nil { klog.Errorln(err) return err } configMap = applyCert(configMap, csr) _, err = o.k8sClient.CoreV1().ConfigMaps(constants.KubeSphereControlNamespace).Update(context.Background(), configMap, metav1.UpdateOptions{}) if err != nil { klog.Errorln(err) return err } return nil } func applyCert(cm *corev1.ConfigMap, csr *certificatesv1beta1.CertificateSigningRequest) *corev1.ConfigMap { data := []byte(cm.Data[kubeconfigFileName]) kubeconfig, err := clientcmd.Load(data) if err != nil { klog.Error(err) return cm } username := getControlledUsername(cm) privateKey := csr.Annotations[privateKeyAnnotation] clientCert := csr.Status.Certificate kubeconfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{ username: { ClientKeyData: []byte(privateKey), ClientCertificateData: clientCert, }, } data, err = clientcmd.Write(*kubeconfig) if err != nil { klog.Error(err) return cm } cm.Data[kubeconfigFileName] = string(data) return cm } func getControlledUsername(cm *corev1.ConfigMap) string { for _, ownerReference := range cm.OwnerReferences { if ownerReference.Kind == iamv1alpha2.ResourceKindUser { return ownerReference.Name } } return "" } func isExpirated(cm *corev1.ConfigMap, username string) bool { data := []byte(cm.Data[kubeconfigFileName]) kubeconfig, err := clientcmd.Load(data) if err != nil { klog.Errorln(err) return true } authInfo, ok := kubeconfig.AuthInfos[username] if ok { clientCert, err := certutil.ParseCertsPEM(authInfo.ClientCertificateData) if err != nil { klog.Errorln(err) return true } for _, cert := range clientCert { if cert.NotAfter.Before(time.Now().Add(residual)) { return true } } return false } //ignore the kubeconfig, since it's not approved yet. return false }