admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add real time cpu/memory monitor api for container level

Browse Source
This commit is contained in:
Calvin Yu
2018-05-28 10:42:27 +08:00
parent 3067c47681
commit f5cb88afc2
6 changed files with 306 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
pkg/apis/v1alpha/containers/containers_handler.go Normal file
View File

@@ -0,0 +1,60 @@
/*
Copyright 2018 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package containers
import (
"github.com/emicklei/go-restful"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/filter/route"
"kubesphere.io/kubesphere/pkg/models"
)
func Register(ws *restful.WebService) {
ws.Route(ws.GET("/namespaces/{namespace}/pods/{podname}/containers").To(handleContainersUnderNameSpaceAndPod).Filter(route.RouteLogging)).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/nodes/{nodename}/namespaces/{namespace}/pods/{podname}/containers").To(handleContainersUnderNodeAndNameSpaceAndPod).Filter(route.RouteLogging)).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
}
func handleContainersUnderNameSpaceAndPod(request *restful.Request, response *restful.Response) {
var result constants.ResultMessage
var resultNameSpaces []models.ResultNameSpaceForContainer
var resultNameSpace models.ResultNameSpaceForContainer
resultNameSpace = models.FormatContainersMetrics("", request.PathParameter("namespace"), request.PathParameter("podname"))
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
result.Data = resultNameSpaces
response.WriteAsJson(result)
}
func handleContainersUnderNodeAndNameSpaceAndPod(request *restful.Request, response *restful.Response) {
var result constants.ResultMessage
var resultNameSpaces []models.ResultNameSpaceForContainer
var resultNameSpace models.ResultNameSpaceForContainer
resultNameSpace = models.FormatContainersMetrics(request.PathParameter("nodename"), request.PathParameter("namespace"), request.PathParameter("podname"))
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
result.Data = resultNameSpaces
response.WriteAsJson(result)
}

3
pkg/apis/v1alpha/install.go
View File

@@ -18,6 +18,7 @@ package v1alpha
import (
"github.com/emicklei/go-restful"
"kubesphere.io/kubesphere/pkg/apis/v1alpha/containers"
"kubesphere.io/kubesphere/pkg/apis/v1alpha/kubeconfig"
"kubesphere.io/kubesphere/pkg/apis/v1alpha/kubectl"
"kubesphere.io/kubesphere/pkg/apis/v1alpha/nodes"
@@ -32,7 +33,6 @@ func init() {
ws := new(restful.WebService)
ws.Path("/api/v1alpha1")
nodes.Register(ws, "/nodes")
kubeconfig.Register(ws, "/namespaces/{namespace}/kubeconfig")
kubectl.Register(ws, "/namespaces/{namespace}/kubectl")
registries.Register(ws, "/registries")
@@ -40,6 +40,7 @@ func init() {
volumes.Register(ws, "/volumes")
nodes.Register(ws, "/nodes")
pods.Register(ws)
containers.Register(ws)
// add webservice to default container
restful.Add(ws)

21
pkg/apis/v1alpha/pods/pods_handler.go
View File

@@ -23,7 +23,6 @@ import (
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/models"
)
func Register(ws *restful.WebService) {
@@ -35,6 +34,9 @@ func Register(ws *restful.WebService) {
ws.Route(ws.GET("/namespaces/{namespace}/pods").To(handlePodsUnderNameSpace).Filter(route.RouteLogging)).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
ws.Route(ws.GET("/nodes/{nodename}/namespaces/{namespace}/pods").To(handlePodsUnderNodeAndNameSpace).Filter(route.RouteLogging)).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON)
}
func handleAllPods(request *restful.Request, response *restful.Response) {
@@ -46,7 +48,7 @@ func handleAllPods(request *restful.Request, response *restful.Response) {
for _, namespace := range namespaces {
resultNameSpace = models.FormatNameSpaceMetrics(namespace)
resultNameSpace = models.FormatPodsMetrics("", namespace)
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
}
@@ -60,7 +62,20 @@ func handlePodsUnderNameSpace(request *restful.Request, response *restful.Respon
var resultNameSpaces []models.ResultNameSpace
var resultNameSpace models.ResultNameSpace
resultNameSpace = models.FormatNameSpaceMetrics(request.PathParameter("namespace"))
resultNameSpace = models.FormatPodsMetrics("", request.PathParameter("namespace"))
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
result.Data = resultNameSpaces
response.WriteAsJson(result)
}
func handlePodsUnderNodeAndNameSpace(request *restful.Request, response *restful.Response) {
var result constants.ResultMessage
var resultNameSpaces []models.ResultNameSpace
var resultNameSpace models.ResultNameSpace
resultNameSpace = models.FormatPodsMetrics(request.PathParameter("nodename"), request.PathParameter("namespace"))
resultNameSpaces = append(resultNameSpaces, resultNameSpace)

2
pkg/client/heapsterclient.go
View File

@@ -27,7 +27,7 @@ import (
const (
DefaultHeapsterScheme = "http"
DefaultHeapsterService = "heapster" //"heapster"
DefaultHeapsterPort = "80" // use the first exposed port on the service
DefaultHeapsterPort = "80" // use the first exposed port on the service
)
var (

185
pkg/models/containers.go Normal file
View File

@@ -0,0 +1,185 @@
package models
import (
"encoding/json"
"strings"
"github.com/golang/glog"
"kubesphere.io/kubesphere/pkg/client"
ksutil "kubesphere.io/kubesphere/pkg/util"
"fmt"
"strconv"
)
type ResultNameSpaceForContainer struct {
NameSpace string `json:"namespace"`
PodsCount string `json:"pods_count"`
Pods []ResultPodForContainer `json:"pods"`
}
type ResultPodForContainer struct {
PodName string `json:"pod_name"`
ContainersCount string `json:"containers_count"`
Containers []ResultContainer `json:"containers"`
}
type ResultContainer struct {
ContainerName string `json:"container_name"`
CPURequest string `json:"cpu_request"`
CPULimit string `json:"cpu_limit"`
MemoryRequest string `json:"mem_request"`
MemoryLimit string `json:"mem_limit"`
CPU []CPUContainer `json:"cpu"`
Memory []MemoryContainer `json:"memory"`
}
type CPUContainer struct {
TimeStamp string `json:"timestamp"`
UsedCPU string `json:"used_cpu"`
CPUUtilization string `json:"cpu_utilization"`
}
type MemoryContainer struct {
TimeStamp string `json:"timestamp"`
UsedMemory string `json:"used_mem"`
MemoryUtilization string `json:"mem_utilization"`
}
/*
Get all containers under specified namespace in default cluster
*/
func GetContainers(namespace, podName string) []string {
containersList := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers")
var containers []string
dec := json.NewDecoder(strings.NewReader(containersList))
err := dec.Decode(&containers)
if err != nil {
glog.Error(err)
}
return containers
}
func FormatContainersMetrics(nodeName, namespace, podName string) ResultNameSpaceForContainer {
var resultNameSpaceForContainer ResultNameSpaceForContainer
var resultPodsForContainer []ResultPodForContainer
var resultPodForContainer ResultPodForContainer
var pods []string
if nodeName == "" {
pods = GetPods(namespace)
} else {
pods = GetPodsForNode(nodeName, namespace)
}
resultNameSpaceForContainer.NameSpace = namespace
resultNameSpaceForContainer.PodsCount = strconv.Itoa(len(pods))
if podName != "" {
resultPodForContainer.PodName = podName
resultPodForContainer = FormatPodMetricsWithContainers(namespace, podName)
resultPodsForContainer = append(resultPodsForContainer, resultPodForContainer)
resultNameSpaceForContainer.Pods = resultPodsForContainer
return resultNameSpaceForContainer
}
for _, pod := range pods {
resultPodForContainer.PodName = pod
resultPodForContainer = FormatPodMetricsWithContainers(namespace, pod)
resultPodsForContainer = append(resultPodsForContainer, resultPodForContainer)
}
resultNameSpaceForContainer.Pods = resultPodsForContainer
return resultNameSpaceForContainer
}
func FormatPodMetricsWithContainers(namespace, pod string) ResultPodForContainer {
var resultPod ResultPodForContainer
var containers []string
var resultContainers []ResultContainer
var resultContainer ResultContainer
var containerCPUMetrics []CPUContainer
var containerMemMetrics []MemoryContainer
var cpuMetrics CPUContainer
var memMetrics MemoryContainer
resultPod.PodName = pod
containers = GetContainers(namespace, pod)
resultPod.ContainersCount = strconv.Itoa(len(containers))
for _, container := range containers {
resultContainer.ContainerName = container
cpuRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/request")
cpuRequest = ksutil.JsonRawMessage(cpuRequest).Find("metrics").ToList()[0].Find("value").ToString()
if cpuRequest != "" && cpuRequest != "0" {
resultContainer.CPURequest = cpuRequest
} else {
resultContainer.CPURequest = "inf"
}
cpuLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/limit")
cpuLimit = ksutil.JsonRawMessage(cpuLimit).Find("metrics").ToList()[0].Find("value").ToString()
if cpuLimit != "" && cpuLimit != "0" {
resultContainer.CPULimit = cpuLimit
} else {
resultContainer.CPULimit = "inf"
}
memoryRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/request")
resultContainer.MemoryRequest = ConvertMemory(memoryRequest)
memoryLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/limit")
resultContainer.MemoryLimit = ConvertMemory(memoryLimit)
cpuUsageRate := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/usage_rate")
if cpuUsageRate != "" {
metrics := ksutil.JsonRawMessage(cpuUsageRate).Find("metrics").ToList()
for _, metric := range metrics {
timestamp := metric.Find("timestamp")
cpu_utilization, _ := strconv.ParseFloat(metric.Find("value").ToString(), 64)
cpuMetrics.TimeStamp = timestamp.ToString()
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization/1000)
if resultContainer.CPULimit != "inf" {
cpu_limit, _ := strconv.ParseFloat(resultContainer.CPULimit, 64)
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", cpu_limit*cpu_utilization/1000)
} else {
cpuMetrics.UsedCPU = "inf"
}
glog.Info("pod " + pod + " has limit cpu " + resultContainer.CPULimit + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization/1000) + " at time" + timestamp.ToString())
containerCPUMetrics = append(containerCPUMetrics, cpuMetrics)
}
}
resultContainer.CPU = containerCPUMetrics
var used_mem_bytes float64
memUsage := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/usage")
if memUsage != "" {
metrics := ksutil.JsonRawMessage(memUsage).Find("metrics").ToList()
for _, metric := range metrics {
timestamp := metric.Find("timestamp")
used_mem_bytes, _ = strconv.ParseFloat(metric.Find("value").ToString(), 64)
used_mem := used_mem_bytes / 1024 / 1024
memMetrics.TimeStamp = timestamp.ToString()
memMetrics.UsedMemory = fmt.Sprintf("%.1f", used_mem)
if resultContainer.MemoryLimit != "inf" {
mem_limit, _ := strconv.ParseFloat(resultContainer.MemoryLimit, 64)
memMetrics.MemoryUtilization = fmt.Sprintf("%.3f", used_mem/mem_limit)
} else {
memMetrics.MemoryUtilization = "inf"
}
glog.Info("pod " + pod + " has limit mem " + resultContainer.MemoryLimit + " mem utilization " + memMetrics.MemoryUtilization + " at time" + timestamp.ToString())
containerMemMetrics = append(containerMemMetrics, memMetrics)
}
}
resultContainer.Memory = containerMemMetrics
resultContainers = append(resultContainers, resultContainer)
}
resultPod.Containers = resultContainers
return resultPod
}

47
pkg/models/pods.go
View File

@@ -12,6 +12,7 @@ import (
ksutil "kubesphere.io/kubesphere/pkg/util"
"fmt"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"strconv"
)
@@ -62,7 +63,6 @@ func GetNameSpaces() []string {
Get all pods under specified namespace in default cluster
*/
func GetPods(namespace string) []string {
fmt.Println(namespace)
podsList := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods")
var pods []string
dec := json.NewDecoder(strings.NewReader(podsList))
@@ -73,12 +73,30 @@ func GetPods(namespace string) []string {
return pods
}
func FormatNameSpaceMetrics(namespace string) ResultNameSpace {
func GetPodsForNode(nodeName, namespace string) []string {
var pods []string
cli := client.NewK8sClient()
podList, err := cli.CoreV1().Pods(namespace).List(v1.ListOptions{FieldSelector: "spec.nodeName=" + nodeName})
if err != nil {
glog.Error(err)
} else {
for _, pod := range podList.Items {
pods = append(pods, pod.Name)
}
}
return pods
}
func FormatPodsMetrics(nodeName, namespace string) ResultNameSpace {
var resultNameSpace ResultNameSpace
var resultPods []ResultPod
var resultPod ResultPod
pods := GetPods(namespace)
var pods []string
if nodeName == "" {
pods = GetPods(namespace)
} else {
pods = GetPodsForNode(nodeName, namespace)
}
resultNameSpace.NameSpace = namespace
resultNameSpace.PodsCount = strconv.Itoa(len(pods))
@@ -92,6 +110,7 @@ func FormatNameSpaceMetrics(namespace string) ResultNameSpace {
}
func FormatPodMetrics(namespace, pod string) ResultPod {
var resultPod ResultPod
var podCPUMetrics []CPUPod
var podMemMetrics []MemoryPod
@@ -115,10 +134,10 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
resultPod.CPULimit = "inf"
}
memoryRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/request")
resultPod.MemoryRequest = convertMemory(memoryRequest)
resultPod.MemoryRequest = ConvertMemory(memoryRequest)
memoryLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/limit")
resultPod.MemoryLimit = convertMemory(memoryLimit)
resultPod.MemoryLimit = ConvertMemory(memoryLimit)
cpuUsageRate := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/cpu/usage_rate")
if cpuUsageRate != "" {
@@ -171,7 +190,7 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
return resultPod
}
func convertMemory(memBytes string) string {
func ConvertMemory(memBytes string) string {
var mem string
if memBytes != "" {
@@ -194,3 +213,17 @@ func convertMemory(memBytes string) string {
}
return mem
}
func getNodeNameForPod(podName, namespace string) string {
var nodeName string
cli := client.NewK8sClient()
pod, err := cli.CoreV1().Pods(namespace).Get(podName, v1.GetOptions{})
if err != nil {
glog.Error(err)
} else {
nodeName = pod.Spec.NodeName
}
return nodeName
}