refactor monitor apis to adopt new response format
This commit is contained in:
@@ -25,6 +25,9 @@ import (
|
||||
)
|
||||
|
||||
func Register(ws *restful.WebService) {
|
||||
ws.Route(ws.GET("/namespaces/{namespace}/pods/{podname}/containers/{containername}").To(handleContainerUnderNameSpaceAndPod).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
ws.Route(ws.GET("/namespaces/{namespace}/pods/{podname}/containers").To(handleContainersUnderNameSpaceAndPod).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
@@ -33,28 +36,23 @@ func Register(ws *restful.WebService) {
|
||||
Produces(restful.MIME_JSON)
|
||||
}
|
||||
|
||||
func handleContainerUnderNameSpaceAndPod(request *restful.Request, response *restful.Response) {
|
||||
var resultContainer models.ResultContainer
|
||||
resultContainer = models.FormatContainerMetrics(request.PathParameter("namespace"), request.PathParameter("podname"), request.PathParameter("containername"))
|
||||
resultContainer.NodeName = models.GetNodeNameForPod(request.PathParameter("podname"), request.PathParameter("namespace"))
|
||||
response.WriteAsJson(resultContainer)
|
||||
}
|
||||
func handleContainersUnderNameSpaceAndPod(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNameSpaces []models.ResultNameSpaceForContainer
|
||||
var resultNameSpace models.ResultNameSpaceForContainer
|
||||
var resultNameSpace constants.PageableResponse
|
||||
|
||||
resultNameSpace = models.FormatContainersMetrics("", request.PathParameter("namespace"), request.PathParameter("podname"))
|
||||
|
||||
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
|
||||
|
||||
result.Data = resultNameSpaces
|
||||
response.WriteAsJson(result)
|
||||
response.WriteAsJson(resultNameSpace)
|
||||
}
|
||||
|
||||
func handleContainersUnderNodeAndNameSpaceAndPod(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNameSpaces []models.ResultNameSpaceForContainer
|
||||
var resultNameSpace models.ResultNameSpaceForContainer
|
||||
var resultNameSpace constants.PageableResponse
|
||||
|
||||
resultNameSpace = models.FormatContainersMetrics(request.PathParameter("nodename"), request.PathParameter("namespace"), request.PathParameter("podname"))
|
||||
|
||||
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
|
||||
|
||||
result.Data = resultNameSpaces
|
||||
response.WriteAsJson(result)
|
||||
response.WriteAsJson(resultNameSpace)
|
||||
}
|
||||
|
||||
@@ -35,32 +35,28 @@ func Register(ws *restful.WebService, subPath string) {
|
||||
}
|
||||
|
||||
func handleNodes(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNodes []models.ResultNode
|
||||
var result constants.PageableResponse
|
||||
var resultNode models.ResultNode
|
||||
|
||||
nodes := models.GetNodes()
|
||||
|
||||
for _, node := range nodes {
|
||||
|
||||
var total_count int
|
||||
for i, node := range nodes {
|
||||
resultNode = models.FormatNodeMetrics(node)
|
||||
resultNodes = append(resultNodes, resultNode)
|
||||
|
||||
result.Items = append(result.Items, resultNode)
|
||||
total_count = i
|
||||
}
|
||||
total_count = total_count + 1
|
||||
|
||||
result.Data = resultNodes
|
||||
result.TotalCount = total_count
|
||||
response.WriteAsJson(result)
|
||||
}
|
||||
|
||||
func handleSingleNode(request *restful.Request, response *restful.Response) {
|
||||
nodeName := request.PathParameter("nodename")
|
||||
var result constants.ResultMessage
|
||||
var resultNodes []models.ResultNode
|
||||
var resultNode models.ResultNode
|
||||
|
||||
resultNode = models.FormatNodeMetrics(nodeName)
|
||||
resultNodes = append(resultNodes, resultNode)
|
||||
|
||||
result.Data = resultNodes
|
||||
response.WriteAsJson(result)
|
||||
response.WriteAsJson(resultNode)
|
||||
}
|
||||
|
||||
@@ -30,55 +30,75 @@ func Register(ws *restful.WebService) {
|
||||
ws.Route(ws.GET("/pods").To(handleAllPods).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
|
||||
ws.Route(ws.GET("/namespaces/{namespace}/pods/{podname}").To(handlePodUnderNameSpace).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
ws.Route(ws.GET("/namespaces/{namespace}/pods").To(handlePodsUnderNameSpace).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
ws.Route(ws.GET("/nodes/{nodename}/pods").To(handlePodsUnderNode).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
ws.Route(ws.GET("/nodes/{nodename}/namespaces/{namespace}/pods").To(handlePodsUnderNodeAndNameSpace).Filter(route.RouteLogging)).
|
||||
Consumes(restful.MIME_JSON, restful.MIME_XML).
|
||||
Produces(restful.MIME_JSON)
|
||||
}
|
||||
|
||||
func handleAllPods(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNameSpaces []models.ResultNameSpace
|
||||
var resultNameSpace models.ResultNameSpace
|
||||
var result constants.PageableResponse
|
||||
|
||||
namespaces := models.GetNameSpaces()
|
||||
|
||||
for _, namespace := range namespaces {
|
||||
|
||||
resultNameSpace = models.FormatPodsMetrics("", namespace)
|
||||
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
|
||||
|
||||
var total_count int
|
||||
for i, namespace := range namespaces {
|
||||
result = models.FormatPodsMetrics("", namespace)
|
||||
result.Items = append(result.Items, result)
|
||||
total_count = i
|
||||
}
|
||||
|
||||
result.Data = resultNameSpaces
|
||||
result.TotalCount = total_count
|
||||
|
||||
response.WriteAsJson(result)
|
||||
}
|
||||
|
||||
func handlePodsUnderNameSpace(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNameSpaces []models.ResultNameSpace
|
||||
var resultNameSpace models.ResultNameSpace
|
||||
var result constants.PageableResponse
|
||||
|
||||
resultNameSpace = models.FormatPodsMetrics("", request.PathParameter("namespace"))
|
||||
result = models.FormatPodsMetrics("", request.PathParameter("namespace"))
|
||||
|
||||
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
|
||||
|
||||
result.Data = resultNameSpaces
|
||||
response.WriteAsJson(result)
|
||||
}
|
||||
func handlePodsUnderNode(request *restful.Request, response *restful.Response) {
|
||||
var result constants.PageableResponse
|
||||
var resultNameSpace constants.PageableResponse
|
||||
namespaces := models.GetNameSpaces()
|
||||
|
||||
var total_count int
|
||||
for _, namespace := range namespaces {
|
||||
resultNameSpace = models.FormatPodsMetrics(request.PathParameter("nodename"), namespace)
|
||||
|
||||
var sub_total_count int
|
||||
for j, pod := range resultNameSpace.Items {
|
||||
result.Items = append(result.Items, pod)
|
||||
sub_total_count = j
|
||||
}
|
||||
total_count += sub_total_count
|
||||
}
|
||||
result.TotalCount = total_count
|
||||
response.WriteAsJson(result)
|
||||
}
|
||||
func handlePodUnderNameSpace(request *restful.Request, response *restful.Response) {
|
||||
var resultPod models.ResultPod
|
||||
|
||||
resultPod = models.FormatPodMetrics(request.PathParameter("namespace"), request.PathParameter("podname"))
|
||||
|
||||
response.WriteAsJson(resultPod)
|
||||
}
|
||||
|
||||
func handlePodsUnderNodeAndNameSpace(request *restful.Request, response *restful.Response) {
|
||||
var result constants.ResultMessage
|
||||
var resultNameSpaces []models.ResultNameSpace
|
||||
var resultNameSpace models.ResultNameSpace
|
||||
var result constants.PageableResponse
|
||||
|
||||
resultNameSpace = models.FormatPodsMetrics(request.PathParameter("nodename"), request.PathParameter("namespace"))
|
||||
result = models.FormatPodsMetrics(request.PathParameter("nodename"), request.PathParameter("namespace"))
|
||||
|
||||
resultNameSpaces = append(resultNameSpaces, resultNameSpace)
|
||||
|
||||
result.Data = resultNameSpaces
|
||||
response.WriteAsJson(result)
|
||||
}
|
||||
|
||||
@@ -55,3 +55,23 @@ func GetHeapsterMetrics(url string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
func GetCAdvisorMetrics(nodeAddr string) string {
|
||||
|
||||
response, err := http.Get("http://" + nodeAddr + ":10255/stats/summary")
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
os.Exit(1)
|
||||
} else {
|
||||
defer response.Body.Close()
|
||||
|
||||
contents, err := ioutil.ReadAll(response.Body)
|
||||
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
return string(contents)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"github.com/golang/glog"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
"kubesphere.io/kubesphere/pkg/constants"
|
||||
|
||||
ksutil "kubesphere.io/kubesphere/pkg/util"
|
||||
|
||||
@@ -15,18 +16,8 @@ import (
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type ResultNameSpaceForContainer struct {
|
||||
NameSpace string `json:"namespace"`
|
||||
PodsCount string `json:"pods_count"`
|
||||
Pods []ResultPodForContainer `json:"pods"`
|
||||
}
|
||||
|
||||
type ResultPodForContainer struct {
|
||||
PodName string `json:"pod_name"`
|
||||
ContainersCount string `json:"containers_count"`
|
||||
Containers []ResultContainer `json:"containers"`
|
||||
}
|
||||
type ResultContainer struct {
|
||||
NodeName string `json:"node_name"`
|
||||
ContainerName string `json:"container_name"`
|
||||
CPURequest string `json:"cpu_request"`
|
||||
CPULimit string `json:"cpu_limit"`
|
||||
@@ -61,128 +52,100 @@ func GetContainers(namespace, podName string) []string {
|
||||
return containers
|
||||
}
|
||||
|
||||
func FormatContainersMetrics(nodeName, namespace, podName string) ResultNameSpaceForContainer {
|
||||
var resultNameSpaceForContainer ResultNameSpaceForContainer
|
||||
var resultPodsForContainer []ResultPodForContainer
|
||||
func FormatContainersMetrics(nodeName, namespace, podName string) constants.PageableResponse {
|
||||
|
||||
var pods []string
|
||||
if nodeName == "" {
|
||||
pods = GetPods(namespace)
|
||||
} else {
|
||||
pods = GetPodsForNode(nodeName, namespace)
|
||||
}
|
||||
|
||||
resultNameSpaceForContainer.NameSpace = namespace
|
||||
resultNameSpaceForContainer.PodsCount = strconv.Itoa(len(pods))
|
||||
|
||||
if podName != "" {
|
||||
var resultPodForContainer ResultPodForContainer
|
||||
resultPodForContainer.PodName = podName
|
||||
resultPodForContainer = FormatPodMetricsWithContainers(namespace, podName)
|
||||
resultPodsForContainer = append(resultPodsForContainer, resultPodForContainer)
|
||||
resultNameSpaceForContainer.Pods = resultPodsForContainer
|
||||
return resultNameSpaceForContainer
|
||||
}
|
||||
for _, pod := range pods {
|
||||
var resultPodForContainer ResultPodForContainer
|
||||
resultPodForContainer.PodName = pod
|
||||
resultPodForContainer = FormatPodMetricsWithContainers(namespace, pod)
|
||||
resultPodsForContainer = append(resultPodsForContainer, resultPodForContainer)
|
||||
}
|
||||
resultNameSpaceForContainer.Pods = resultPodsForContainer
|
||||
return resultNameSpaceForContainer
|
||||
}
|
||||
|
||||
func FormatPodMetricsWithContainers(namespace, pod string) ResultPodForContainer {
|
||||
|
||||
var resultPod ResultPodForContainer
|
||||
var result constants.PageableResponse
|
||||
var resultContainer ResultContainer
|
||||
var containers []string
|
||||
var resultContainers []ResultContainer
|
||||
var total_count int
|
||||
containers = GetContainers(namespace, podName)
|
||||
|
||||
resultPod.PodName = pod
|
||||
containers = GetContainers(namespace, pod)
|
||||
resultPod.ContainersCount = strconv.Itoa(len(containers))
|
||||
|
||||
for _, container := range containers {
|
||||
var resultContainer ResultContainer
|
||||
var containerCPUMetrics []CPUContainer
|
||||
var containerMemMetrics []MemoryContainer
|
||||
var cpuMetrics CPUContainer
|
||||
var memMetrics MemoryContainer
|
||||
|
||||
resultContainer.ContainerName = container
|
||||
cpuRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/request")
|
||||
cpuRequest = ksutil.JsonRawMessage(cpuRequest).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
if cpuRequest != "" && cpuRequest != "0" {
|
||||
resultContainer.CPURequest = cpuRequest
|
||||
for i, container := range containers {
|
||||
resultContainer = FormatContainerMetrics(namespace, podName, container)
|
||||
if nodeName != "" {
|
||||
resultContainer.NodeName = nodeName
|
||||
} else {
|
||||
resultContainer.CPURequest = "inf"
|
||||
resultContainer.NodeName = GetNodeNameForPod(podName, namespace)
|
||||
}
|
||||
|
||||
cpuLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/limit")
|
||||
cpuLimit = ksutil.JsonRawMessage(cpuLimit).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
if cpuLimit != "" && cpuLimit != "0" {
|
||||
resultContainer.CPULimit = cpuLimit
|
||||
} else {
|
||||
resultContainer.CPULimit = "inf"
|
||||
}
|
||||
memoryRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/request")
|
||||
resultContainer.MemoryRequest = ConvertMemory(memoryRequest)
|
||||
|
||||
memoryLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/limit")
|
||||
resultContainer.MemoryLimit = ConvertMemory(memoryLimit)
|
||||
|
||||
cpuUsageRate := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/cpu/usage_rate")
|
||||
if cpuUsageRate != "" {
|
||||
metrics := ksutil.JsonRawMessage(cpuUsageRate).Find("metrics").ToList()
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
cpu_utilization, _ := strconv.ParseFloat(metric.Find("value").ToString(), 64)
|
||||
cpuMetrics.TimeStamp = timestamp.ToString()
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization/1000)
|
||||
if resultContainer.CPULimit != "inf" {
|
||||
cpu_limit, _ := strconv.ParseFloat(resultContainer.CPULimit, 64)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", cpu_limit*cpu_utilization/1000)
|
||||
} else {
|
||||
cpuMetrics.UsedCPU = "inf"
|
||||
}
|
||||
glog.Info("pod " + pod + " has limit cpu " + resultContainer.CPULimit + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization/1000) + " at time" + timestamp.ToString())
|
||||
containerCPUMetrics = append(containerCPUMetrics, cpuMetrics)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
resultContainer.CPU = containerCPUMetrics
|
||||
|
||||
var used_mem_bytes float64
|
||||
|
||||
memUsage := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/containers/" + container + "/metrics/memory/usage")
|
||||
if memUsage != "" {
|
||||
metrics := ksutil.JsonRawMessage(memUsage).Find("metrics").ToList()
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
used_mem_bytes, _ = strconv.ParseFloat(metric.Find("value").ToString(), 64)
|
||||
used_mem := used_mem_bytes / 1024 / 1024
|
||||
memMetrics.TimeStamp = timestamp.ToString()
|
||||
memMetrics.UsedMemory = fmt.Sprintf("%.1f", used_mem)
|
||||
if resultContainer.MemoryLimit != "inf" {
|
||||
mem_limit, _ := strconv.ParseFloat(resultContainer.MemoryLimit, 64)
|
||||
memMetrics.MemoryUtilization = fmt.Sprintf("%.3f", used_mem/mem_limit)
|
||||
} else {
|
||||
memMetrics.MemoryUtilization = "inf"
|
||||
}
|
||||
|
||||
glog.Info("pod " + pod + " has limit mem " + resultContainer.MemoryLimit + " mem utilization " + memMetrics.MemoryUtilization + " at time" + timestamp.ToString())
|
||||
containerMemMetrics = append(containerMemMetrics, memMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
resultContainer.Memory = containerMemMetrics
|
||||
resultContainers = append(resultContainers, resultContainer)
|
||||
result.Items = append(result.Items, resultContainer)
|
||||
total_count = i
|
||||
}
|
||||
resultPod.Containers = resultContainers
|
||||
result.TotalCount = total_count + 1
|
||||
|
||||
return resultPod
|
||||
return result
|
||||
}
|
||||
|
||||
func FormatContainerMetrics(namespace, podName, containerName string) ResultContainer {
|
||||
var resultContainer ResultContainer
|
||||
var containerCPUMetrics []CPUContainer
|
||||
var containerMemMetrics []MemoryContainer
|
||||
var cpuMetrics CPUContainer
|
||||
var memMetrics MemoryContainer
|
||||
|
||||
resultContainer.ContainerName = containerName
|
||||
cpuRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/cpu/request")
|
||||
cpuRequest = ksutil.JsonRawMessage(cpuRequest).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
if cpuRequest != "" && cpuRequest != "0" {
|
||||
resultContainer.CPURequest = cpuRequest
|
||||
} else {
|
||||
resultContainer.CPURequest = "inf"
|
||||
}
|
||||
|
||||
cpuLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/cpu/limit")
|
||||
cpuLimit = ksutil.JsonRawMessage(cpuLimit).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
if cpuLimit != "" && cpuLimit != "0" {
|
||||
resultContainer.CPULimit = cpuLimit
|
||||
} else {
|
||||
resultContainer.CPULimit = "inf"
|
||||
}
|
||||
memoryRequest := ksutil.JsonRawMessage(client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/memory/request")).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
resultContainer.MemoryRequest = ConvertMemory(memoryRequest)
|
||||
|
||||
memoryLimit := ksutil.JsonRawMessage(client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/memory/limit")).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
resultContainer.MemoryLimit = ConvertMemory(memoryLimit)
|
||||
|
||||
cpuUsageRate := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/cpu/usage_rate")
|
||||
if cpuUsageRate != "" {
|
||||
metrics := ksutil.JsonRawMessage(cpuUsageRate).Find("metrics").ToList()
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
cpu_utilization, _ := strconv.ParseFloat(ConvertCPUUsageRate(metric.Find("value").ToString()), 64)
|
||||
cpuMetrics.TimeStamp = timestamp.ToString()
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization)
|
||||
if resultContainer.CPULimit != "inf" {
|
||||
cpu_limit, _ := strconv.ParseFloat(resultContainer.CPULimit, 64)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", cpu_limit*cpu_utilization)
|
||||
} else {
|
||||
cpuMetrics.UsedCPU = "inf"
|
||||
}
|
||||
glog.Info("pod " + podName + " has limit cpu " + resultContainer.CPULimit + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization) + " at time" + timestamp.ToString())
|
||||
containerCPUMetrics = append(containerCPUMetrics, cpuMetrics)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
resultContainer.CPU = containerCPUMetrics
|
||||
|
||||
var used_mem_bytes float64
|
||||
|
||||
memUsage := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + podName + "/containers/" + containerName + "/metrics/memory/usage")
|
||||
if memUsage != "" {
|
||||
metrics := ksutil.JsonRawMessage(memUsage).Find("metrics").ToList()
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
used_mem_bytes, _ = strconv.ParseFloat(metric.Find("value").ToString(), 64)
|
||||
used_mem := used_mem_bytes / 1024 / 1024
|
||||
memMetrics.TimeStamp = timestamp.ToString()
|
||||
memMetrics.UsedMemory = fmt.Sprintf("%.1f", used_mem)
|
||||
memMetrics.MemoryUtilization = fmt.Sprintf("%.3f", CalculateMemoryUsage(memoryRequest, memoryLimit, metric.Find("value").ToString()))
|
||||
glog.Info("pod " + podName + " has limit mem " + resultContainer.MemoryLimit + " mem utilization " + memMetrics.MemoryUtilization + " at time" + timestamp.ToString())
|
||||
containerMemMetrics = append(containerMemMetrics, memMetrics)
|
||||
}
|
||||
}
|
||||
|
||||
resultContainer.Memory = containerMemMetrics
|
||||
|
||||
return resultContainer
|
||||
}
|
||||
|
||||
@@ -23,24 +23,38 @@ import (
|
||||
|
||||
"github.com/golang/glog"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
|
||||
ksutil "kubesphere.io/kubesphere/pkg/util"
|
||||
v1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"kubesphere.io/kubesphere/pkg/client"
|
||||
ksutil "kubesphere.io/kubesphere/pkg/util"
|
||||
|
||||
"fmt"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type ResultNodes struct {
|
||||
Nodes []ResultNode `json:"nodes"`
|
||||
}
|
||||
const (
|
||||
//status: "False"
|
||||
OutOfDisk = "OutOfDisk"
|
||||
//status: "False"
|
||||
MemoryPressure = "MemoryPressure"
|
||||
//status: "False"
|
||||
DiskPressure = "DiskPressure"
|
||||
//status: "False"
|
||||
PIDPressure = "PIDPressure"
|
||||
//status: "True"
|
||||
KubeletReady = "Ready"
|
||||
)
|
||||
|
||||
type ResultNode struct {
|
||||
NodeName string `json:"node_name"`
|
||||
PodsCount string `json:"pods_count"`
|
||||
PodsCapacity string `json:"pods_capacity"`
|
||||
CPU []CPUNode `json:"cpu"`
|
||||
Memory []MemoryNode `json:"memory"`
|
||||
NodeName string `json:"node_name"`
|
||||
NodeStatus string `json:"node_status"`
|
||||
PodsCount string `json:"pods_count"`
|
||||
PodsCapacity string `json:"pods_capacity"`
|
||||
UsedFS string `json:"used_fs"`
|
||||
TotalFS string `json:"total_fs"`
|
||||
FSUtilization string `json:"fs_utilization"`
|
||||
CPU []CPUNode `json:"cpu"`
|
||||
Memory []MemoryNode `json:"memory"`
|
||||
}
|
||||
|
||||
type CPUNode struct {
|
||||
@@ -98,13 +112,13 @@ func FormatNodeMetrics(nodeName string) ResultNode {
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
cpu_utilization, _ := strconv.ParseFloat(metric.Find("value").ToString(), 64)
|
||||
cpu_utilization, _ := strconv.ParseFloat(ConvertCPUUsageRate(metric.Find("value").ToString()), 64)
|
||||
cpuMetrics.TimeStamp = timestamp.ToString()
|
||||
cpuMetrics.TotalCPU = fmt.Sprintf("%.1f", total_cpu)
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization/1000)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", total_cpu*cpu_utilization/1000)
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", total_cpu*cpu_utilization)
|
||||
|
||||
glog.Info("node " + nodeName + " has total cpu " + fmt.Sprintf("%.1f", total_cpu) + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization/1000) + " at time" + timestamp.ToString())
|
||||
glog.Info("node " + nodeName + " has total cpu " + fmt.Sprintf("%.1f", total_cpu) + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization) + " at time" + timestamp.ToString())
|
||||
nodeCPUMetrics = append(nodeCPUMetrics, cpuMetrics)
|
||||
}
|
||||
|
||||
@@ -139,25 +153,65 @@ func FormatNodeMetrics(nodeName string) ResultNode {
|
||||
}
|
||||
|
||||
resultNode.NodeName = nodeName
|
||||
resultNode.PodsCount = strconv.Itoa(len(GetPodsForNode(nodeName,"")))
|
||||
resultNode.PodsCapacity = getPodsCapacity(nodeName)
|
||||
resultNode.PodsCount = strconv.Itoa(len(GetPodsForNode(nodeName, "")))
|
||||
nodeResObj := getNodeResObj(nodeName)
|
||||
resultNode.PodsCapacity = nodeResObj.Status.Capacity.Pods().String()
|
||||
resultNode.NodeStatus = getNodeStatus(nodeResObj)
|
||||
resultNode.UsedFS, resultNode.TotalFS, resultNode.FSUtilization = getNodeFileSystemStatus(nodeResObj)
|
||||
resultNode.CPU = nodeCPUMetrics
|
||||
resultNode.Memory = nodeMemMetrics
|
||||
|
||||
return resultNode
|
||||
}
|
||||
|
||||
func getPodsCapacity(nodeName string) string {
|
||||
var pods_capacity string
|
||||
func getNodeResObj(nodeName string) *v1.Node {
|
||||
cli := client.NewK8sClient()
|
||||
|
||||
node, err := cli.CoreV1().Nodes().Get(nodeName,metav1.GetOptions{})
|
||||
node, err := cli.CoreV1().Nodes().Get(nodeName, metav1.GetOptions{})
|
||||
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
} else {
|
||||
pods_capacity = node.Status.Capacity.Pods().String()
|
||||
return node
|
||||
}
|
||||
fmt.Println(pods_capacity)
|
||||
return pods_capacity
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func getNodeStatus(node *v1.Node) string {
|
||||
|
||||
status := "Ready"
|
||||
conditions := node.Status.Conditions
|
||||
for _, cond := range conditions {
|
||||
if cond.Type == DiskPressure && cond.Status == "True" {
|
||||
status = "NotReady"
|
||||
break
|
||||
}
|
||||
if cond.Type == OutOfDisk && cond.Status == "True" {
|
||||
status = "NotReady"
|
||||
break
|
||||
}
|
||||
if cond.Type == MemoryPressure && cond.Status == "True" {
|
||||
status = "NotReady"
|
||||
}
|
||||
if cond.Type == PIDPressure && cond.Status == "True" {
|
||||
status = "NotReady"
|
||||
break
|
||||
}
|
||||
if cond.Type == KubeletReady && cond.Status == "False" {
|
||||
status = "NotReady"
|
||||
break
|
||||
}
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
func getNodeFileSystemStatus(node *v1.Node) (string, string, string) {
|
||||
|
||||
nodeMetricsAsStr := client.GetCAdvisorMetrics(node.Annotations["alpha.kubernetes.io/provided-node-ip"])
|
||||
if nodeMetricsAsStr != "" {
|
||||
usedBytesAsStr, _ := strconv.ParseFloat(ksutil.JsonRawMessage(nodeMetricsAsStr).Find("node").Find("fs").Find("usedBytes").ToString(), 64)
|
||||
capacityBytesAsStr, _ := strconv.ParseFloat(ksutil.JsonRawMessage(nodeMetricsAsStr).Find("node").Find("fs").Find("capacityBytes").ToString(), 64)
|
||||
return fmt.Sprintf("%.1f", usedBytesAsStr/1024/1024/1024), fmt.Sprintf("%.1f", capacityBytesAsStr/1024/1024/1024), fmt.Sprintf("%.3f", usedBytesAsStr/capacityBytesAsStr)
|
||||
}
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
@@ -15,18 +15,14 @@ import (
|
||||
"strconv"
|
||||
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"kubesphere.io/kubesphere/pkg/constants"
|
||||
"math"
|
||||
)
|
||||
|
||||
type ResultNameSpaces struct {
|
||||
NameSpaces []ResultNameSpace `json:"namespaces"`
|
||||
}
|
||||
type ResultNameSpace struct {
|
||||
NameSpace string `json:"namespace"`
|
||||
PodsCount string `json:"pods_count"`
|
||||
Pods []ResultPod `json:"pods"`
|
||||
}
|
||||
type ResultPod struct {
|
||||
PodName string `json:"pod_name"`
|
||||
NameSpace string `json:"namespace"`
|
||||
NodeName string `json:"node_name"`
|
||||
CPURequest string `json:"cpu_request"`
|
||||
CPULimit string `json:"cpu_limit"`
|
||||
MemoryRequest string `json:"mem_request"`
|
||||
@@ -88,9 +84,9 @@ func GetPodsForNode(nodeName, namespace string) []string {
|
||||
return pods
|
||||
}
|
||||
|
||||
func FormatPodsMetrics(nodeName, namespace string) ResultNameSpace {
|
||||
var resultNameSpace ResultNameSpace
|
||||
var resultPods []ResultPod
|
||||
func FormatPodsMetrics(nodeName, namespace string) constants.PageableResponse {
|
||||
var result constants.PageableResponse
|
||||
|
||||
var resultPod ResultPod
|
||||
var pods []string
|
||||
if nodeName == "" {
|
||||
@@ -99,15 +95,19 @@ func FormatPodsMetrics(nodeName, namespace string) ResultNameSpace {
|
||||
pods = GetPodsForNode(nodeName, namespace)
|
||||
}
|
||||
|
||||
resultNameSpace.NameSpace = namespace
|
||||
resultNameSpace.PodsCount = strconv.Itoa(len(pods))
|
||||
|
||||
for _, pod := range pods {
|
||||
var total_count int
|
||||
for i, pod := range pods {
|
||||
resultPod = FormatPodMetrics(namespace, pod)
|
||||
resultPods = append(resultPods, resultPod)
|
||||
if nodeName != "" {
|
||||
resultPod.NodeName = nodeName
|
||||
} else {
|
||||
resultPod.NodeName = GetNodeNameForPod(pod, namespace)
|
||||
}
|
||||
result.Items = append(result.Items, resultPod)
|
||||
total_count = i
|
||||
}
|
||||
resultNameSpace.Pods = resultPods
|
||||
return resultNameSpace
|
||||
result.TotalCount = total_count + 1
|
||||
return result
|
||||
}
|
||||
|
||||
func FormatPodMetrics(namespace, pod string) ResultPod {
|
||||
@@ -119,6 +119,7 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
|
||||
var memMetrics MemoryPod
|
||||
|
||||
resultPod.PodName = pod
|
||||
resultPod.NameSpace = namespace
|
||||
cpuRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/cpu/request")
|
||||
cpuRequest = ksutil.JsonRawMessage(cpuRequest).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
if cpuRequest != "" && cpuRequest != "0" {
|
||||
@@ -134,10 +135,10 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
|
||||
} else {
|
||||
resultPod.CPULimit = "inf"
|
||||
}
|
||||
memoryRequest := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/request")
|
||||
memoryRequest := ksutil.JsonRawMessage(client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/request")).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
resultPod.MemoryRequest = ConvertMemory(memoryRequest)
|
||||
|
||||
memoryLimit := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/limit")
|
||||
memoryLimit := ksutil.JsonRawMessage(client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/memory/limit")).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
resultPod.MemoryLimit = ConvertMemory(memoryLimit)
|
||||
|
||||
cpuUsageRate := client.GetHeapsterMetrics("/namespaces/" + namespace + "/pods/" + pod + "/metrics/cpu/usage_rate")
|
||||
@@ -146,16 +147,16 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
|
||||
|
||||
for _, metric := range metrics {
|
||||
timestamp := metric.Find("timestamp")
|
||||
cpu_utilization, _ := strconv.ParseFloat(metric.Find("value").ToString(), 64)
|
||||
cpu_utilization, _ := strconv.ParseFloat(ConvertCPUUsageRate(metric.Find("value").ToString()), 64)
|
||||
cpuMetrics.TimeStamp = timestamp.ToString()
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization/1000)
|
||||
cpuMetrics.CPUUtilization = fmt.Sprintf("%.3f", cpu_utilization)
|
||||
if resultPod.CPULimit != "inf" {
|
||||
cpu_limit, _ := strconv.ParseFloat(resultPod.CPULimit, 64)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", cpu_limit*cpu_utilization/1000)
|
||||
cpuMetrics.UsedCPU = fmt.Sprintf("%.1f", cpu_limit*cpu_utilization)
|
||||
} else {
|
||||
cpuMetrics.UsedCPU = "inf"
|
||||
}
|
||||
glog.Info("pod " + pod + " has limit cpu " + resultPod.CPULimit + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization/1000) + " at time" + timestamp.ToString())
|
||||
glog.Info("pod " + pod + " has limit cpu " + resultPod.CPULimit + " CPU utilization " + fmt.Sprintf("%.3f", cpu_utilization) + " at time" + timestamp.ToString())
|
||||
podCPUMetrics = append(podCPUMetrics, cpuMetrics)
|
||||
}
|
||||
|
||||
@@ -175,12 +176,7 @@ func FormatPodMetrics(namespace, pod string) ResultPod {
|
||||
used_mem := used_mem_bytes / 1024 / 1024
|
||||
memMetrics.TimeStamp = timestamp.ToString()
|
||||
memMetrics.UsedMemory = fmt.Sprintf("%.1f", used_mem)
|
||||
if resultPod.MemoryLimit != "inf" {
|
||||
mem_limit, _ := strconv.ParseFloat(resultPod.MemoryLimit, 64)
|
||||
memMetrics.MemoryUtilization = fmt.Sprintf("%.3f", used_mem/mem_limit)
|
||||
} else {
|
||||
memMetrics.MemoryUtilization = "inf"
|
||||
}
|
||||
memMetrics.MemoryUtilization = fmt.Sprintf("%.3f", CalculateMemoryUsage(memoryRequest, memoryLimit, metric.Find("value").ToString()))
|
||||
|
||||
glog.Info("pod " + pod + " has limit mem " + resultPod.MemoryLimit + " mem utilization " + memMetrics.MemoryUtilization + " at time" + timestamp.ToString())
|
||||
podMemMetrics = append(podMemMetrics, memMetrics)
|
||||
@@ -195,11 +191,8 @@ func ConvertMemory(memBytes string) string {
|
||||
var mem string
|
||||
|
||||
if memBytes != "" {
|
||||
memMetric := ksutil.JsonRawMessage(memBytes).Find("metrics").ToList()[0].Find("value").ToString()
|
||||
|
||||
if memMetric != "" && memMetric != "0" {
|
||||
|
||||
memBytes, error := strconv.ParseFloat(memMetric, 64)
|
||||
if memBytes != "" && memBytes != "0" {
|
||||
memBytes, error := strconv.ParseFloat(memBytes, 64)
|
||||
if error == nil {
|
||||
mem = fmt.Sprintf("%.3f", memBytes/1024/1024)
|
||||
} else {
|
||||
@@ -215,7 +208,65 @@ func ConvertMemory(memBytes string) string {
|
||||
return mem
|
||||
}
|
||||
|
||||
func getNodeNameForPod(podName, namespace string) string {
|
||||
func CalculateMemoryUsage(requestMem, limitMem, usedMem string) float64 {
|
||||
var requestMemInBytes, limitMemInBytes, usedMemInBytes, memUsage float64
|
||||
if requestMem != "" && requestMem != "0" && requestMem != "inf" {
|
||||
requestMemInBytes, _ = strconv.ParseFloat(requestMem, 64)
|
||||
} else {
|
||||
glog.Info("memory request is not set")
|
||||
requestMemInBytes = 0
|
||||
}
|
||||
if limitMem != "" && limitMem != "0" && limitMem != "inf" {
|
||||
limitMemInBytes, _ = strconv.ParseFloat(limitMem, 64)
|
||||
} else {
|
||||
glog.Info("memory limit is not set")
|
||||
limitMemInBytes = 0
|
||||
}
|
||||
if usedMem != "" && usedMem != "0" {
|
||||
usedMemInBytes, _ = strconv.ParseFloat(usedMem, 64)
|
||||
} else {
|
||||
usedMemInBytes = 0
|
||||
}
|
||||
|
||||
if usedMemInBytes > 0 {
|
||||
if requestMemInBytes > 0 && limitMemInBytes > 0 {
|
||||
if usedMemInBytes > requestMemInBytes {
|
||||
glog.Info("used memory is higher than memory request")
|
||||
memUsage = usedMemInBytes / limitMemInBytes
|
||||
} else {
|
||||
memUsage = usedMemInBytes / requestMemInBytes
|
||||
}
|
||||
} else if requestMemInBytes > 0 && limitMemInBytes == 0 {
|
||||
if usedMemInBytes > requestMemInBytes {
|
||||
glog.Info("used memory is higher than memory request")
|
||||
memUsage = 0
|
||||
} else {
|
||||
memUsage = usedMemInBytes / requestMemInBytes
|
||||
}
|
||||
} else if requestMemInBytes == 0 && limitMemInBytes > 0 {
|
||||
if usedMemInBytes <= limitMemInBytes {
|
||||
memUsage = usedMemInBytes / limitMemInBytes
|
||||
}
|
||||
} else {
|
||||
memUsage = 0
|
||||
}
|
||||
} else {
|
||||
memUsage = 0
|
||||
}
|
||||
return memUsage
|
||||
}
|
||||
|
||||
func ConvertCPUUsageRate(cpuUsageRate string) string {
|
||||
if cpuUsageRate != "" && cpuUsageRate != "0" {
|
||||
rate, _ := strconv.ParseFloat(cpuUsageRate, 64)
|
||||
rateBase := math.Pow10(strings.Count(cpuUsageRate, "") - 1)
|
||||
return fmt.Sprintf("%.3f", rate/rateBase)
|
||||
} else {
|
||||
return "0"
|
||||
}
|
||||
}
|
||||
|
||||
func GetNodeNameForPod(podName, namespace string) string {
|
||||
var nodeName string
|
||||
cli := client.NewK8sClient()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user