Merge remote-tracking branch 'upstream/master'

# Conflicts: # cmd/ks-apiserver/app/server.go
2019-04-01 02:48:10 +08:00
parent 1b06d5a9ee 01082acedb
commit 744bd053e3
296 changed files with 59820 additions and 1262 deletions
--- a/pkg/apiserver/logging/logging.go
+++ b/pkg/apiserver/logging/logging.go
@@ -0,0 +1,216 @@
+/*
+
+ Copyright 2019 The KubeSphere Authors.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+*/
+
+package logging
+
+import (
+	"github.com/emicklei/go-restful"
+	"github.com/golang/glog"
+	"kubesphere.io/kubesphere/pkg/models/log"
+	es "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch"
+	fb "kubesphere.io/kubesphere/pkg/simple/client/fluentbit"
+	"net/http"
+	"strconv"
+)
+
+func LoggingQueryCluster(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelCluster, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryWorkspace(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelWorkspace, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryNamespace(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelNamespace, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryWorkload(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelWorkload, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryPod(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelPod, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryContainer(request *restful.Request, response *restful.Response) {
+	res := logQuery(log.QueryLevelContainer, request)
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryFluentbitFilters(request *restful.Request, response *restful.Response) {
+	res := log.FluentbitFiltersQuery()
+	response.WriteAsJson(res)
+}
+
+func LoggingUpdateFluentbitFilters(request *restful.Request, response *restful.Response) {
+
+	var res *log.FluentbitFiltersResult
+
+	filters := new([]log.FluentbitFilter)
+
+	err := request.ReadEntity(&filters)
+	if err != nil {
+		res = &log.FluentbitFiltersResult{Status: http.StatusBadRequest}
+	} else {
+		res = log.FluentbitFiltersUpdate(filters)
+	}
+
+	response.WriteAsJson(res)
+}
+
+func LoggingQueryFluentbitOutputs(request *restful.Request, response *restful.Response) {
+	res := log.FluentbitOutputsQuery()
+	response.WriteAsJson(res)
+}
+
+func LoggingInsertFluentbitOutput(request *restful.Request, response *restful.Response) {
+
+	var output fb.OutputPlugin
+	var res *log.FluentbitOutputsResult
+
+	err := request.ReadEntity(&output)
+	if err != nil {
+		glog.Errorln(err)
+		res = &log.FluentbitOutputsResult{Status: http.StatusBadRequest}
+	} else {
+		res = log.FluentbitOutputInsert(output)
+	}
+
+	response.WriteAsJson(res)
+}
+
+func LoggingUpdateFluentbitOutput(request *restful.Request, response *restful.Response) {
+
+	var output fb.OutputPlugin
+
+	id := request.PathParameter("output")
+
+	err := request.ReadEntity(&output)
+	if err != nil {
+		glog.Errorln(err)
+		res := &log.FluentbitOutputsResult{Status: http.StatusBadRequest}
+		response.WriteAsJson(res)
+		return
+	}
+
+	res := log.FluentbitOutputUpdate(output, id)
+	response.WriteAsJson(res)
+}
+
+func LoggingDeleteFluentbitOutput(request *restful.Request, response *restful.Response) {
+
+	var res *log.FluentbitOutputsResult
+
+	id := request.PathParameter("output")
+	res = log.FluentbitOutputDelete(id)
+
+	response.WriteAsJson(res)
+}
+
+func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult {
+	var param es.QueryParameters
+
+	param.Operation = request.QueryParameter("operation")
+
+	switch level {
+	case log.QueryLevelCluster:
+		{
+			param.NamespaceFilled, param.Namespaces = log.QueryWorkspace(request.QueryParameter("workspaces"), request.QueryParameter("workspace_query"))
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.QueryParameter("namespaces"), param.NamespaceFilled, param.Namespaces)
+			param.NamespaceQuery = request.QueryParameter("namespace_query")
+			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
+			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
+			param.PodQuery = request.QueryParameter("pod_query")
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
+			param.ContainerQuery = request.QueryParameter("container_query")
+		}
+	case log.QueryLevelWorkspace:
+		{
+			param.NamespaceFilled, param.Namespaces = log.QueryWorkspace(request.PathParameter("workspace"), "")
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.QueryParameter("namespaces"), param.NamespaceFilled, param.Namespaces)
+			param.NamespaceQuery = request.QueryParameter("namespace_query")
+			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
+			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
+			param.PodQuery = request.QueryParameter("pod_query")
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
+			param.ContainerQuery = request.QueryParameter("container_query")
+		}
+	case log.QueryLevelNamespace:
+		{
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
+			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
+			param.PodQuery = request.QueryParameter("pod_query")
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
+			param.ContainerQuery = request.QueryParameter("container_query")
+		}
+	case log.QueryLevelWorkload:
+		{
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.PodFilled, param.Pods = log.QueryWorkload(request.PathParameter("workload"), "", param.Namespaces)
+			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
+			param.PodQuery = request.QueryParameter("pod_query")
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
+			param.ContainerQuery = request.QueryParameter("container_query")
+		}
+	case log.QueryLevelPod:
+		{
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.PodFilled, param.Pods = log.MatchPod(request.PathParameter("pod"), false, nil)
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
+			param.ContainerQuery = request.QueryParameter("container_query")
+		}
+	case log.QueryLevelContainer:
+		{
+			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.PodFilled, param.Pods = log.MatchPod(request.PathParameter("pod"), false, nil)
+			param.ContainerFilled, param.Containers = log.MatchContainer(request.PathParameter("container"))
+		}
+	}
+
+	if len(param.Namespaces) == 1 {
+		param.Workspace = log.GetWorkspaceOfNamesapce(param.Namespaces[0])
+	}
+
+	param.Interval = request.QueryParameter("interval")
+
+	param.LogQuery = request.QueryParameter("log_query")
+	param.StartTime = request.QueryParameter("start_time")
+	param.EndTime = request.QueryParameter("end_time")
+	param.Sort = request.QueryParameter("sort")
+
+	var err error
+	param.From, err = strconv.ParseInt(request.QueryParameter("from"), 10, 64)
+	if err != nil {
+		param.From = 0
+	}
+	param.Size, err = strconv.ParseInt(request.QueryParameter("size"), 10, 64)
+	if err != nil {
+		param.Size = 10
+	}
+
+	glog.Infof("LogQuery with %v", param)
+
+	return es.Query(param)
+}
--- a/pkg/apiserver/monitoring/monitoring.go
+++ b/pkg/apiserver/monitoring/monitoring.go
@@ -20,11 +20,11 @@ package monitoring
 import (
 	"github.com/emicklei/go-restful"
 	"kubesphere.io/kubesphere/pkg/models/metrics"
-	"kubesphere.io/kubesphere/pkg/simple/client/prometheus"
+	prom "kubesphere.io/kubesphere/pkg/simple/client/prometheus"
 )

 func MonitorPod(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)
 	podName := requestParams.PodName
 	metricName := requestParams.MetricsName
 	if podName != "" {
@@ -32,7 +32,8 @@ func MonitorPod(request *restful.Request, response *restful.Response) {
 		queryType, params, nullRule := metrics.AssemblePodMetricRequestInfo(requestParams, metricName)
 		var res *metrics.FormatedMetric
 		if !nullRule {
-			res = metrics.GetMetric(queryType, params, metricName)
+			metricsStr := prom.SendMonitoringRequest(queryType, params)
+			res = metrics.ReformatJson(metricsStr, metricName, map[string]string{"pod_name": ""})
 		}
 		response.WriteAsJson(res)

@@ -40,21 +41,20 @@ func MonitorPod(request *restful.Request, response *restful.Response) {
 		// multiple
 		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelPod)
 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelPodName)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
-
 		response.WriteAsJson(pagedMetrics)
 	}
 }

 func MonitorContainer(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)
 	metricName := requestParams.MetricsName
 	if requestParams.MetricsFilter != "" {
 		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelContainer)
 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelContainerName)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)

@@ -68,7 +68,7 @@ func MonitorContainer(request *restful.Request, response *restful.Response) {
 }

 func MonitorWorkload(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)

 	rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkload)

@@ -80,10 +80,10 @@ func MonitorWorkload(request *restful.Request, response *restful.Response) {
 	// sorting
 	if wlKind == "" {

-		sortedMetrics, maxMetricCount = metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkload)
+		sortedMetrics, maxMetricCount = metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 	} else {

-		sortedMetrics, maxMetricCount = metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelPodName)
+		sortedMetrics, maxMetricCount = metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 	}

 	// paging
@@ -95,19 +95,18 @@ func MonitorWorkload(request *restful.Request, response *restful.Response) {

 func MonitorAllWorkspaces(request *restful.Request, response *restful.Response) {

-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)

 	tp := requestParams.Tp
-	if tp == "_statistics" {
+	if tp == "statistics" {
 		// merge multiple metric: all-devops, all-roles, all-projects...this api is designed for admin
 		res := metrics.MonitorAllWorkspacesStatistics()
-
 		response.WriteAsJson(res)

 	} else if tp == "rank" {
 		rawMetrics := metrics.MonitorAllWorkspaces(requestParams)
 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelWorkspace)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)

@@ -119,20 +118,19 @@ func MonitorAllWorkspaces(request *restful.Request, response *restful.Response)
 }

 func MonitorOneWorkspace(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)

 	tp := requestParams.Tp
 	if tp == "rank" {
 		// multiple
 		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelWorkspace)
 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
-
 		response.WriteAsJson(pagedMetrics)

-	} else if tp == "_statistics" {
+	} else if tp == "statistics" {
 		wsName := requestParams.WsName

 		// merge multiple metric: devops, roles, projects...
@@ -145,34 +143,35 @@ func MonitorOneWorkspace(request *restful.Request, response *restful.Response) {
 }

 func MonitorNamespace(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)
 	metricName := requestParams.MetricsName
 	nsName := requestParams.NsName
 	if nsName != "" {
 		// single
 		queryType, params := metrics.AssembleNamespaceMetricRequestInfo(requestParams, metricName)
-		res := metrics.GetMetric(queryType, params, metricName)
+		metricsStr := prom.SendMonitoringRequest(queryType, params)
+		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{"namespace": ""})
 		response.WriteAsJson(res)
 	} else {
 		// multiple
 		rawMetrics := metrics.MonitorAllMetrics(requestParams, metrics.MetricLevelNamespace)
 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNamespace)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)
-
 		response.WriteAsJson(pagedMetrics)
 	}
 }

 func MonitorCluster(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)

 	metricName := requestParams.MetricsName
 	if metricName != "" {
 		// single
 		queryType, params := metrics.AssembleClusterMetricRequestInfo(requestParams, metricName)
-		res := metrics.GetMetric(queryType, params, metricName)
+		metricsStr := prom.SendMonitoringRequest(queryType, params)
+		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{"cluster": "local"})

 		response.WriteAsJson(res)
 	} else {
@@ -183,15 +182,17 @@ func MonitorCluster(request *restful.Request, response *restful.Response) {
 }

 func MonitorNode(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
+	requestParams := prom.ParseMonitoringRequestParams(request)

 	metricName := requestParams.MetricsName
 	if metricName != "" {
 		// single
 		queryType, params := metrics.AssembleNodeMetricRequestInfo(requestParams, metricName)
-		res := metrics.GetMetric(queryType, params, metricName)
+		metricsStr := prom.SendMonitoringRequest(queryType, params)
+		res := metrics.ReformatJson(metricsStr, metricName, map[string]string{"node": ""})
 		nodeAddress := metrics.GetNodeAddressInfo()
 		metrics.AddNodeAddressMetric(res, nodeAddress)
+
 		response.WriteAsJson(res)
 	} else {
 		// multiple
@@ -203,18 +204,10 @@ func MonitorNode(request *restful.Request, response *restful.Response) {
 		}

 		// sorting
-		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics, metrics.MetricLevelNode)
+		sortedMetrics, maxMetricCount := metrics.Sort(requestParams.SortMetricName, requestParams.SortType, rawMetrics)
 		// paging
 		pagedMetrics := metrics.Page(requestParams.PageNum, requestParams.LimitNum, sortedMetrics, maxMetricCount)

 		response.WriteAsJson(pagedMetrics)
 	}
 }
-
-// k8s component(controller, scheduler, etcd) status
-func MonitorComponentStatus(request *restful.Request, response *restful.Response) {
-	requestParams := prometheus.ParseMonitoringRequestParams(request)
-
-	status := metrics.MonitorComponentStatus(requestParams)
-	response.WriteAsJson(status)
-}
--- a/pkg/apiserver/runtime/runtime.go
+++ b/pkg/apiserver/runtime/runtime.go
@@ -23,7 +23,7 @@ import (
 )

 const (
-	ApiRootPath = "/apis"
+	ApiRootPath = "/kapis"
 )

 // container holds all webservice of apiserver
--- a/pkg/apiserver/servicemesh/metrics/handlers.go
+++ b/pkg/apiserver/servicemesh/metrics/handlers.go
@@ -8,22 +8,29 @@ import (

 // Get app metrics
 func GetAppMetrics(request *restful.Request, response *restful.Response) {
-	handlers.AppMetrics(response.ResponseWriter, request.Request)
+	handlers.AppMetrics(request, response)
 }

 // Get workload metrics
 func GetWorkloadMetrics(request *restful.Request, response *restful.Response) {
-	handlers.WorkloadMetrics(response.ResponseWriter, request.Request)
+	namespace := request.PathParameter("namespace")
+	workload := request.PathParameter("workload")
+
+	if len(namespace) > 0 && len(workload) > 0 {
+		request.Request.URL.RawQuery = fmt.Sprintf("%s&namespaces=%s&workload=%s", request.Request.URL.RawQuery, namespace, workload)
+	}
+
+	handlers.WorkloadMetrics(request, response)
 }

 // Get service metrics
 func GetServiceMetrics(request *restful.Request, response *restful.Response) {
-	handlers.ServiceMetrics(response.ResponseWriter, request.Request)
+	handlers.ServiceMetrics(request, response)
 }

 // Get namespace metrics
 func GetNamespaceMetrics(request *restful.Request, response *restful.Response) {
-	handlers.NamespaceMetrics(response.ResponseWriter, request.Request)
+	handlers.NamespaceMetrics(request, response)
 }

 // Get service graph for namespace
@@ -34,10 +41,30 @@ func GetNamespaceGraph(request *restful.Request, response *restful.Response) {
 		request.Request.URL.RawQuery = fmt.Sprintf("%s&namespaces=%s", request.Request.URL.RawQuery, namespace)
 	}

-	handlers.GraphNamespaces(response.ResponseWriter, request.Request)
+	handlers.GetNamespaceGraph(request, response)
 }

 // Get service graph for namespaces
 func GetNamespacesGraph(request *restful.Request, response *restful.Response) {
-	handlers.GraphNamespaces(response.ResponseWriter, request.Request)
+	handlers.GraphNamespaces(request, response)
+}
+
+// Get namespace health
+func GetNamespaceHealth(request *restful.Request, response *restful.Response) {
+	handlers.NamespaceHealth(request, response)
+}
+
+// Get workload health
+func GetWorkloadHealth(request *restful.Request, response *restful.Response) {
+	handlers.WorkloadHealth(request, response)
+}
+
+// Get app health
+func GetAppHealth(request *restful.Request, response *restful.Response) {
+	handlers.AppHealth(request, response)
+}
+
+// Get service health
+func GetServiceHealth(request *restful.Request, response *restful.Response) {
+	handlers.ServiceHealth(request, response)
 }