Merge pull request #394 from huanggze/logging-dev

fix: log query starttime must be greater than the namespace creation time

pkg/apiserver/logging/logging.go

@@ -138,6 +138,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 		{
 			param.NamespaceFilled, param.Namespaces = log.QueryWorkspace(request.QueryParameter("workspaces"), request.QueryParameter("workspace_query"))
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.QueryParameter("namespaces"), param.NamespaceFilled, param.Namespaces)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.NamespaceQuery = request.QueryParameter("namespace_query")
 			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
@@ -149,6 +150,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 		{
 			param.NamespaceFilled, param.Namespaces = log.QueryWorkspace(request.PathParameter("workspace"), "")
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.QueryParameter("namespaces"), param.NamespaceFilled, param.Namespaces)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.NamespaceQuery = request.QueryParameter("namespace_query")
 			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
@@ -159,6 +161,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelNamespace:
 		{
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.PodFilled, param.Pods = log.QueryWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
 			param.PodQuery = request.QueryParameter("pod_query")
@@ -168,6 +171,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelWorkload:
 		{
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.PodFilled, param.Pods = log.QueryWorkload(request.PathParameter("workload"), "", param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.QueryParameter("pods"), param.PodFilled, param.Pods)
 			param.PodQuery = request.QueryParameter("pod_query")
@@ -177,6 +181,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelPod:
 		{
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.PathParameter("pod"), false, nil)
 			param.ContainerFilled, param.Containers = log.MatchContainer(request.QueryParameter("containers"))
 			param.ContainerQuery = request.QueryParameter("container_query")
@@ -184,6 +189,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelContainer:
 		{
 			param.NamespaceFilled, param.Namespaces = log.MatchNamespace(request.PathParameter("namespace"), false, nil)
+			param.NamespaceFilled, param.NamespaceWithCreationTime = log.GetNamespaceCreationTimeMap(param.Namespaces)
 			param.PodFilled, param.Pods = log.MatchPod(request.PathParameter("pod"), false, nil)
 			param.ContainerFilled, param.Containers = log.MatchContainer(request.PathParameter("container"))
 		}

pkg/models/log/logcollector.go

@@ -24,7 +24,9 @@ import (
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/informers"
 	"reflect"
+	"strconv"
 	"strings"
+	"time"
 )
 
 func intersection(s1, s2 []string) (inter []string) {
@@ -181,6 +183,37 @@ func MatchNamespace(namespaceMatch string, namespaceFilled bool, namespaces []st
 	return true, namespacesMatch
 }
 
+func GetNamespaceCreationTimeMap(namespaces []string) (bool, map[string]string) {
+
+	namespaceWithCreationTime := make(map[string]string)
+
+	nsLister := informers.SharedInformerFactory().Core().V1().Namespaces().Lister()
+
+	if len(namespaces) == 0 {
+		nsList, err := nsLister.List(labels.Everything())
+		if err != nil {
+			glog.Error("failed to list namespace, error: ", err)
+			return true, namespaceWithCreationTime
+		}
+
+		for _, ns := range nsList {
+			namespaces = append(namespaces, ns.Name)
+		}
+	}
+
+	for _, item := range namespaces {
+		ns, err := nsLister.Get(item)
+		if err != nil {
+			glog.Error("failed to get namespace, error: ", err)
+			continue
+		}
+
+		namespaceWithCreationTime[ns.Name] = strconv.FormatInt(ns.CreationTimestamp.UnixNano()/int64(time.Millisecond), 10)
+	}
+
+	return true, namespaceWithCreationTime
+}
+
 func QueryWorkload(workloadMatch string, workloadQuery string, namespaces []string) (bool, []string) {
 	if workloadMatch == "" && workloadQuery == "" {
 		return false, nil

pkg/simple/client/elasticsearch/esclient.go

@@ -58,7 +58,7 @@ type Request struct {
 	From          int64         `json:"from"`
 	Size          int64         `json:"size"`
 	Sorts         []Sort        `json:"sort,omitempty"`
-	MainQuery     MainQuery     `json:"query"`
+	MainQuery     BoolQuery     `json:"query"`
 	Aggs          interface{}   `json:"aggs,omitempty"`
 	MainHighLight MainHighLight `json:"highlight,omitempty"`
 }
@@ -71,11 +71,11 @@ type Order struct {
 	Order string `json:"order"`
 }
 
-type MainQuery struct {
-	MainBoolQuery MainBoolQuery `json:"bool"`
+type BoolQuery struct {
+	BoolMusts BoolMusts `json:"bool"`
 }
 
-type MainBoolQuery struct {
+type BoolMusts struct {
 	Musts []interface{} `json:"must"`
 }
 
@@ -173,17 +173,24 @@ type DateHistogram struct {
 
 func createQueryRequest(param QueryParameters) (int, []byte, error) {
 	var request Request
-	var mainBoolQuery MainBoolQuery
+	var mainBoolQuery BoolMusts
 
 	if param.NamespaceFilled {
 		var shouldMatchPhrase ShouldMatchPhrase
-		if len(param.Namespaces) == 0 {
+		if len(param.NamespaceWithCreationTime) == 0 {
 			matchPhrase := MatchPhrase{map[string]interface{}{"kubernetes.namespace_name.key_word": QueryWord{""}}}
 			shouldMatchPhrase.Shoulds = append(shouldMatchPhrase.Shoulds, matchPhrase)
 		} else {
-			for _, namespace := range param.Namespaces {
+			for namespace, creationTime := range param.NamespaceWithCreationTime {
+				var boolQuery BoolQuery
+
 				matchPhrase := MatchPhrase{map[string]interface{}{"kubernetes.namespace_name.keyword": QueryWord{namespace}}}
-				shouldMatchPhrase.Shoulds = append(shouldMatchPhrase.Shoulds, matchPhrase)
+				rangeQuery := RangeQuery{RangeSpec{TimeRange{creationTime, ""}}}
+
+				boolQuery.BoolMusts.Musts = append(boolQuery.BoolMusts.Musts, matchPhrase)
+				boolQuery.BoolMusts.Musts = append(boolQuery.BoolMusts.Musts, rangeQuery)
+
+				shouldMatchPhrase.Shoulds = append(shouldMatchPhrase.Shoulds, boolQuery)
 			}
 		}
 		shouldMatchPhrase.MinimumShouldMatch = 1
@@ -278,7 +285,7 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
 		request.MainHighLight = mainHighLight
 	}
 
-	request.MainQuery = MainQuery{mainBoolQuery}
+	request.MainQuery = BoolQuery{mainBoolQuery}
 
 	queryRequest, err := json.Marshal(request)
 
@@ -544,12 +551,13 @@ func parseQueryResult(operation int, param QueryParameters, body []byte, query [
 }
 
 type QueryParameters struct {
-	NamespaceFilled bool
-	Namespaces      []string
-	PodFilled       bool
-	Pods            []string
-	ContainerFilled bool
-	Containers      []string
+	NamespaceFilled           bool
+	Namespaces                []string
+	NamespaceWithCreationTime map[string]string
+	PodFilled                 bool
+	Pods                      []string
+	ContainerFilled           bool
+	Containers                []string
 
 	NamespaceQuery string
 	PodQuery       string