allow to search deleted workload logs

Signed-off-by: huanggze <loganhuang@yunify.com>

pkg/apiserver/logging/logging.go

@@ -170,7 +170,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 			stringutils.Split(request.QueryParameter("workspaces"), ","),
 			stringutils.Split(strings.ToLower(request.QueryParameter("workspace_query")), ",")) // case-insensitive
 		param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces)
-		param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces)
+		param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",")
+		param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",")
 		param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",")
 		param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",")
 		param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",")
@@ -181,7 +182,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 			stringutils.Split(strings.ToLower(request.QueryParameter("namespace_query")), ","), // case-insensitive
 			stringutils.Split(request.PathParameter("workspace"), ","), nil)                    // case-insensitive
 		param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces)
-		param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces)
+		param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",")
+		param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",")
 		param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",")
 		param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",")
 		param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",")
@@ -189,7 +191,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelNamespace:
 		namespaces := []string{request.PathParameter("namespace")}
 		param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces)
-		param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces)
+		param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",")
+		param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",")
 		param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",")
 		param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",")
 		param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",")
@@ -197,7 +200,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult
 	case log.QueryLevelWorkload:
 		namespaces := []string{request.PathParameter("namespace")}
 		param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces)
-		param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.PathParameter("workload"), "", namespaces)
+		param.WorkloadFilter = []string{request.PathParameter("workload")}
 		param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",")
 		param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",")
 		param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",")

pkg/models/log/logcollector.go

@@ -24,71 +24,11 @@ import (
 	"kubesphere.io/kubesphere/pkg/constants"
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/utils/stringutils"
-	"reflect"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func in(value interface{}, container interface{}) int {
-	if container == nil {
-		return -1
-	}
-	containerValue := reflect.ValueOf(container)
-	switch reflect.TypeOf(container).Kind() {
-	case reflect.Slice, reflect.Array:
-		for i := 0; i < containerValue.Len(); i++ {
-			if containerValue.Index(i).Interface() == value {
-				return i
-			}
-		}
-	case reflect.Map:
-		if containerValue.MapIndex(reflect.ValueOf(value)).IsValid() {
-			return -1
-		}
-	default:
-		return -1
-	}
-	return -1
-}
-
-func getWorkloadName(name string, kind string) string {
-	if kind == "ReplicaSet" {
-		lastIndex := strings.LastIndex(name, "-")
-		if lastIndex >= 0 {
-			return name[:lastIndex]
-		}
-	}
-
-	return name
-}
-
-func matchLabel(label string, labelsMatch []string) bool {
-	var result = false
-
-	for _, labelMatch := range labelsMatch {
-		if strings.Compare(label, labelMatch) == 0 {
-			result = true
-			break
-		}
-	}
-
-	return result
-}
-
-func queryLabel(label string, labelsQuery []string) bool {
-	var result = false
-
-	for _, labelQuery := range labelsQuery {
-		if strings.Contains(label, labelQuery) {
-			result = true
-			break
-		}
-	}
-
-	return result
-}
-
 // list namespaces that match search conditions
 func MatchNamespace(nsFilter []string, nsQuery []string, wsFilter []string, wsQuery []string) (bool, []string) {
 
@@ -149,86 +89,3 @@ func MakeNamespaceCreationTimeMap(namespaces []string) map[string]string {
 
 	return namespaceWithCreationTime
 }
-
-func MatchWorkload(workloadMatch string, workloadQuery string, namespaces []string) (bool, []string) {
-	if workloadMatch == "" && workloadQuery == "" {
-		return false, nil
-	}
-
-	podLister := informers.SharedInformerFactory().Core().V1().Pods().Lister()
-	podList, err := podLister.List(labels.Everything())
-	if err != nil {
-		glog.Error("failed to list pods, error: ", err)
-		return true, nil
-	}
-
-	var pods []string
-
-	var hasMatch = false
-	var workloadsMatch []string
-	if workloadMatch != "" {
-		workloadsMatch = strings.Split(strings.Replace(workloadMatch, ",", " ", -1), " ")
-		hasMatch = true
-	}
-
-	var hasQuery = false
-	var workloadsQuery []string
-	if workloadQuery != "" {
-		workloadsQuery = strings.Split(strings.ToLower(strings.Replace(workloadQuery, ",", " ", -1)), " ")
-		hasQuery = true
-	}
-
-	if namespaces == nil {
-		for _, pod := range podList {
-			/*if len(pod.ObjectMeta.OwnerReferences) > 0 {
-				glog.Infof("List Pod %v:%v:%v", pod.Name, pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind)
-			}*/
-			if len(pod.ObjectMeta.OwnerReferences) > 0 {
-				var podCanAppend = true
-				workloadName := getWorkloadName(pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind)
-				if hasMatch {
-					if !matchLabel(workloadName, workloadsMatch) {
-						podCanAppend = false
-					}
-				}
-				if hasQuery {
-					if !queryLabel(strings.ToLower(workloadName), workloadsQuery) {
-						podCanAppend = false
-					}
-				}
-
-				if podCanAppend {
-					pods = append(pods, pod.Name)
-				}
-			}
-		}
-	} else {
-		for _, pod := range podList {
-			/*if len(pod.ObjectMeta.OwnerReferences) > 0 {
-				glog.Infof("List Pod %v:%v:%v", pod.Name, pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind)
-			}*/
-			if len(pod.ObjectMeta.OwnerReferences) > 0 && in(pod.Namespace, namespaces) >= 0 {
-				var podCanAppend = true
-				workloadName := getWorkloadName(pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind)
-				if hasMatch {
-					if !matchLabel(workloadName, workloadsMatch) {
-						podCanAppend = false
-					}
-				}
-				if hasQuery {
-					if !queryLabel(strings.ToLower(workloadName), workloadsQuery) {
-						podCanAppend = false
-					}
-				}
-
-				if podCanAppend {
-					pods = append(pods, pod.Name)
-				}
-			}
-		}
-	}
-
-	// if workloads is equal to nil, indicates no workload matched
-	// it causes the query to return no result
-	return pods == nil, pods
-}

pkg/simple/client/elasticsearch/esclient.go

@@ -26,8 +26,19 @@ import (
 )
 
 const (
+	OperationQuery int = iota
+	OperationStatistics
+	OperationHistogram
+
 	matchPhrase = iota
 	matchPhrasePrefix
+	regexpQuery
+
+	podNameMaxLength = 63
+	// max 10 characters + 1 hyphen
+	replicaSetSuffixMaxLength = 11
+	// a unique random string as suffix, 5 characters + 1 hyphen
+	randSuffixLength = 6
 
 	fieldPodName       = "kubernetes.pod_name"
 	fieldContainerName = "kubernetes.container_name"
@@ -47,13 +58,6 @@ var (
 	client Client
 )
 
-type Config struct {
-	Host         string
-	Port         string
-	Index        string
-	VersionMajor string
-}
-
 func (cfg *Config) WriteESConfigs() {
 	mu.Lock()
 	defer mu.Unlock()
@@ -68,109 +72,6 @@ func (cfg *Config) WriteESConfigs() {
 	client = NewForConfig(config)
 }
 
-type Request struct {
-	From          int64         `json:"from"`
-	Size          int64         `json:"size"`
-	Sorts         []Sort        `json:"sort,omitempty"`
-	MainQuery     BoolQuery     `json:"query"`
-	Aggs          interface{}   `json:"aggs,omitempty"`
-	MainHighLight MainHighLight `json:"highlight,omitempty"`
-}
-
-type Sort struct {
-	Order Order `json:"time"`
-}
-
-type Order struct {
-	Order string `json:"order"`
-}
-
-type BoolQuery struct {
-	Bool interface{} `json:"bool"`
-}
-
-// user filter instead of must
-// filter ignores scoring
-type BoolFilter struct {
-	Filter []interface{} `json:"filter"`
-}
-
-type BoolShould struct {
-	Should             []interface{} `json:"should"`
-	MinimumShouldMatch int64         `json:"minimum_should_match"`
-}
-
-type RangeQuery struct {
-	RangeSpec RangeSpec `json:"range"`
-}
-
-type RangeSpec struct {
-	TimeRange TimeRange `json:"time"`
-}
-
-type TimeRange struct {
-	Gte string `json:"gte,omitempty"`
-	Lte string `json:"lte,omitempty"`
-}
-
-type MatchPhrase struct {
-	MatchPhrase map[string]string `json:"match_phrase"`
-}
-
-type MatchPhrasePrefix struct {
-	MatchPhrasePrefix interface{} `json:"match_phrase_prefix"`
-}
-
-type MainHighLight struct {
-	Fields       []interface{} `json:"fields,omitempty"`
-	FragmentSize int           `json:"fragment_size"`
-}
-
-type LogHighLightField struct {
-	FieldContent EmptyField `json:"log"`
-}
-
-type NamespaceHighLightField struct {
-	FieldContent EmptyField `json:"kubernetes.namespace_name.keyword"`
-}
-
-type PodHighLightField struct {
-	FieldContent EmptyField `json:"kubernetes.pod_name.keyword"`
-}
-
-type ContainerHighLightField struct {
-	FieldContent EmptyField `json:"kubernetes.container_name.keyword"`
-}
-
-type EmptyField struct {
-}
-
-// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting
-type StatisticsAggs struct {
-	ContainerAgg ContainerAgg `json:"containers"`
-}
-
-type ContainerAgg struct {
-	Cardinality AggField `json:"cardinality"`
-}
-
-type AggField struct {
-	Field string `json:"field"`
-}
-
-type HistogramAggs struct {
-	HistogramAgg HistogramAgg `json:"histogram"`
-}
-
-type HistogramAgg struct {
-	DateHistogram DateHistogram `json:"date_histogram"`
-}
-
-type DateHistogram struct {
-	Field    string `json:"field"`
-	Interval string `json:"interval"`
-}
-
 func createQueryRequest(param QueryParameters) (int, []byte, error) {
 	var request Request
 	var mainBoolQuery BoolFilter
@@ -192,7 +93,7 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
 		mainBoolQuery.Filter = append(mainBoolQuery.Filter, BoolQuery{Bool: boolShould})
 	}
 	if param.WorkloadFilter != nil {
-		boolQuery := makeBoolShould(matchPhrase, fieldPodNameKeyword, param.WorkloadFilter)
+		boolQuery := makeBoolShould(regexpQuery, fieldPodNameKeyword, param.WorkloadFilter)
 		mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery)
 	}
 	if param.PodFilter != nil {
@@ -204,6 +105,10 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
 		mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery)
 	}
 
+	if param.WorkloadQuery != nil {
+		boolQuery := makeBoolShould(matchPhrasePrefix, fieldPodName, param.WorkloadQuery)
+		mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery)
+	}
 	if param.PodQuery != nil {
 		boolQuery := makeBoolShould(matchPhrasePrefix, fieldPodName, param.PodQuery)
 		mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery)
@@ -278,6 +183,8 @@ func makeBoolShould(queryType int, field string, list []string) BoolQuery {
 			q = MatchPhrase{MatchPhrase: map[string]string{field: phrase}}
 		case matchPhrasePrefix:
 			q = MatchPhrasePrefix{MatchPhrasePrefix: map[string]string{field: phrase}}
+		case regexpQuery:
+			q = RegexpQuery{Regexp: map[string]string{field: makePodNameRegexp(phrase)}}
 		}
 
 		should = append(should, q)
@@ -291,127 +198,31 @@ func makeBoolShould(queryType int, field string, list []string) BoolQuery {
 	}
 }
 
-// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html
+func makePodNameRegexp(workloadName string) string {
-// Response from the elasticsearch engine
+	var regexp string
-type Response struct {
+	if len(workloadName) <= podNameMaxLength-replicaSetSuffixMaxLength-randSuffixLength {
-	Status       int             `json:"status"`
+		// match deployment pods, eg. <deploy>-579dfbcddd-24znw
-	Workspace    string          `json:"workspace,omitempty"`
+		// replicaset rand string is limited to vowels
-	Shards       Shards          `json:"_shards"`
+		// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go#L83
-	Hits         Hits            `json:"hits"`
+		regexp += workloadName + "-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|"
-	Aggregations json.RawMessage `json:"aggregations"`
+		// match statefulset pods, eg. <sts>-0
+		regexp += workloadName + "-[0-9]+|"
+		// match pods of daemonset or job, eg. <ds>-29tdk, <job>-5xqvl
+		regexp += workloadName + "-[a-z0-9]{5}"
+	} else if len(workloadName) <= podNameMaxLength-randSuffixLength {
+		replicaSetSuffixLength := podNameMaxLength - randSuffixLength - len(workloadName)
+		regexp += fmt.Sprintf("%s%d%s", workloadName+"-[bcdfghjklmnpqrstvwxz2456789]{", replicaSetSuffixLength, "}[a-z0-9]{5}|")
+		regexp += workloadName + "-[0-9]+|"
+		regexp += workloadName + "-[a-z0-9]{5}"
+	} else {
+		// Rand suffix may overwrites the workload name if the name is too long
+		// This won't happen for StatefulSet because a statefulset pod will fail to create
+		regexp += workloadName[:podNameMaxLength-randSuffixLength+1] + "[a-z0-9]{5}|"
+		regexp += workloadName + "-[0-9]+"
 	}
-
+	return regexp
-type Shards struct {
-	Total      int64 `json:"total"`
-	Successful int64 `json:"successful"`
-	Skipped    int64 `json:"skipped"`
-	Failed     int64 `json:"failed"`
 }
 
-type Hits struct {
-	// As of ElasticSearch v7.x, hits.total is changed
-	Total interface{} `json:"total"`
-	Hits  []Hit       `json:"hits"`
-}
-
-type Hit struct {
-	Source    Source    `json:"_source"`
-	HighLight HighLight `json:"highlight"`
-	Sort      []int64   `json:"sort"`
-}
-
-type Source struct {
-	Log        string     `json:"log"`
-	Time       string     `json:"time"`
-	Kubernetes Kubernetes `json:"kubernetes"`
-}
-
-type Kubernetes struct {
-	Namespace string `json:"namespace_name"`
-	Pod       string `json:"pod_name"`
-	Container string `json:"container_name"`
-	Host      string `json:"host"`
-}
-
-type HighLight struct {
-	LogHighLights       []string `json:"log,omitempty" description:"log messages to highlight"`
-	NamespaceHighLights []string `json:"kubernetes.namespace_name.keyword,omitempty" description:"namespaces to highlight"`
-	PodHighLights       []string `json:"kubernetes.pod_name.keyword,omitempty" description:"pods to highlight"`
-	ContainerHighLights []string `json:"kubernetes.container_name.keyword,omitempty" description:"containers to highlight"`
-}
-
-type LogRecord struct {
-	Time      int64     `json:"time,omitempty" description:"log timestamp"`
-	Log       string    `json:"log,omitempty" description:"log message"`
-	Namespace string    `json:"namespace,omitempty" description:"namespace"`
-	Pod       string    `json:"pod,omitempty" description:"pod name"`
-	Container string    `json:"container,omitempty" description:"container name"`
-	Host      string    `json:"host,omitempty" description:"node id"`
-	HighLight HighLight `json:"highlight,omitempty" description:"highlighted log fragment"`
-}
-
-type ReadResult struct {
-	Total   int64       `json:"total" description:"total number of matched results"`
-	From    int64       `json:"from" description:"the offset from the result set"`
-	Size    int64       `json:"size" description:"the amount of hits to be returned"`
-	Records []LogRecord `json:"records,omitempty" description:"actual array of results"`
-}
-
-// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs
-type StatisticsResponseAggregations struct {
-	ContainerCount ContainerCount `json:"containers"`
-}
-
-type ContainerCount struct {
-	Value int64 `json:"value"`
-}
-
-type HistogramAggregations struct {
-	HistogramAggregation HistogramAggregation `json:"histogram"`
-}
-
-type HistogramAggregation struct {
-	Histograms []HistogramStatistics `json:"buckets"`
-}
-
-type HistogramStatistics struct {
-	Time  int64 `json:"key"`
-	Count int64 `json:"doc_count"`
-}
-
-type HistogramRecord struct {
-	Time  int64 `json:"time" description:"timestamp"`
-	Count int64 `json:"count" description:"total number of logs at intervals"`
-}
-
-type StatisticsResult struct {
-	Containers int64 `json:"containers" description:"total number of containers"`
-	Logs       int64 `json:"logs" description:"total number of logs"`
-}
-
-type HistogramResult struct {
-	Total      int64             `json:"total" description:"total number of logs"`
-	StartTime  int64             `json:"start_time" description:"start time"`
-	EndTime    int64             `json:"end_time" description:"end time"`
-	Interval   string            `json:"interval" description:"interval"`
-	Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"`
-}
-
-// Wrap elasticsearch response
-type QueryResult struct {
-	Status     int               `json:"status,omitempty" description:"query status"`
-	Error      string            `json:"error,omitempty" description:"debugging information"`
-	Read       *ReadResult       `json:"query,omitempty" description:"query results"`
-	Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"`
-	Histogram  *HistogramResult  `json:"histogram,omitempty" description:"histogram results"`
-}
-
-const (
-	OperationQuery int = iota
-	OperationStatistics
-	OperationHistogram
-)
-
 func calcTimestamp(input string) int64 {
 	var t time.Time
 	var err error
@@ -521,38 +332,11 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
 	return &queryResult
 }
 
-type QueryParameters struct {
-	// when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace
-	NamespaceNotFound bool
-	// a map of namespace with creation time
-	NamespaceWithCreationTime map[string]string
-
-	// when true, indicates the provided `workloads` or `workload_query` doesn't match any workload
-	WorkloadNotFound bool
-	WorkloadFilter   []string
-
-	PodFilter []string
-	PodQuery  []string
-
-	ContainerFilter []string
-	ContainerQuery  []string
-
-	LogQuery []string
-
-	Operation string
-	Interval  string
-	StartTime string
-	EndTime   string
-	Sort      string
-	From      int64
-	Size      int64
-}
-
 func Query(param QueryParameters) *QueryResult {
 
 	var queryResult = new(QueryResult)
 
-	if param.NamespaceNotFound || param.WorkloadNotFound {
+	if param.NamespaceNotFound {
 		queryResult = new(QueryResult)
 		queryResult.Status = http.StatusOK
 		switch param.Operation {

pkg/simple/client/elasticsearch/types.go

@@ -0,0 +1,261 @@
+package esclient
+
+import (
+	"encoding/json"
+)
+
+// elasticsearch client config
+type Config struct {
+	Host         string
+	Port         string
+	Index        string
+	VersionMajor string
+}
+
+type QueryParameters struct {
+	// when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace
+	NamespaceNotFound bool
+	// a map of namespace with creation time
+	NamespaceWithCreationTime map[string]string
+
+	// filter for literally matching
+	// query for fuzzy matching
+	WorkloadFilter  []string
+	WorkloadQuery   []string
+	PodFilter       []string
+	PodQuery        []string
+	ContainerFilter []string
+	ContainerQuery  []string
+	LogQuery        []string
+
+	Operation string
+	Interval  string
+	StartTime string
+	EndTime   string
+	Sort      string
+	From      int64
+	Size      int64
+}
+
+// elasticsearch request body
+type Request struct {
+	From          int64         `json:"from"`
+	Size          int64         `json:"size"`
+	Sorts         []Sort        `json:"sort,omitempty"`
+	MainQuery     BoolQuery     `json:"query"`
+	Aggs          interface{}   `json:"aggs,omitempty"`
+	MainHighLight MainHighLight `json:"highlight,omitempty"`
+}
+
+type Sort struct {
+	Order Order `json:"time"`
+}
+
+type Order struct {
+	Order string `json:"order"`
+}
+
+type BoolQuery struct {
+	Bool interface{} `json:"bool"`
+}
+
+// user filter instead of must
+// filter ignores scoring
+type BoolFilter struct {
+	Filter []interface{} `json:"filter"`
+}
+
+type BoolShould struct {
+	Should             []interface{} `json:"should"`
+	MinimumShouldMatch int64         `json:"minimum_should_match"`
+}
+
+type RangeQuery struct {
+	RangeSpec RangeSpec `json:"range"`
+}
+
+type RangeSpec struct {
+	TimeRange TimeRange `json:"time"`
+}
+
+type TimeRange struct {
+	Gte string `json:"gte,omitempty"`
+	Lte string `json:"lte,omitempty"`
+}
+
+type MatchPhrase struct {
+	MatchPhrase map[string]string `json:"match_phrase"`
+}
+
+type MatchPhrasePrefix struct {
+	MatchPhrasePrefix interface{} `json:"match_phrase_prefix"`
+}
+
+type RegexpQuery struct {
+	Regexp interface{} `json:"regexp"`
+}
+
+type MainHighLight struct {
+	Fields       []interface{} `json:"fields,omitempty"`
+	FragmentSize int           `json:"fragment_size"`
+}
+
+type LogHighLightField struct {
+	FieldContent EmptyField `json:"log"`
+}
+
+type NamespaceHighLightField struct {
+	FieldContent EmptyField `json:"kubernetes.namespace_name.keyword"`
+}
+
+type PodHighLightField struct {
+	FieldContent EmptyField `json:"kubernetes.pod_name.keyword"`
+}
+
+type ContainerHighLightField struct {
+	FieldContent EmptyField `json:"kubernetes.container_name.keyword"`
+}
+
+type EmptyField struct {
+}
+
+// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting
+type StatisticsAggs struct {
+	ContainerAgg ContainerAgg `json:"containers"`
+}
+
+type ContainerAgg struct {
+	Cardinality AggField `json:"cardinality"`
+}
+
+type AggField struct {
+	Field string `json:"field"`
+}
+
+type HistogramAggs struct {
+	HistogramAgg HistogramAgg `json:"histogram"`
+}
+
+type HistogramAgg struct {
+	DateHistogram DateHistogram `json:"date_histogram"`
+}
+
+type DateHistogram struct {
+	Field    string `json:"field"`
+	Interval string `json:"interval"`
+}
+
+// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html
+// Response body from the elasticsearch engine
+type Response struct {
+	Status       int             `json:"status"`
+	Workspace    string          `json:"workspace,omitempty"`
+	Shards       Shards          `json:"_shards"`
+	Hits         Hits            `json:"hits"`
+	Aggregations json.RawMessage `json:"aggregations"`
+}
+
+type Shards struct {
+	Total      int64 `json:"total"`
+	Successful int64 `json:"successful"`
+	Skipped    int64 `json:"skipped"`
+	Failed     int64 `json:"failed"`
+}
+
+type Hits struct {
+	// As of ElasticSearch v7.x, hits.total is changed
+	Total interface{} `json:"total"`
+	Hits  []Hit       `json:"hits"`
+}
+
+type Hit struct {
+	Source    Source    `json:"_source"`
+	HighLight HighLight `json:"highlight"`
+	Sort      []int64   `json:"sort"`
+}
+
+type Source struct {
+	Log        string     `json:"log"`
+	Time       string     `json:"time"`
+	Kubernetes Kubernetes `json:"kubernetes"`
+}
+
+type Kubernetes struct {
+	Namespace string `json:"namespace_name"`
+	Pod       string `json:"pod_name"`
+	Container string `json:"container_name"`
+	Host      string `json:"host"`
+}
+
+type HighLight struct {
+	LogHighLights       []string `json:"log,omitempty" description:"log messages to highlight"`
+	NamespaceHighLights []string `json:"kubernetes.namespace_name.keyword,omitempty" description:"namespaces to highlight"`
+	PodHighLights       []string `json:"kubernetes.pod_name.keyword,omitempty" description:"pods to highlight"`
+	ContainerHighLights []string `json:"kubernetes.container_name.keyword,omitempty" description:"containers to highlight"`
+}
+
+type LogRecord struct {
+	Time      int64     `json:"time,omitempty" description:"log timestamp"`
+	Log       string    `json:"log,omitempty" description:"log message"`
+	Namespace string    `json:"namespace,omitempty" description:"namespace"`
+	Pod       string    `json:"pod,omitempty" description:"pod name"`
+	Container string    `json:"container,omitempty" description:"container name"`
+	Host      string    `json:"host,omitempty" description:"node id"`
+	HighLight HighLight `json:"highlight,omitempty" description:"highlighted log fragment"`
+}
+
+type ReadResult struct {
+	Total   int64       `json:"total" description:"total number of matched results"`
+	From    int64       `json:"from" description:"the offset from the result set"`
+	Size    int64       `json:"size" description:"the amount of hits to be returned"`
+	Records []LogRecord `json:"records,omitempty" description:"actual array of results"`
+}
+
+// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs
+type StatisticsResponseAggregations struct {
+	ContainerCount ContainerCount `json:"containers"`
+}
+
+type ContainerCount struct {
+	Value int64 `json:"value"`
+}
+
+type HistogramAggregations struct {
+	HistogramAggregation HistogramAggregation `json:"histogram"`
+}
+
+type HistogramAggregation struct {
+	Histograms []HistogramStatistics `json:"buckets"`
+}
+
+type HistogramStatistics struct {
+	Time  int64 `json:"key"`
+	Count int64 `json:"doc_count"`
+}
+
+type HistogramRecord struct {
+	Time  int64 `json:"time" description:"timestamp"`
+	Count int64 `json:"count" description:"total number of logs at intervals"`
+}
+
+type StatisticsResult struct {
+	Containers int64 `json:"containers" description:"total number of containers"`
+	Logs       int64 `json:"logs" description:"total number of logs"`
+}
+
+type HistogramResult struct {
+	Total      int64             `json:"total" description:"total number of logs"`
+	StartTime  int64             `json:"start_time" description:"start time"`
+	EndTime    int64             `json:"end_time" description:"end time"`
+	Interval   string            `json:"interval" description:"interval"`
+	Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"`
+}
+
+// Wrap elasticsearch response
+type QueryResult struct {
+	Status     int               `json:"status,omitempty" description:"query status"`
+	Error      string            `json:"error,omitempty" description:"debugging information"`
+	Read       *ReadResult       `json:"query,omitempty" description:"query results"`
+	Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"`
+	Histogram  *HistogramResult  `json:"histogram,omitempty" description:"histogram results"`
+}