From 8d85a563866e80de9723f34637a192910ff4639c Mon Sep 17 00:00:00 2001 From: huanggze Date: Wed, 11 Sep 2019 10:35:44 +0800 Subject: [PATCH] allow to search deleted workload logs Signed-off-by: huanggze --- pkg/apiserver/logging/logging.go | 11 +- pkg/models/log/logcollector.go | 143 ---------- pkg/simple/client/elasticsearch/esclient.go | 300 +++----------------- pkg/simple/client/elasticsearch/types.go | 261 +++++++++++++++++ 4 files changed, 310 insertions(+), 405 deletions(-) create mode 100644 pkg/simple/client/elasticsearch/types.go diff --git a/pkg/apiserver/logging/logging.go b/pkg/apiserver/logging/logging.go index 461df0850..7238ce9a5 100644 --- a/pkg/apiserver/logging/logging.go +++ b/pkg/apiserver/logging/logging.go @@ -170,7 +170,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult stringutils.Split(request.QueryParameter("workspaces"), ","), stringutils.Split(strings.ToLower(request.QueryParameter("workspace_query")), ",")) // case-insensitive param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces) - param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces) + param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",") + param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",") param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",") param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",") param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",") @@ -181,7 +182,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult stringutils.Split(strings.ToLower(request.QueryParameter("namespace_query")), ","), // case-insensitive stringutils.Split(request.PathParameter("workspace"), ","), nil) // case-insensitive param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces) - param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces) + param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",") + param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",") param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",") param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",") param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",") @@ -189,7 +191,8 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult case log.QueryLevelNamespace: namespaces := []string{request.PathParameter("namespace")} param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces) - param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.QueryParameter("workloads"), request.QueryParameter("workload_query"), namespaces) + param.WorkloadFilter = stringutils.Split(request.QueryParameter("workloads"), ",") + param.WorkloadQuery = stringutils.Split(request.QueryParameter("workload_query"), ",") param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",") param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",") param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",") @@ -197,7 +200,7 @@ func logQuery(level log.LogQueryLevel, request *restful.Request) *es.QueryResult case log.QueryLevelWorkload: namespaces := []string{request.PathParameter("namespace")} param.NamespaceWithCreationTime = log.MakeNamespaceCreationTimeMap(namespaces) - param.WorkloadNotFound, param.WorkloadFilter = log.MatchWorkload(request.PathParameter("workload"), "", namespaces) + param.WorkloadFilter = []string{request.PathParameter("workload")} param.PodFilter = stringutils.Split(request.QueryParameter("pods"), ",") param.PodQuery = stringutils.Split(request.QueryParameter("pod_query"), ",") param.ContainerFilter = stringutils.Split(request.QueryParameter("containers"), ",") diff --git a/pkg/models/log/logcollector.go b/pkg/models/log/logcollector.go index 78f159cbd..dcfcb1952 100644 --- a/pkg/models/log/logcollector.go +++ b/pkg/models/log/logcollector.go @@ -24,71 +24,11 @@ import ( "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/utils/stringutils" - "reflect" "strconv" "strings" "time" ) -func in(value interface{}, container interface{}) int { - if container == nil { - return -1 - } - containerValue := reflect.ValueOf(container) - switch reflect.TypeOf(container).Kind() { - case reflect.Slice, reflect.Array: - for i := 0; i < containerValue.Len(); i++ { - if containerValue.Index(i).Interface() == value { - return i - } - } - case reflect.Map: - if containerValue.MapIndex(reflect.ValueOf(value)).IsValid() { - return -1 - } - default: - return -1 - } - return -1 -} - -func getWorkloadName(name string, kind string) string { - if kind == "ReplicaSet" { - lastIndex := strings.LastIndex(name, "-") - if lastIndex >= 0 { - return name[:lastIndex] - } - } - - return name -} - -func matchLabel(label string, labelsMatch []string) bool { - var result = false - - for _, labelMatch := range labelsMatch { - if strings.Compare(label, labelMatch) == 0 { - result = true - break - } - } - - return result -} - -func queryLabel(label string, labelsQuery []string) bool { - var result = false - - for _, labelQuery := range labelsQuery { - if strings.Contains(label, labelQuery) { - result = true - break - } - } - - return result -} - // list namespaces that match search conditions func MatchNamespace(nsFilter []string, nsQuery []string, wsFilter []string, wsQuery []string) (bool, []string) { @@ -149,86 +89,3 @@ func MakeNamespaceCreationTimeMap(namespaces []string) map[string]string { return namespaceWithCreationTime } - -func MatchWorkload(workloadMatch string, workloadQuery string, namespaces []string) (bool, []string) { - if workloadMatch == "" && workloadQuery == "" { - return false, nil - } - - podLister := informers.SharedInformerFactory().Core().V1().Pods().Lister() - podList, err := podLister.List(labels.Everything()) - if err != nil { - glog.Error("failed to list pods, error: ", err) - return true, nil - } - - var pods []string - - var hasMatch = false - var workloadsMatch []string - if workloadMatch != "" { - workloadsMatch = strings.Split(strings.Replace(workloadMatch, ",", " ", -1), " ") - hasMatch = true - } - - var hasQuery = false - var workloadsQuery []string - if workloadQuery != "" { - workloadsQuery = strings.Split(strings.ToLower(strings.Replace(workloadQuery, ",", " ", -1)), " ") - hasQuery = true - } - - if namespaces == nil { - for _, pod := range podList { - /*if len(pod.ObjectMeta.OwnerReferences) > 0 { - glog.Infof("List Pod %v:%v:%v", pod.Name, pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind) - }*/ - if len(pod.ObjectMeta.OwnerReferences) > 0 { - var podCanAppend = true - workloadName := getWorkloadName(pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind) - if hasMatch { - if !matchLabel(workloadName, workloadsMatch) { - podCanAppend = false - } - } - if hasQuery { - if !queryLabel(strings.ToLower(workloadName), workloadsQuery) { - podCanAppend = false - } - } - - if podCanAppend { - pods = append(pods, pod.Name) - } - } - } - } else { - for _, pod := range podList { - /*if len(pod.ObjectMeta.OwnerReferences) > 0 { - glog.Infof("List Pod %v:%v:%v", pod.Name, pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind) - }*/ - if len(pod.ObjectMeta.OwnerReferences) > 0 && in(pod.Namespace, namespaces) >= 0 { - var podCanAppend = true - workloadName := getWorkloadName(pod.ObjectMeta.OwnerReferences[0].Name, pod.ObjectMeta.OwnerReferences[0].Kind) - if hasMatch { - if !matchLabel(workloadName, workloadsMatch) { - podCanAppend = false - } - } - if hasQuery { - if !queryLabel(strings.ToLower(workloadName), workloadsQuery) { - podCanAppend = false - } - } - - if podCanAppend { - pods = append(pods, pod.Name) - } - } - } - } - - // if workloads is equal to nil, indicates no workload matched - // it causes the query to return no result - return pods == nil, pods -} diff --git a/pkg/simple/client/elasticsearch/esclient.go b/pkg/simple/client/elasticsearch/esclient.go index 745ff5f25..a6d527e18 100644 --- a/pkg/simple/client/elasticsearch/esclient.go +++ b/pkg/simple/client/elasticsearch/esclient.go @@ -26,8 +26,19 @@ import ( ) const ( + OperationQuery int = iota + OperationStatistics + OperationHistogram + matchPhrase = iota matchPhrasePrefix + regexpQuery + + podNameMaxLength = 63 + // max 10 characters + 1 hyphen + replicaSetSuffixMaxLength = 11 + // a unique random string as suffix, 5 characters + 1 hyphen + randSuffixLength = 6 fieldPodName = "kubernetes.pod_name" fieldContainerName = "kubernetes.container_name" @@ -47,13 +58,6 @@ var ( client Client ) -type Config struct { - Host string - Port string - Index string - VersionMajor string -} - func (cfg *Config) WriteESConfigs() { mu.Lock() defer mu.Unlock() @@ -68,109 +72,6 @@ func (cfg *Config) WriteESConfigs() { client = NewForConfig(config) } -type Request struct { - From int64 `json:"from"` - Size int64 `json:"size"` - Sorts []Sort `json:"sort,omitempty"` - MainQuery BoolQuery `json:"query"` - Aggs interface{} `json:"aggs,omitempty"` - MainHighLight MainHighLight `json:"highlight,omitempty"` -} - -type Sort struct { - Order Order `json:"time"` -} - -type Order struct { - Order string `json:"order"` -} - -type BoolQuery struct { - Bool interface{} `json:"bool"` -} - -// user filter instead of must -// filter ignores scoring -type BoolFilter struct { - Filter []interface{} `json:"filter"` -} - -type BoolShould struct { - Should []interface{} `json:"should"` - MinimumShouldMatch int64 `json:"minimum_should_match"` -} - -type RangeQuery struct { - RangeSpec RangeSpec `json:"range"` -} - -type RangeSpec struct { - TimeRange TimeRange `json:"time"` -} - -type TimeRange struct { - Gte string `json:"gte,omitempty"` - Lte string `json:"lte,omitempty"` -} - -type MatchPhrase struct { - MatchPhrase map[string]string `json:"match_phrase"` -} - -type MatchPhrasePrefix struct { - MatchPhrasePrefix interface{} `json:"match_phrase_prefix"` -} - -type MainHighLight struct { - Fields []interface{} `json:"fields,omitempty"` - FragmentSize int `json:"fragment_size"` -} - -type LogHighLightField struct { - FieldContent EmptyField `json:"log"` -} - -type NamespaceHighLightField struct { - FieldContent EmptyField `json:"kubernetes.namespace_name.keyword"` -} - -type PodHighLightField struct { - FieldContent EmptyField `json:"kubernetes.pod_name.keyword"` -} - -type ContainerHighLightField struct { - FieldContent EmptyField `json:"kubernetes.container_name.keyword"` -} - -type EmptyField struct { -} - -// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting -type StatisticsAggs struct { - ContainerAgg ContainerAgg `json:"containers"` -} - -type ContainerAgg struct { - Cardinality AggField `json:"cardinality"` -} - -type AggField struct { - Field string `json:"field"` -} - -type HistogramAggs struct { - HistogramAgg HistogramAgg `json:"histogram"` -} - -type HistogramAgg struct { - DateHistogram DateHistogram `json:"date_histogram"` -} - -type DateHistogram struct { - Field string `json:"field"` - Interval string `json:"interval"` -} - func createQueryRequest(param QueryParameters) (int, []byte, error) { var request Request var mainBoolQuery BoolFilter @@ -192,7 +93,7 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) { mainBoolQuery.Filter = append(mainBoolQuery.Filter, BoolQuery{Bool: boolShould}) } if param.WorkloadFilter != nil { - boolQuery := makeBoolShould(matchPhrase, fieldPodNameKeyword, param.WorkloadFilter) + boolQuery := makeBoolShould(regexpQuery, fieldPodNameKeyword, param.WorkloadFilter) mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery) } if param.PodFilter != nil { @@ -204,6 +105,10 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) { mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery) } + if param.WorkloadQuery != nil { + boolQuery := makeBoolShould(matchPhrasePrefix, fieldPodName, param.WorkloadQuery) + mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery) + } if param.PodQuery != nil { boolQuery := makeBoolShould(matchPhrasePrefix, fieldPodName, param.PodQuery) mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery) @@ -278,6 +183,8 @@ func makeBoolShould(queryType int, field string, list []string) BoolQuery { q = MatchPhrase{MatchPhrase: map[string]string{field: phrase}} case matchPhrasePrefix: q = MatchPhrasePrefix{MatchPhrasePrefix: map[string]string{field: phrase}} + case regexpQuery: + q = RegexpQuery{Regexp: map[string]string{field: makePodNameRegexp(phrase)}} } should = append(should, q) @@ -291,127 +198,31 @@ func makeBoolShould(queryType int, field string, list []string) BoolQuery { } } -// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html -// Response from the elasticsearch engine -type Response struct { - Status int `json:"status"` - Workspace string `json:"workspace,omitempty"` - Shards Shards `json:"_shards"` - Hits Hits `json:"hits"` - Aggregations json.RawMessage `json:"aggregations"` +func makePodNameRegexp(workloadName string) string { + var regexp string + if len(workloadName) <= podNameMaxLength-replicaSetSuffixMaxLength-randSuffixLength { + // match deployment pods, eg. -579dfbcddd-24znw + // replicaset rand string is limited to vowels + // https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go#L83 + regexp += workloadName + "-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|" + // match statefulset pods, eg. -0 + regexp += workloadName + "-[0-9]+|" + // match pods of daemonset or job, eg. -29tdk, -5xqvl + regexp += workloadName + "-[a-z0-9]{5}" + } else if len(workloadName) <= podNameMaxLength-randSuffixLength { + replicaSetSuffixLength := podNameMaxLength - randSuffixLength - len(workloadName) + regexp += fmt.Sprintf("%s%d%s", workloadName+"-[bcdfghjklmnpqrstvwxz2456789]{", replicaSetSuffixLength, "}[a-z0-9]{5}|") + regexp += workloadName + "-[0-9]+|" + regexp += workloadName + "-[a-z0-9]{5}" + } else { + // Rand suffix may overwrites the workload name if the name is too long + // This won't happen for StatefulSet because a statefulset pod will fail to create + regexp += workloadName[:podNameMaxLength-randSuffixLength+1] + "[a-z0-9]{5}|" + regexp += workloadName + "-[0-9]+" + } + return regexp } -type Shards struct { - Total int64 `json:"total"` - Successful int64 `json:"successful"` - Skipped int64 `json:"skipped"` - Failed int64 `json:"failed"` -} - -type Hits struct { - // As of ElasticSearch v7.x, hits.total is changed - Total interface{} `json:"total"` - Hits []Hit `json:"hits"` -} - -type Hit struct { - Source Source `json:"_source"` - HighLight HighLight `json:"highlight"` - Sort []int64 `json:"sort"` -} - -type Source struct { - Log string `json:"log"` - Time string `json:"time"` - Kubernetes Kubernetes `json:"kubernetes"` -} - -type Kubernetes struct { - Namespace string `json:"namespace_name"` - Pod string `json:"pod_name"` - Container string `json:"container_name"` - Host string `json:"host"` -} - -type HighLight struct { - LogHighLights []string `json:"log,omitempty" description:"log messages to highlight"` - NamespaceHighLights []string `json:"kubernetes.namespace_name.keyword,omitempty" description:"namespaces to highlight"` - PodHighLights []string `json:"kubernetes.pod_name.keyword,omitempty" description:"pods to highlight"` - ContainerHighLights []string `json:"kubernetes.container_name.keyword,omitempty" description:"containers to highlight"` -} - -type LogRecord struct { - Time int64 `json:"time,omitempty" description:"log timestamp"` - Log string `json:"log,omitempty" description:"log message"` - Namespace string `json:"namespace,omitempty" description:"namespace"` - Pod string `json:"pod,omitempty" description:"pod name"` - Container string `json:"container,omitempty" description:"container name"` - Host string `json:"host,omitempty" description:"node id"` - HighLight HighLight `json:"highlight,omitempty" description:"highlighted log fragment"` -} - -type ReadResult struct { - Total int64 `json:"total" description:"total number of matched results"` - From int64 `json:"from" description:"the offset from the result set"` - Size int64 `json:"size" description:"the amount of hits to be returned"` - Records []LogRecord `json:"records,omitempty" description:"actual array of results"` -} - -// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs -type StatisticsResponseAggregations struct { - ContainerCount ContainerCount `json:"containers"` -} - -type ContainerCount struct { - Value int64 `json:"value"` -} - -type HistogramAggregations struct { - HistogramAggregation HistogramAggregation `json:"histogram"` -} - -type HistogramAggregation struct { - Histograms []HistogramStatistics `json:"buckets"` -} - -type HistogramStatistics struct { - Time int64 `json:"key"` - Count int64 `json:"doc_count"` -} - -type HistogramRecord struct { - Time int64 `json:"time" description:"timestamp"` - Count int64 `json:"count" description:"total number of logs at intervals"` -} - -type StatisticsResult struct { - Containers int64 `json:"containers" description:"total number of containers"` - Logs int64 `json:"logs" description:"total number of logs"` -} - -type HistogramResult struct { - Total int64 `json:"total" description:"total number of logs"` - StartTime int64 `json:"start_time" description:"start time"` - EndTime int64 `json:"end_time" description:"end time"` - Interval string `json:"interval" description:"interval"` - Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"` -} - -// Wrap elasticsearch response -type QueryResult struct { - Status int `json:"status,omitempty" description:"query status"` - Error string `json:"error,omitempty" description:"debugging information"` - Read *ReadResult `json:"query,omitempty" description:"query results"` - Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"` - Histogram *HistogramResult `json:"histogram,omitempty" description:"histogram results"` -} - -const ( - OperationQuery int = iota - OperationStatistics - OperationHistogram -) - func calcTimestamp(input string) int64 { var t time.Time var err error @@ -521,38 +332,11 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR return &queryResult } -type QueryParameters struct { - // when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace - NamespaceNotFound bool - // a map of namespace with creation time - NamespaceWithCreationTime map[string]string - - // when true, indicates the provided `workloads` or `workload_query` doesn't match any workload - WorkloadNotFound bool - WorkloadFilter []string - - PodFilter []string - PodQuery []string - - ContainerFilter []string - ContainerQuery []string - - LogQuery []string - - Operation string - Interval string - StartTime string - EndTime string - Sort string - From int64 - Size int64 -} - func Query(param QueryParameters) *QueryResult { var queryResult = new(QueryResult) - if param.NamespaceNotFound || param.WorkloadNotFound { + if param.NamespaceNotFound { queryResult = new(QueryResult) queryResult.Status = http.StatusOK switch param.Operation { diff --git a/pkg/simple/client/elasticsearch/types.go b/pkg/simple/client/elasticsearch/types.go new file mode 100644 index 000000000..0cb5958e8 --- /dev/null +++ b/pkg/simple/client/elasticsearch/types.go @@ -0,0 +1,261 @@ +package esclient + +import ( + "encoding/json" +) + +// elasticsearch client config +type Config struct { + Host string + Port string + Index string + VersionMajor string +} + +type QueryParameters struct { + // when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace + NamespaceNotFound bool + // a map of namespace with creation time + NamespaceWithCreationTime map[string]string + + // filter for literally matching + // query for fuzzy matching + WorkloadFilter []string + WorkloadQuery []string + PodFilter []string + PodQuery []string + ContainerFilter []string + ContainerQuery []string + LogQuery []string + + Operation string + Interval string + StartTime string + EndTime string + Sort string + From int64 + Size int64 +} + +// elasticsearch request body +type Request struct { + From int64 `json:"from"` + Size int64 `json:"size"` + Sorts []Sort `json:"sort,omitempty"` + MainQuery BoolQuery `json:"query"` + Aggs interface{} `json:"aggs,omitempty"` + MainHighLight MainHighLight `json:"highlight,omitempty"` +} + +type Sort struct { + Order Order `json:"time"` +} + +type Order struct { + Order string `json:"order"` +} + +type BoolQuery struct { + Bool interface{} `json:"bool"` +} + +// user filter instead of must +// filter ignores scoring +type BoolFilter struct { + Filter []interface{} `json:"filter"` +} + +type BoolShould struct { + Should []interface{} `json:"should"` + MinimumShouldMatch int64 `json:"minimum_should_match"` +} + +type RangeQuery struct { + RangeSpec RangeSpec `json:"range"` +} + +type RangeSpec struct { + TimeRange TimeRange `json:"time"` +} + +type TimeRange struct { + Gte string `json:"gte,omitempty"` + Lte string `json:"lte,omitempty"` +} + +type MatchPhrase struct { + MatchPhrase map[string]string `json:"match_phrase"` +} + +type MatchPhrasePrefix struct { + MatchPhrasePrefix interface{} `json:"match_phrase_prefix"` +} + +type RegexpQuery struct { + Regexp interface{} `json:"regexp"` +} + +type MainHighLight struct { + Fields []interface{} `json:"fields,omitempty"` + FragmentSize int `json:"fragment_size"` +} + +type LogHighLightField struct { + FieldContent EmptyField `json:"log"` +} + +type NamespaceHighLightField struct { + FieldContent EmptyField `json:"kubernetes.namespace_name.keyword"` +} + +type PodHighLightField struct { + FieldContent EmptyField `json:"kubernetes.pod_name.keyword"` +} + +type ContainerHighLightField struct { + FieldContent EmptyField `json:"kubernetes.container_name.keyword"` +} + +type EmptyField struct { +} + +// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting +type StatisticsAggs struct { + ContainerAgg ContainerAgg `json:"containers"` +} + +type ContainerAgg struct { + Cardinality AggField `json:"cardinality"` +} + +type AggField struct { + Field string `json:"field"` +} + +type HistogramAggs struct { + HistogramAgg HistogramAgg `json:"histogram"` +} + +type HistogramAgg struct { + DateHistogram DateHistogram `json:"date_histogram"` +} + +type DateHistogram struct { + Field string `json:"field"` + Interval string `json:"interval"` +} + +// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html +// Response body from the elasticsearch engine +type Response struct { + Status int `json:"status"` + Workspace string `json:"workspace,omitempty"` + Shards Shards `json:"_shards"` + Hits Hits `json:"hits"` + Aggregations json.RawMessage `json:"aggregations"` +} + +type Shards struct { + Total int64 `json:"total"` + Successful int64 `json:"successful"` + Skipped int64 `json:"skipped"` + Failed int64 `json:"failed"` +} + +type Hits struct { + // As of ElasticSearch v7.x, hits.total is changed + Total interface{} `json:"total"` + Hits []Hit `json:"hits"` +} + +type Hit struct { + Source Source `json:"_source"` + HighLight HighLight `json:"highlight"` + Sort []int64 `json:"sort"` +} + +type Source struct { + Log string `json:"log"` + Time string `json:"time"` + Kubernetes Kubernetes `json:"kubernetes"` +} + +type Kubernetes struct { + Namespace string `json:"namespace_name"` + Pod string `json:"pod_name"` + Container string `json:"container_name"` + Host string `json:"host"` +} + +type HighLight struct { + LogHighLights []string `json:"log,omitempty" description:"log messages to highlight"` + NamespaceHighLights []string `json:"kubernetes.namespace_name.keyword,omitempty" description:"namespaces to highlight"` + PodHighLights []string `json:"kubernetes.pod_name.keyword,omitempty" description:"pods to highlight"` + ContainerHighLights []string `json:"kubernetes.container_name.keyword,omitempty" description:"containers to highlight"` +} + +type LogRecord struct { + Time int64 `json:"time,omitempty" description:"log timestamp"` + Log string `json:"log,omitempty" description:"log message"` + Namespace string `json:"namespace,omitempty" description:"namespace"` + Pod string `json:"pod,omitempty" description:"pod name"` + Container string `json:"container,omitempty" description:"container name"` + Host string `json:"host,omitempty" description:"node id"` + HighLight HighLight `json:"highlight,omitempty" description:"highlighted log fragment"` +} + +type ReadResult struct { + Total int64 `json:"total" description:"total number of matched results"` + From int64 `json:"from" description:"the offset from the result set"` + Size int64 `json:"size" description:"the amount of hits to be returned"` + Records []LogRecord `json:"records,omitempty" description:"actual array of results"` +} + +// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs +type StatisticsResponseAggregations struct { + ContainerCount ContainerCount `json:"containers"` +} + +type ContainerCount struct { + Value int64 `json:"value"` +} + +type HistogramAggregations struct { + HistogramAggregation HistogramAggregation `json:"histogram"` +} + +type HistogramAggregation struct { + Histograms []HistogramStatistics `json:"buckets"` +} + +type HistogramStatistics struct { + Time int64 `json:"key"` + Count int64 `json:"doc_count"` +} + +type HistogramRecord struct { + Time int64 `json:"time" description:"timestamp"` + Count int64 `json:"count" description:"total number of logs at intervals"` +} + +type StatisticsResult struct { + Containers int64 `json:"containers" description:"total number of containers"` + Logs int64 `json:"logs" description:"total number of logs"` +} + +type HistogramResult struct { + Total int64 `json:"total" description:"total number of logs"` + StartTime int64 `json:"start_time" description:"start time"` + EndTime int64 `json:"end_time" description:"end time"` + Interval string `json:"interval" description:"interval"` + Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"` +} + +// Wrap elasticsearch response +type QueryResult struct { + Status int `json:"status,omitempty" description:"query status"` + Error string `json:"error,omitempty" description:"debugging information"` + Read *ReadResult `json:"query,omitempty" description:"query results"` + Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"` + Histogram *HistogramResult `json:"histogram,omitempty" description:"histogram results"` +}