admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

logging: integrate new IAM

Browse Source 
Signed-off-by: huanggze <loganhuang@yunify.com>
This commit is contained in:
huanggze
2020-05-21 11:16:49 +08:00
parent 48db63b91f
commit 9b6f38d857
36 changed files with 1735 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
pkg/simple/client/logging/elasticsearch/api_body.go
View File

@@ -2,6 +2,7 @@ package elasticsearch
import (
"fmt"
"github.com/json-iterator/go"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"time"
@@ -13,6 +14,9 @@ const (
replicaSetSuffixMaxLength = 11 // max 10 characters + 1 hyphen
)
// TODO: elastic/go-elasticsearch is working on Query DSL support.
// See https://github.com/elastic/go-elasticsearch/issues/42.
// We need refactor our query body builder when that is ready.
type bodyBuilder struct {
Body
}
@@ -22,75 +26,9 @@ func newBodyBuilder() *bodyBuilder {
}
func (bb *bodyBuilder) bytes() ([]byte, error) {
return json.Marshal(bb.Body)
return jsoniter.Marshal(bb.Body)
}
// The mainBody func builds api body for query.
// TODO: Should use an elegant pakcage for building query body, but `elastic/go-elasticsearch` doesn't provide it currently.
//
// Example:
// GET kapis/logging.kubesphere.io/v1alpha2/cluster?start_time=0&end_time=156576063993&namespaces=kubesphere-system&pod_query=ks-apiserver
// -----
//{
// "from":0,
// "size":10,
// "sort":[
// {
// "time": "desc"
// }
// ],
// "query":{
// "bool":{
// "filter":[
// {
// "bool":{
// "should":[
// {
// "bool":{
// "filter":[
// {
// "match_phrase":{
// "kubernetes.namespace_name.keyword":"kubesphere-system"
// }
// },
// {
// "range":{
// "time":{
// "gte":"1572315987000"
// }
// }
// }
// ]
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "bool":{
// "should":[
// {
// "match_phrase_prefix":{
// "kubernetes.pod_name":"ks-apiserver"
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "range":{
// "time":{
// "gte":"0",
// "lte":"156576063993"
// }
// }
// }
// ]
// }
// }
//}
func (bb *bodyBuilder) mainBool(sf logging.SearchFilter) *bodyBuilder {
var ms []Match
@@ -207,10 +145,6 @@ func (bb *bodyBuilder) cardinalityAggregation() *bodyBuilder {
}
func (bb *bodyBuilder) dateHistogramAggregation(interval string) *bodyBuilder {
if interval == "" {
interval = "15m"
}
bb.Body.Aggs = &Aggs{
DateHistogramAggregation: &DateHistogramAggregation{
&DateHistogram{
@@ -232,12 +166,8 @@ func (bb *bodyBuilder) size(n int64) *bodyBuilder {
return bb
}
func (bb *bodyBuilder) sort(o string) *bodyBuilder {
if o != "asc" {
o = "desc"
}
bb.Sorts = []map[string]string{{"time": o}}
func (bb *bodyBuilder) sort(order string) *bodyBuilder {
bb.Sorts = []map[string]string{{"time": order}}
return bb
}
@@ -268,7 +198,7 @@ func podNameRegexp(workloadName string) string {
func parseResponse(body []byte) (Response, error) {
var res Response
err := json.Unmarshal(body, &res)
err := jsoniter.Unmarshal(body, &res)
if err != nil {
klog.Error(err)
return Response{}, err

137
pkg/simple/client/logging/elasticsearch/api_body_test.go
View File

@@ -1,50 +1,117 @@
package elasticsearch
import (
"fmt"
"github.com/google/go-cmp/cmp"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"reflect"
"testing"
"time"
)
func TestCardinalityAggregation(t *testing.T) {
var test = struct {
description string
searchFilter logging.SearchFilter
expected *bodyBuilder
func TestMainBool(t *testing.T) {
var tests = []struct {
filter logging.SearchFilter
expected string
}{
description: "add cardinality aggregation",
searchFilter: logging.SearchFilter{
LogSearch: []string{"info"},
{
filter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"default": time.Unix(1589981934, 0),
},
},
expected: "api_body_1.json",
},
expected: &bodyBuilder{Body{
Query: &Query{
Bool: Bool{
Filter: []Match{
{
Bool: &Bool{
Should: []Match{
{
MatchPhrasePrefix: map[string]string{"log": "info"},
},
},
MinimumShouldMatch: 1,
},
},
},
},
{
filter: logging.SearchFilter{
WorkloadFilter: []string{"mysql"},
Starttime: time.Unix(1589980934, 0),
Endtime: time.Unix(1589981934, 0),
},
Aggs: &Aggs{
CardinalityAggregation: &CardinalityAggregation{
Cardinality: &Cardinality{Field: "kubernetes.docker_id.keyword"},
},
expected: "api_body_2.json",
},
{
filter: logging.SearchFilter{
PodFilter: []string{"mysql"},
PodSearch: []string{"mysql-a8w3s-10945j"},
LogSearch: []string{"info"},
},
}},
expected: "api_body_3.json",
},
{
filter: logging.SearchFilter{
ContainerFilter: []string{"mysql-1"},
ContainerSearch: []string{"mysql-3"},
},
expected: "api_body_4.json",
},
}
t.Run(test.description, func(t *testing.T) {
body := newBodyBuilder().mainBool(test.searchFilter).cardinalityAggregation()
if diff := cmp.Diff(body, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().mainBool(test.filter).Body
if diff := cmp.Diff(result, expected); diff != "" {
fmt.Printf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func TestCardinalityAggregation(t *testing.T) {
var tests = []struct {
expected string
}{
{
expected: "api_body_5.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().cardinalityAggregation().Body
if !reflect.DeepEqual(result, expected) {
t.Fatalf("expected: %v, but got %v", expected, result)
}
})
}
}
func TestDateHistogramAggregation(t *testing.T) {
var tests = []struct {
expected string
}{
{
expected: "api_body_6.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().dateHistogramAggregation("15m").Body
if !reflect.DeepEqual(result, expected) {
t.Fatalf("expected: %v, but got %v", expected, result)
}
})
}
}

73
pkg/simple/client/logging/elasticsearch/elasticsearch.go
View File

@@ -4,12 +4,12 @@ import (
"bytes"
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/json-iterator/go"
"io"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/utils/stringutils"
"strings"
)
@@ -20,8 +20,6 @@ const (
ElasticV7 = "7"
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
// Elasticsearch implement logging interface
type Elasticsearch struct {
c client
@@ -29,8 +27,7 @@ type Elasticsearch struct {
// versioned es client interface
type client interface {
// Perform Search API
Search(body []byte) ([]byte, error)
Search(body []byte, scroll bool) ([]byte, error)
Scroll(id string) ([]byte, error)
ClearScroll(id string)
GetTotalHitCount(v interface{}) int64
@@ -83,11 +80,10 @@ func detectVersionMajor(host string) (string, error) {
if err != nil {
return "", err
}
defer res.Body.Close()
var b map[string]interface{}
if err = json.NewDecoder(res.Body).Decode(&b); err != nil {
if err = jsoniter.NewDecoder(res.Body).Decode(&b); err != nil {
return "", err
}
if res.IsError() {
@@ -116,7 +112,7 @@ func (es Elasticsearch) GetCurrentStats(sf logging.SearchFilter) (logging.Statis
return logging.Statistics{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, true)
if err != nil {
return logging.Statistics{}, err
}
@@ -142,7 +138,7 @@ func (es Elasticsearch) CountLogsByInterval(sf logging.SearchFilter, interval st
return logging.Histogram{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, false)
if err != nil {
return logging.Histogram{}, err
}
@@ -174,7 +170,7 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
return logging.Logs{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, false)
if err != nil {
return logging.Logs{}, err
}
@@ -200,33 +196,50 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
var id string
var from int64 = 0
var size int64 = 1000
var data []string
res, err := es.SearchLogs(sf, from, size, "desc")
defer es.ClearScroll(id)
// Initial Search
body, err := newBodyBuilder().
mainBool(sf).
from(0).
size(1000).
sort("desc").
bytes()
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
b, err := es.c.Search(body, true)
defer es.ClearScroll(id)
if err != nil {
return err
}
res, err := parseResponse(b)
if err != nil {
return err
}
id = res.ScrollId
for _, hit := range res.AllHits {
data = append(data, hit.Log)
}
if len(data) == 0 {
return nil
}
// limit to retrieve max 100k records
for i := 0; i < 100; i++ {
res, id, err = es.scroll(id)
data, id, err = es.scroll(id)
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
if len(data) == 0 {
return nil
}
output := new(bytes.Buffer)
for _, r := range res.Records {
output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(r.Log)))
for _, l := range data {
output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(l)))
}
_, err = io.Copy(w, output)
if err != nil {
@@ -236,24 +249,22 @@ func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
return nil
}
func (es *Elasticsearch) scroll(id string) (logging.Logs, string, error) {
func (es *Elasticsearch) scroll(id string) ([]string, string, error) {
b, err := es.c.Scroll(id)
if err != nil {
return logging.Logs{}, id, err
return nil, id, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Logs{}, id, err
return nil, id, err
}
var l logging.Logs
var data []string
for _, hit := range res.AllHits {
l.Records = append(l.Records, logging.Record{
Log: hit.Log,
})
data = append(data, hit.Log)
}
return l, res.ScrollId, nil
return data, res.ScrollId, nil
}
func (es *Elasticsearch) ClearScroll(id string) {

515
pkg/simple/client/logging/elasticsearch/elasticsearch_test.go
View File

@@ -1,70 +1,46 @@
package elasticsearch
import (
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/json-iterator/go"
"io/ioutil"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func MockElasticsearchService(pattern string, fakeResp string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
res.Write([]byte(fakeResp))
})
return httptest.NewServer(mux)
}
func TestDetectVersionMajor(t *testing.T) {
var tests = []struct {
description string
fakeResp string
expected string
expectedError bool
fakeResp string
expected string
}{
{
description: "detect es 6.x version number",
fakeResp: `{
"name" : "elasticsearch-logging-data-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
"version" : {
"number" : "6.7.0",
"build_flavor" : "oss",
"build_type" : "docker",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}`,
expected: ElasticV6,
expectedError: false,
fakeResp: "es6_detect_version_major_200.json",
expected: ElasticV6,
},
{
fakeResp: "es7_detect_version_major_200.json",
expected: ElasticV7,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
es := mockElasticsearchService("/", test.fakeResp, http.StatusOK)
defer es.Close()
v, err := detectVersionMajor(es.URL)
if err == nil && test.expectedError {
t.Fatalf("expected error while got nothing")
} else if err != nil && !test.expectedError {
result, err := detectVersionMajor(es.URL)
if err != nil {
t.Fatal(err)
}
if v != test.expected {
t.Fatalf("expected get version %s, but got %s", test.expected, v)
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
@@ -72,297 +48,202 @@ func TestDetectVersionMajor(t *testing.T) {
func TestGetCurrentStats(t *testing.T) {
var tests = []struct {
description string
searchFilter logging.SearchFilter
fakeVersion string
fakeResp string
expected logging.Statistics
expectedError bool
fakeVersion string
fakeResp string
fakeCode int
expected logging.Statistics
expectedErr string
}{
{
description: "[es 6.x] run as admin",
searchFilter: logging.SearchFilter{},
fakeVersion: ElasticV6,
fakeResp: `{
"took": 171,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 241222,
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Hn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:29.015Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:29.015492329Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "I31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.103Z",
"log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
"time": "2020-02-28T19:25:33.103075891Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "JX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.113Z",
"log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
"time": "2020-02-28T19:25:33.113110332Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Kn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:34.168983384Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "LH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:25:34.168997393Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "NX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:42.868Z",
"log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n",
"time": "2020-02-28T19:25:42.868578188Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Q31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:13.881Z",
"log": "- name: BasePath\n",
"time": "2020-02-28T19:26:13.881180681Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "S31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:26:14.597702238Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "TH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
"time": "2020-02-28T19:26:14.597708432Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "UX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:15.920Z",
"log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
"time": "2020-02-28T19:26:15.920650572Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 93
}
}
}`,
fakeVersion: ElasticV6,
fakeResp: "es6_get_current_stats_200.json",
fakeCode: http.StatusOK,
expected: logging.Statistics{
Containers: 93,
Logs: 241222,
},
expectedError: false,
},
{
description: "[es 6.x] index not found",
searchFilter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"workspace-1-project-a": time.Unix(1582000000, 0),
"workspace-1-project-b": time.Unix(1582333333, 0),
},
},
fakeVersion: ElasticV6,
fakeResp: `{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
}
],
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
},
"status": 404
}`,
fakeResp: "es6_get_current_stats_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index",
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_get_current_stats_200.json",
fakeCode: http.StatusOK,
expected: logging.Statistics{
Containers: 0,
Logs: 0,
Containers: 48,
Logs: 9726,
},
expectedError: true,
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_get_current_stats_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-2020.05.2]",
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
defer es.Close()
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
clientv5 := Elasticsearch{c: v5.New(es.URL, "ks-logstash-log")}
clientv6 := Elasticsearch{c: v6.New(es.URL, "ks-logstash-log")}
clientv7 := Elasticsearch{c: v7.New(es.URL, "ks-logstash-log")}
es := newElasticsearchClient(srv, test.fakeVersion)
var stats logging.Statistics
var err error
switch test.fakeVersion {
case ElasticV5:
stats, err = clientv5.GetCurrentStats(test.searchFilter)
case ElasticV6:
stats, err = clientv6.GetCurrentStats(test.searchFilter)
case ElasticV7:
stats, err = clientv7.GetCurrentStats(test.searchFilter)
result, err := es.GetCurrentStats(logging.SearchFilter{})
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if err != nil && !test.expectedError {
t.Fatal(err)
} else if diff := cmp.Diff(stats, test.expected); diff != "" {
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}
func TestCountLogsByInterval(t *testing.T) {
var tests = []struct {
fakeVersion string
fakeResp string
fakeCode int
expected logging.Histogram
expectedErr string
}{
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_200.json",
fakeCode: http.StatusOK,
expected: logging.Histogram{
Total: 10000,
Buckets: []logging.Bucket{
{
Time: 1589644800000,
Count: 410,
},
{
Time: 1589646600000,
Count: 7465,
},
{
Time: 1589648400000,
Count: 12790,
},
},
},
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_400.json",
fakeCode: http.StatusBadRequest,
expectedErr: "type: search_phase_execution_exception, reason: all shards failed",
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-20]",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
es := newElasticsearchClient(srv, test.fakeVersion)
result, err := es.CountLogsByInterval(logging.SearchFilter{}, "15m")
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}
func TestSearchLogs(t *testing.T) {
var tests = []struct {
fakeVersion string
fakeResp string
fakeCode int
expected string
expectedErr string
}{
{
fakeVersion: ElasticV7,
fakeResp: "es7_search_logs_200.json",
fakeCode: http.StatusOK,
expected: "es7_search_logs_200_result.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected logging.Logs
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
es := newElasticsearchClient(srv, test.fakeVersion)
result, err := es.SearchLogs(logging.SearchFilter{}, 0, 10, "asc")
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func mockElasticsearchService(pattern, fakeResp string, fakeCode int) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
b, _ := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", fakeResp))
res.WriteHeader(fakeCode)
res.Write(b)
})
return httptest.NewServer(mux)
}
func newElasticsearchClient(srv *httptest.Server, version string) Elasticsearch {
var es Elasticsearch
switch version {
case ElasticV5:
es = Elasticsearch{c: v5.New(srv.URL, "ks-logstash-log")}
case ElasticV6:
es = Elasticsearch{c: v6.New(srv.URL, "ks-logstash-log")}
case ElasticV7:
es = Elasticsearch{c: v7.New(srv.URL, "ks-logstash-log")}
}
return es
}
func JsonFromFile(expectedFile string, expectedJsonPtr interface{}) error {
json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile))
if err != nil {
return err
}
err = jsoniter.Unmarshal(json, expectedJsonPtr)
if err != nil {
return err
}
return nil
}

33
pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json vendored Normal file
View File

@@ -0,0 +1,33 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"bool":{
"filter":[
{
"match_phrase":{
"kubernetes.namespace_name.keyword":"default"
}
},
{
"range":{
"time":{
"gte":"2020-05-20T21:38:54+08:00"
}
}
}
]
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

28
pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json vendored Normal file
View File

@@ -0,0 +1,28 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"regexp":{
"kubernetes.pod_name.keyword":"mysql-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|mysql-[0-9]+|mysql-[a-z0-9]{5}"
}
}
],
"minimum_should_match":1
}
},
{
"range":{
"time":{
"gte":"2020-05-20T21:22:14+08:00",
"lte":"2020-05-20T21:38:54+08:00"
}
}
}
]
}
}
}

44
pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json vendored Normal file
View File

@@ -0,0 +1,44 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"match_phrase":{
"kubernetes.pod_name.keyword":"mysql"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"kubernetes.pod_name":"mysql-a8w3s-10945j"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"log":"info"
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

32
pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json vendored Normal file
View File

@@ -0,0 +1,32 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"match_phrase":{
"kubernetes.container_name.keyword":"mysql-1"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"kubernetes.container_name":"mysql-3"
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

9
pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json vendored Normal file
View File

@@ -0,0 +1,9 @@
{
"aggs":{
"container_count":{
"cardinality":{
"field":"kubernetes.docker_id.keyword"
}
}
}
}

10
pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json vendored Normal file
View File

@@ -0,0 +1,10 @@
{
"aggs":{
"log_count_over_time":{
"date_histogram":{
"field":"time",
"interval":"15m"
}
}
}
}

17
pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"name" : "elasticsearch-logging-data-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
"version" : {
"number" : "6.7.0",
"build_flavor" : "oss",
"build_type" : "docker",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}

211
pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json vendored Normal file
View File

@@ -0,0 +1,211 @@
{
"took": 171,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 241222,
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Hn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:29.015Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:29.015492329Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "I31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.103Z",
"log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
"time": "2020-02-28T19:25:33.103075891Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "JX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.113Z",
"log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
"time": "2020-02-28T19:25:33.113110332Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Kn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:34.168983384Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "LH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:25:34.168997393Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "NX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:42.868Z",
"log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n",
"time": "2020-02-28T19:25:42.868578188Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Q31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:13.881Z",
"log": "- name: BasePath\n",
"time": "2020-02-28T19:26:13.881180681Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "S31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:26:14.597702238Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "TH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
"time": "2020-02-28T19:26:14.597708432Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "UX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:15.920Z",
"log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
"time": "2020-02-28T19:26:15.920650572Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 93
}
}
}

21
pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
}
],
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
},
"status": 404
}

230
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json vendored Normal file
View File

@@ -0,0 +1,230 @@
{
"took": 23,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "jxt2MXIBlcWZ594bpIVN",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:07.648Z",
"log": "ls: cannot access '/calico-secrets': No such file or directory\n",
"time": "2020-05-16T16:00:07.64848716Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.164Z",
"log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
"time": "2020-05-16T16:00:08.164003996Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "CNI plugin version: v3.7.3\n",
"time": "2020-05-16T16:00:08.350585959Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kht2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "/host/secondary-bin-dir is non-writeable, skipping\n",
"time": "2020-05-16T16:00:08.350625112Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kxt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
"time": "2020-05-16T16:00:08.350692011Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": "CNI config: {\n",
"time": "2020-05-16T16:00:08.454417144Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": " \"name\": \"cni0\",\n",
"time": "2020-05-16T16:00:08.454452649Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
}
]
},
"aggregations": {
"log_count_over_time": {
"buckets": [
{
"key_as_string": "2020-05-16T16:00:00.000Z",
"key": 1589644800000,
"doc_count": 410
},
{
"key_as_string": "2020-05-16T16:30:00.000Z",
"key": 1589646600000,
"doc_count": 7465
},
{
"key_as_string": "2020-05-16T17:00:00.000Z",
"key": 1589648400000,
"doc_count": 12790
}
]
}
}
}

47
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json vendored Normal file
View File

@@ -0,0 +1,47 @@
{
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
},
{
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "ks-logstash-log-2020.05.16",
"node": "Zr2OFlfeSJmK_W3Re4UBlg",
"reason": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
},
{
"shard": 0,
"index": "ks-logstash-log-2020.05.20",
"node": "pbGNYbV3QUuV5yJAgxbp3g",
"reason": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
}
],
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
}
},
"status": 400
}

21
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-20]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-20",
"index_uuid": "_na_",
"index": "ks-logstash-log-20"
}
],
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-20]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-20",
"index_uuid": "_na_",
"index": "ks-logstash-log-20"
},
"status": 404
}

17
pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"name" : "elasticsearch-master-2",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "_A-3shR0R0i-2M9CzOWP8g",
"version" : {
"number" : "7.7.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "81a1e9eda8e6183f5237786246f6dced26a10eaf",
"build_date" : "2020-05-12T02:01:37.602180Z",
"build_snapshot" : false,
"lucene_version" : "8.5.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}

214
pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json vendored Normal file
View File

@@ -0,0 +1,214 @@
{
"took": 1455,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 9726,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "jxt2MXIBlcWZ594bpIVN",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:07.648Z",
"log": "ls: cannot access '/calico-secrets': No such file or directory\n",
"time": "2020-05-16T16:00:07.64848716Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.164Z",
"log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
"time": "2020-05-16T16:00:08.164003996Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "CNI plugin version: v3.7.3\n",
"time": "2020-05-16T16:00:08.350585959Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kht2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "/host/secondary-bin-dir is non-writeable, skipping\n",
"time": "2020-05-16T16:00:08.350625112Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kxt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
"time": "2020-05-16T16:00:08.350692011Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": "CNI config: {\n",
"time": "2020-05-16T16:00:08.454417144Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": " \"name\": \"cni0\",\n",
"time": "2020-05-16T16:00:08.454452649Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 48
}
}
}

21
pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-2020.05.2]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-2020.05.2",
"index_uuid": "_na_",
"index": "ks-logstash-log-2020.05.2"
}
],
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-2020.05.2]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-2020.05.2",
"index_uuid": "_na_",
"index": "ks-logstash-log-2020.05.2"
},
"status": 404
}

76
pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json vendored Normal file
View File

@@ -0,0 +1,76 @@
{
"took": 772,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "istio-system",
"host": "node0",
"container_name": "mixer",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
}
]
}
}

29
pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json vendored Normal file
View File

@@ -0,0 +1,29 @@
{
"total": 10000,
"records": [
{
"time": "2020-05-16T16:00:42.608962452Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"namespace": "kubesphere-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "config-init",
"host": "master0"
},
{
"time": "2020-05-16T16:00:42.670430525Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"namespace": "kubesphere-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "config-init",
"host": "master0"
},
{
"time": "2020-05-16T16:00:42.731865428Z",
"log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
"namespace": "istio-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "mixer",
"host": "node0"
}
]
}

11
pkg/simple/client/logging/elasticsearch/versions/v5/v5.go
View File

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.client.Search.WithScroll(time.Minute))
}
response, err := e.client.Search(opts...)
if err != nil {
return nil, err
}

11
pkg/simple/client/logging/elasticsearch/versions/v6/v6.go
View File

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{Client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.Client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.Client.Search.WithContext(context.Background()),
e.Client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.Client.Search.WithBody(bytes.NewBuffer(body)),
e.Client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.Client.Search.WithScroll(time.Minute))
}
response, err := e.Client.Search(opts...)
if err != nil {
return nil, err
}

12
pkg/simple/client/logging/elasticsearch/versions/v7/v7.go
View File

@@ -29,13 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithTrackTotalHits(true),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.client.Search.WithScroll(time.Minute))
}
response, err := e.client.Search(opts...)
if err != nil {
return nil, err
}

4
pkg/simple/client/logging/interface.go
View File

@@ -6,13 +6,9 @@ import (
)
type Interface interface {
// Current stats about log store, eg. total number of logs and containers
GetCurrentStats(sf SearchFilter) (Statistics, error)
CountLogsByInterval(sf SearchFilter, interval string) (Histogram, error)
SearchLogs(sf SearchFilter, from, size int64, order string) (Logs, error)
ExportLogs(sf SearchFilter, w io.Writer) error
}