admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[WIP] logging refactor (#1794)

Browse Source 
* refactor logging

Signed-off-by: huanggze <loganhuang@yunify.com>

* refactor logging

Signed-off-by: huanggze <loganhuang@yunify.com>
This commit is contained in:
Guangzhe Huang
2020-03-02 10:53:43 +08:00
committed by GitHub
parent a9e1183f3c
commit 6c6bfb2677
60 changed files with 1582 additions and 2966 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
pkg/simple/client/logging/elasticsearch/api_body.go Normal file
View File

@@ -0,0 +1,277 @@
package elasticsearch
import (
"fmt"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"time"
)
const (
podNameMaxLength = 63
podNameSuffixLength = 6 // 5 characters + 1 hyphen
replicaSetSuffixMaxLength = 11 // max 10 characters + 1 hyphen
)
type bodyBuilder struct {
Body
}
func newBodyBuilder() *bodyBuilder {
return &bodyBuilder{}
}
func (bb *bodyBuilder) bytes() ([]byte, error) {
return json.Marshal(bb.Body)
}
// The mainBody func builds api body for query.
// TODO: Should use an elegant pakcage for building query body, but `elastic/go-elasticsearch` doesn't provide it currently.
//
// Example:
// GET kapis/logging.kubesphere.io/v1alpha2/cluster?start_time=0&end_time=156576063993&namespaces=kubesphere-system&pod_query=ks-apiserver
// -----
//{
// "from":0,
// "size":10,
// "sort":[
// {
// "time": "desc"
// }
// ],
// "query":{
// "bool":{
// "filter":[
// {
// "bool":{
// "should":[
// {
// "bool":{
// "filter":[
// {
// "match_phrase":{
// "kubernetes.namespace_name.keyword":"kubesphere-system"
// }
// },
// {
// "range":{
// "time":{
// "gte":"1572315987000"
// }
// }
// }
// ]
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "bool":{
// "should":[
// {
// "match_phrase_prefix":{
// "kubernetes.pod_name":"ks-apiserver"
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "range":{
// "time":{
// "gte":"0",
// "lte":"156576063993"
// }
// }
// }
// ]
// }
// }
//}
func (bb *bodyBuilder) mainBool(sf logging.SearchFilter) *bodyBuilder {
var ms []Match
// literal matching
if len(sf.NamespaceFilter) != 0 {
var b Bool
for ns := range sf.NamespaceFilter {
match := Match{
Bool: &Bool{
Filter: []Match{
{
MatchPhrase: map[string]string{
"kubernetes.namespace_name.keyword": ns,
},
},
{
Range: &Range{
Time: &Time{
Gte: func() *time.Time { t := sf.NamespaceFilter[ns]; return &t }(),
},
},
},
},
},
}
b.Should = append(b.Should, match)
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.WorkloadFilter != nil {
var b Bool
for _, wk := range sf.WorkloadFilter {
b.Should = append(b.Should, Match{Regexp: map[string]string{"kubernetes.pod_name.keyword": podNameRegexp(wk)}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.PodFilter != nil {
var b Bool
for _, po := range sf.PodFilter {
b.Should = append(b.Should, Match{MatchPhrase: map[string]string{"kubernetes.pod_name.keyword": po}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.ContainerFilter != nil {
var b Bool
for _, c := range sf.ContainerFilter {
b.Should = append(b.Should, Match{MatchPhrase: map[string]string{"kubernetes.container_name.keyword": c}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
// fuzzy matching
if sf.WorkloadSearch != nil {
var b Bool
for _, wk := range sf.WorkloadSearch {
b.Should = append(b.Should, Match{MatchPhrasePrefix: map[string]string{"kubernetes.pod_name": wk}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.PodSearch != nil {
var b Bool
for _, po := range sf.PodSearch {
b.Should = append(b.Should, Match{MatchPhrasePrefix: map[string]string{"kubernetes.pod_name": po}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.ContainerSearch != nil {
var b Bool
for _, c := range sf.ContainerSearch {
b.Should = append(b.Should, Match{MatchPhrasePrefix: map[string]string{"kubernetes.container_name": c}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if sf.LogSearch != nil {
var b Bool
for _, l := range sf.LogSearch {
b.Should = append(b.Should, Match{MatchPhrasePrefix: map[string]string{"log": l}})
}
b.MinimumShouldMatch = 1
ms = append(ms, Match{Bool: &b})
}
if !sf.Starttime.IsZero() || !sf.Endtime.IsZero() {
fromTo := Match{
Range: &Range{&Time{
Gte: &sf.Starttime,
Lte: &sf.Endtime,
}},
}
ms = append(ms, fromTo)
}
bb.Body.Query = &Query{Bool{Filter: ms}}
return bb
}
func (bb *bodyBuilder) cardinalityAggregation() *bodyBuilder {
bb.Body.Aggs = &Aggs{
CardinalityAggregation: &CardinalityAggregation{
&Cardinality{
Field: "kubernetes.docker_id.keyword",
},
},
}
return bb
}
func (bb *bodyBuilder) dateHistogramAggregation(interval string) *bodyBuilder {
if interval == "" {
interval = "15m"
}
bb.Body.Aggs = &Aggs{
DateHistogramAggregation: &DateHistogramAggregation{
&DateHistogram{
Field: "time",
Interval: interval,
},
},
}
return bb
}
func (bb *bodyBuilder) from(n int64) *bodyBuilder {
bb.From = n
return bb
}
func (bb *bodyBuilder) size(n int64) *bodyBuilder {
bb.Size = n
return bb
}
func (bb *bodyBuilder) sort(o string) *bodyBuilder {
if o != "asc" {
o = "desc"
}
bb.Sorts = []map[string]string{{"time": o}}
return bb
}
func podNameRegexp(workloadName string) string {
var regexp string
if len(workloadName) <= podNameMaxLength-replicaSetSuffixMaxLength-podNameSuffixLength {
// match deployment pods, eg. <deploy>-579dfbcddd-24znw
// replicaset rand string is limited to vowels
// https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/util/rand/rand.go#L83
regexp += workloadName + "-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|"
// match statefulset pods, eg. <sts>-0
regexp += workloadName + "-[0-9]+|"
// match pods of daemonset or job, eg. <ds>-29tdk, <job>-5xqvl
regexp += workloadName + "-[a-z0-9]{5}"
} else if len(workloadName) <= podNameMaxLength-podNameSuffixLength {
replicaSetSuffixLength := podNameMaxLength - podNameSuffixLength - len(workloadName)
regexp += fmt.Sprintf("%s%d%s", workloadName+"-[bcdfghjklmnpqrstvwxz2456789]{", replicaSetSuffixLength, "}[a-z0-9]{5}|")
regexp += workloadName + "-[0-9]+|"
regexp += workloadName + "-[a-z0-9]{5}"
} else {
// Rand suffix may overwrites the workload name if the name is too long
// This won't happen for StatefulSet because long name will cause ReplicaSet fails during StatefulSet creation.
regexp += workloadName[:podNameMaxLength-podNameSuffixLength+1] + "[a-z0-9]{5}|"
regexp += workloadName + "-[0-9]+"
}
return regexp
}
func parseResponse(body []byte) (Response, error) {
var res Response
err := json.Unmarshal(body, &res)
if err != nil {
klog.Error(err)
return Response{}, err
}
return res, nil
}

122
pkg/simple/client/logging/elasticsearch/api_body_test.go Normal file
View File

@@ -0,0 +1,122 @@
package elasticsearch
import (
"github.com/google/go-cmp/cmp"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"testing"
"time"
)
func TestMainBool(t *testing.T) {
var tests = []struct {
description string
searchFilter logging.SearchFilter
expected *bodyBuilder
}{
{
description: "filter 2 namespaces",
searchFilter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"kubesphere-system": time.Unix(1582000000, 0),
"kubesphere-logging-system": time.Unix(1582969999, 0),
},
},
expected: &bodyBuilder{Body{
Query: &Query{
Bool: Bool{
Filter: []Match{
{
Bool: &Bool{
Should: []Match{
{
Bool: &Bool{
Filter: []Match{
{
MatchPhrase: map[string]string{"kubernetes.namespace_name.keyword": "kubesphere-system"},
},
{
Range: &Range{&Time{Gte: func() *time.Time { t := time.Unix(1582000000, 0); return &t }()}},
},
},
},
},
{
Bool: &Bool{
Filter: []Match{
{
MatchPhrase: map[string]string{"kubernetes.namespace_name.keyword": "kubesphere-logging-system"},
},
{
Range: &Range{&Time{Gte: func() *time.Time { t := time.Unix(1582969999, 0); return &t }()}},
},
},
},
},
},
MinimumShouldMatch: 1,
},
},
},
},
},
}},
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
body, err := newBodyBuilder().mainBool(test.searchFilter).bytes()
expected, _ := test.expected.bytes()
if err != nil {
t.Fatal(err)
}
if diff := cmp.Diff(body, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func TestCardinalityAggregation(t *testing.T) {
var test = struct {
description string
searchFilter logging.SearchFilter
expected *bodyBuilder
}{
description: "add cardinality aggregation",
searchFilter: logging.SearchFilter{
LogSearch: []string{"info"},
},
expected: &bodyBuilder{Body{
Query: &Query{
Bool: Bool{
Filter: []Match{
{
Bool: &Bool{
Should: []Match{
{
MatchPhrasePrefix: map[string]string{"log": "info"},
},
},
MinimumShouldMatch: 1,
},
},
},
},
},
Aggs: &Aggs{
CardinalityAggregation: &CardinalityAggregation{
Cardinality: &Cardinality{Field: "kubernetes.docker_id.keyword"},
},
},
}},
}
t.Run(test.description, func(t *testing.T) {
body := newBodyBuilder().mainBool(test.searchFilter).cardinalityAggregation()
if diff := cmp.Diff(body, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}

127
pkg/simple/client/logging/elasticsearch/api_body_types.go Normal file
View File

@@ -0,0 +1,127 @@
package elasticsearch
import "time"
// --------------------------------------------- Request Body ---------------------------------------------
// More info: https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html
type Body struct {
From int64 `json:"from,omitempty"`
Size int64 `json:"size,omitempty"`
Sorts []map[string]string `json:"sort,omitempty"`
*Query `json:"query,omitempty"`
*Aggs `json:"aggs,omitempty"`
}
type Query struct {
Bool `json:"bool,omitempty"`
}
// Example:
// {bool: {filter: <[]Match>}}
// {bool: {should: <[]Match>, minimum_should_match: 1}}
type Bool struct {
Filter []Match `json:"filter,omitempty"`
Should []Match `json:"should,omitempty"`
MinimumShouldMatch int32 `json:"minimum_should_match,omitempty"`
}
// Example: []Match
// [
// {
// bool: <Bool>
// },
// {
// match_phrase: {
// <string>: <string>
// }
// },
// ...
// ]
type Match struct {
*Bool `json:"bool,omitempty"`
MatchPhrase map[string]string `json:"match_phrase,omitempty"`
MatchPhrasePrefix map[string]string `json:"match_phrase_prefix,omitempty"`
Regexp map[string]string `json:"regexp,omitempty"`
*Range `json:"range,omitempty"`
}
type Range struct {
*Time `json:"time,omitempty"`
}
type Time struct {
Gte *time.Time `json:"gte,omitempty"`
Lte *time.Time `json:"lte,omitempty"`
}
type Aggs struct {
*CardinalityAggregation `json:"container_count,omitempty"`
*DateHistogramAggregation `json:"log_count_over_time,omitempty"`
}
type CardinalityAggregation struct {
*Cardinality `json:"cardinality,omitempty"`
}
type Cardinality struct {
Field string `json:"field,omitempty"`
}
type DateHistogramAggregation struct {
*DateHistogram `json:"date_histogram,omitempty"`
}
type DateHistogram struct {
Field string `json:"field,omitempty"`
Interval string `json:"interval,omitempty"`
}
// --------------------------------------------- Response Body ---------------------------------------------
type Response struct {
ScrollId string `json:"_scroll_id,omitempty"`
Hits `json:"hits,omitempty"`
Aggregations `json:"aggregations,omitempty"`
}
type Hits struct {
Total interface{} `json:"total"` // `As of Elasticsearch v7.x, hits.total is changed incompatibly
AllHits []Hit `json:"hits"`
}
type Hit struct {
Source `json:"_source"`
Sort []int64 `json:"sort"`
}
type Source struct {
Log string `json:"log"`
Time string `json:"time"`
Kubernetes `json:"kubernetes"`
}
type Kubernetes struct {
Namespace string `json:"namespace_name"`
Pod string `json:"pod_name"`
Container string `json:"container_name"`
Host string `json:"host"`
}
type Aggregations struct {
ContainerCount `json:"container_count"`
LogCountOverTime `json:"log_count_over_time"`
}
type ContainerCount struct {
Value int64 `json:"value"`
}
type LogCountOverTime struct {
Buckets []Bucket `json:"buckets"`
}
type Bucket struct {
Time int64 `json:"key"`
Count int64 `json:"doc_count"`
}

263
pkg/simple/client/logging/elasticsearch/elasticsearch.go Normal file
View File

@@ -0,0 +1,263 @@
package elasticsearch
import (
"bytes"
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"io"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/utils/stringutils"
"strings"
)
const (
ElasticV5 = "5"
ElasticV6 = "6"
ElasticV7 = "7"
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
// Elasticsearch implement logging interface
type Elasticsearch struct {
c client
}
// versioned es client interface
type client interface {
// Perform Search API
Search(body []byte) ([]byte, error)
Scroll(id string) ([]byte, error)
ClearScroll(id string)
GetTotalHitCount(v interface{}) int64
}
func NewElasticsearch(options *Options) (*Elasticsearch, error) {
var version, index string
es := &Elasticsearch{}
if options.Version == "" {
var err error
version, err = detectVersionMajor(options.Host)
if err != nil {
return nil, err
}
} else {
version = options.Version
}
if options.IndexPrefix != "" {
index = options.IndexPrefix
} else {
index = "logstash"
}
switch version {
case ElasticV5:
es.c = v5.New(options.Host, index)
case ElasticV6:
es.c = v6.New(options.Host, index)
case ElasticV7:
es.c = v7.New(options.Host, index)
default:
return nil, fmt.Errorf("unsupported elasticsearch version %s", version)
}
return es, nil
}
func (es *Elasticsearch) ES() *client {
return &es.c
}
func detectVersionMajor(host string) (string, error) {
// Info APIs are backward compatible with versions of v5.x, v6.x and v7.x
es := v6.New(host, "")
res, err := es.Client.Info(
es.Client.Info.WithContext(context.Background()),
)
if err != nil {
return "", err
}
defer res.Body.Close()
var b map[string]interface{}
if err = json.NewDecoder(res.Body).Decode(&b); err != nil {
return "", err
}
if res.IsError() {
// Print the response status and error information.
e, _ := b["error"].(map[string]interface{})
return "", fmt.Errorf("[%s] type: %v, reason: %v", res.Status(), e["type"], e["reason"])
}
// get the major version
version, _ := b["version"].(map[string]interface{})
number, _ := version["number"].(string)
if number == "" {
return "", fmt.Errorf("failed to detect elastic version number")
}
v := strings.Split(number, ".")[0]
return v, nil
}
func (es Elasticsearch) GetCurrentStats(sf logging.SearchFilter) (logging.Statistics, error) {
body, err := newBodyBuilder().
mainBool(sf).
cardinalityAggregation().
bytes()
if err != nil {
return logging.Statistics{}, err
}
b, err := es.c.Search(body)
if err != nil {
return logging.Statistics{}, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Statistics{}, err
}
return logging.Statistics{
Containers: res.Value,
Logs: es.c.GetTotalHitCount(res.Total),
},
nil
}
func (es Elasticsearch) CountLogsByInterval(sf logging.SearchFilter, interval string) (logging.Histogram, error) {
body, err := newBodyBuilder().
mainBool(sf).
dateHistogramAggregation(interval).
bytes()
if err != nil {
return logging.Histogram{}, err
}
b, err := es.c.Search(body)
if err != nil {
return logging.Histogram{}, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Histogram{}, err
}
var h logging.Histogram
h.Total = es.c.GetTotalHitCount(res.Total)
for _, b := range res.Buckets {
h.Buckets = append(h.Buckets, logging.Bucket{
Time: b.Time,
Count: b.Count,
})
}
return h, nil
}
func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string) (logging.Logs, error) {
body, err := newBodyBuilder().
mainBool(sf).
from(f).
size(s).
sort(o).
bytes()
if err != nil {
return logging.Logs{}, err
}
b, err := es.c.Search(body)
if err != nil {
return logging.Logs{}, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Logs{}, err
}
var l logging.Logs
l.Total = es.c.GetTotalHitCount(res.Total)
for _, hit := range res.AllHits {
l.Records = append(l.Records, logging.Record{
Log: hit.Log,
Time: hit.Time,
Namespace: hit.Namespace,
Pod: hit.Pod,
Container: hit.Container,
})
}
return l, nil
}
func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
var id string
var from int64 = 0
var size int64 = 1000
res, err := es.SearchLogs(sf, from, size, "desc")
defer es.ClearScroll(id)
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
return nil
}
// limit to retrieve max 100k records
for i := 0; i < 100; i++ {
res, id, err = es.scroll(id)
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
return nil
}
output := new(bytes.Buffer)
for _, r := range res.Records {
output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(r.Log)))
}
_, err = io.Copy(w, output)
if err != nil {
return err
}
}
return nil
}
func (es *Elasticsearch) scroll(id string) (logging.Logs, string, error) {
b, err := es.c.Scroll(id)
if err != nil {
return logging.Logs{}, id, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Logs{}, id, err
}
var l logging.Logs
for _, hit := range res.AllHits {
l.Records = append(l.Records, logging.Record{
Log: hit.Log,
})
}
return l, res.ScrollId, nil
}
func (es *Elasticsearch) ClearScroll(id string) {
if id != "" {
es.c.ClearScroll(id)
}
}

368
pkg/simple/client/logging/elasticsearch/elasticsearch_test.go Normal file
View File

@@ -0,0 +1,368 @@
package elasticsearch
import (
"github.com/google/go-cmp/cmp"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func MockElasticsearchService(pattern string, fakeResp string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
res.Write([]byte(fakeResp))
})
return httptest.NewServer(mux)
}
func TestDetectVersionMajor(t *testing.T) {
var tests = []struct {
description string
fakeResp string
expected string
expectedError bool
}{
{
description: "detect es 6.x version number",
fakeResp: `{
"name" : "elasticsearch-logging-data-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
"version" : {
"number" : "6.7.0",
"build_flavor" : "oss",
"build_type" : "docker",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}`,
expected: ElasticV6,
expectedError: false,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
defer es.Close()
v, err := detectVersionMajor(es.URL)
if err == nil && test.expectedError {
t.Fatalf("expected error while got nothing")
} else if err != nil && !test.expectedError {
t.Fatal(err)
}
if v != test.expected {
t.Fatalf("expected get version %s, but got %s", test.expected, v)
}
})
}
}
func TestGetCurrentStats(t *testing.T) {
var tests = []struct {
description string
searchFilter logging.SearchFilter
fakeVersion string
fakeResp string
expected logging.Statistics
expectedError bool
}{
{
description: "[es 6.x] run as admin",
searchFilter: logging.SearchFilter{},
fakeVersion: ElasticV6,
fakeResp: `{
"took": 171,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 241222,
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Hn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:29.015Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:29.015492329Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "I31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.103Z",
"log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
"time": "2020-02-28T19:25:33.103075891Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "JX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.113Z",
"log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
"time": "2020-02-28T19:25:33.113110332Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Kn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:34.168983384Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "LH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:25:34.168997393Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "NX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:42.868Z",
"log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n",
"time": "2020-02-28T19:25:42.868578188Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Q31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:13.881Z",
"log": "- name: BasePath\n",
"time": "2020-02-28T19:26:13.881180681Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "S31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:26:14.597702238Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "TH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
"time": "2020-02-28T19:26:14.597708432Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "UX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:15.920Z",
"log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
"time": "2020-02-28T19:26:15.920650572Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 93
}
}
}`,
expected: logging.Statistics{
Containers: 93,
Logs: 241222,
},
expectedError: false,
},
{
description: "[es 6.x] index not found",
searchFilter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"workspace-1-project-a": time.Unix(1582000000, 0),
"workspace-1-project-b": time.Unix(1582333333, 0),
},
},
fakeVersion: ElasticV6,
fakeResp: `{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
}
],
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
},
"status": 404
}`,
expected: logging.Statistics{
Containers: 0,
Logs: 0,
},
expectedError: true,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
defer es.Close()
clientv5 := Elasticsearch{c: v5.New(es.URL, "ks-logstash-log")}
clientv6 := Elasticsearch{c: v6.New(es.URL, "ks-logstash-log")}
clientv7 := Elasticsearch{c: v7.New(es.URL, "ks-logstash-log")}
var stats logging.Statistics
var err error
switch test.fakeVersion {
case ElasticV5:
stats, err = clientv5.GetCurrentStats(test.searchFilter)
case ElasticV6:
stats, err = clientv6.GetCurrentStats(test.searchFilter)
case ElasticV7:
stats, err = clientv7.GetCurrentStats(test.searchFilter)
}
if err != nil && !test.expectedError {
t.Fatal(err)
} else if diff := cmp.Diff(stats, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}

46
pkg/simple/client/logging/elasticsearch/options.go Normal file
View File

@@ -0,0 +1,46 @@
package elasticsearch
import (
"github.com/spf13/pflag"
"kubesphere.io/kubesphere/pkg/utils/reflectutils"
)
type Options struct {
Host string `json:"host" yaml:"host"`
IndexPrefix string `json:"indexPrefix,omitempty" yaml:"indexPrefix"`
Version string `json:"version" yaml:"version"`
}
func NewElasticSearchOptions() *Options {
return &Options{
Host: "",
IndexPrefix: "fluentbit",
Version: "",
}
}
func (s *Options) ApplyTo(options *Options) {
if s.Host != "" {
reflectutils.Override(options, s)
}
}
func (s *Options) Validate() []error {
errs := []error{}
return errs
}
func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
fs.StringVar(&s.Host, "elasticsearch-host", c.Host, ""+
"Elasticsearch logging service host. KubeSphere is using elastic as log store, "+
"if this filed left blank, KubeSphere will use kubernetes builtin log API instead, and"+
" the following elastic search options will be ignored.")
fs.StringVar(&s.IndexPrefix, "index-prefix", c.IndexPrefix, ""+
"Index name prefix. KubeSphere will retrieve logs against indices matching the prefix.")
fs.StringVar(&s.Version, "elasticsearch-version", c.Version, ""+
"Elasticsearch major version, e.g. 5/6/7, if left blank, will detect automatically."+
"Currently, minimum supported version is 5.x")
}

88
pkg/simple/client/logging/elasticsearch/versions/v5/v5.go Normal file
View File

@@ -0,0 +1,88 @@
package v5
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v5"
"github.com/elastic/go-elasticsearch/v5/esapi"
"io/ioutil"
"k8s.io/klog"
"time"
)
type Elastic struct {
client *elasticsearch.Client
index string
}
func New(address string, index string) *Elastic {
client, err := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
if err != nil {
klog.Error(err)
return nil
}
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
return ioutil.ReadAll(response.Body)
}
func (e *Elastic) Scroll(id string) ([]byte, error) {
response, err := e.client.Scroll(
e.client.Scroll.WithContext(context.Background()),
e.client.Scroll.WithScrollID(id),
e.client.Scroll.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
return ioutil.ReadAll(response.Body)
}
func (e *Elastic) ClearScroll(scrollId string) {
response, _ := e.client.ClearScroll(
e.client.ClearScroll.WithContext(context.Background()),
e.client.ClearScroll.WithScrollID(scrollId))
defer response.Body.Close()
}
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
f, _ := v.(float64)
return int64(f)
}
func parseError(response *esapi.Response) error {
var e map[string]interface{}
if err := json.NewDecoder(response.Body).Decode(&e); err != nil {
return err
} else {
// Print the response status and error information.
e, _ := e["error"].(map[string]interface{})
return fmt.Errorf("type: %v, reason: %v", e["type"], e["reason"])
}
}

88
pkg/simple/client/logging/elasticsearch/versions/v6/v6.go Normal file
View File

@@ -0,0 +1,88 @@
package v6
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v6"
"github.com/elastic/go-elasticsearch/v6/esapi"
"io/ioutil"
"k8s.io/klog"
"time"
)
type Elastic struct {
Client *elasticsearch.Client
index string
}
func New(address string, index string) *Elastic {
client, err := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
if err != nil {
klog.Error(err)
return nil
}
return &Elastic{Client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.Client.Search(
e.Client.Search.WithContext(context.Background()),
e.Client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.Client.Search.WithBody(bytes.NewBuffer(body)),
e.Client.Search.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
return ioutil.ReadAll(response.Body)
}
func (e *Elastic) Scroll(id string) ([]byte, error) {
response, err := e.Client.Scroll(
e.Client.Scroll.WithContext(context.Background()),
e.Client.Scroll.WithScrollID(id),
e.Client.Scroll.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
return ioutil.ReadAll(response.Body)
}
func (e *Elastic) ClearScroll(scrollId string) {
response, _ := e.Client.ClearScroll(
e.Client.ClearScroll.WithContext(context.Background()),
e.Client.ClearScroll.WithScrollID(scrollId))
defer response.Body.Close()
}
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
f, _ := v.(float64)
return int64(f)
}
func parseError(response *esapi.Response) error {
var e map[string]interface{}
if err := json.NewDecoder(response.Body).Decode(&e); err != nil {
return err
} else {
// Print the response status and error information.
e, _ := e["error"].(map[string]interface{})
return fmt.Errorf("type: %v, reason: %v", e["type"], e["reason"])
}
}

91
pkg/simple/client/logging/elasticsearch/versions/v7/v7.go Normal file
View File

@@ -0,0 +1,91 @@
package v7
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/elastic/go-elasticsearch/v7"
"github.com/elastic/go-elasticsearch/v7/esapi"
"io/ioutil"
"k8s.io/klog"
"time"
)
type Elastic struct {
client *elasticsearch.Client
index string
}
func New(address string, index string) *Elastic {
client, err := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
if err != nil {
klog.Error(err)
return nil
}
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithTrackTotalHits(true),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
return ioutil.ReadAll(response.Body)
}
func (e *Elastic) Scroll(id string) ([]byte, error) {
response, err := e.client.Scroll(
e.client.Scroll.WithContext(context.Background()),
e.client.Scroll.WithScrollID(id),
e.client.Scroll.WithScroll(time.Minute))
if err != nil {
return nil, err
}
defer response.Body.Close()
if response.IsError() {
return nil, parseError(response)
}
b, err := ioutil.ReadAll(response.Body)
return b, err
}
func (e *Elastic) ClearScroll(scrollId string) {
response, _ := e.client.ClearScroll(
e.client.ClearScroll.WithContext(context.Background()),
e.client.ClearScroll.WithScrollID(scrollId))
defer response.Body.Close()
}
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
m, _ := v.(map[string]interface{})
f, _ := m["value"].(float64)
return int64(f)
}
func parseError(response *esapi.Response) error {
var e map[string]interface{}
if err := json.NewDecoder(response.Body).Decode(&e); err != nil {
return err
} else {
// Print the response status and error information.
e, _ := e["error"].(map[string]interface{})
return fmt.Errorf("type: %v, reason: %v", e["type"], e["reason"])
}
}

69
pkg/simple/client/logging/interface.go Normal file
View File

@@ -0,0 +1,69 @@
package logging
import (
"io"
"time"
)
type Interface interface {
// Current stats about log store, eg. total number of logs and containers
GetCurrentStats(sf SearchFilter) (Statistics, error)
CountLogsByInterval(sf SearchFilter, interval string) (Histogram, error)
SearchLogs(sf SearchFilter, from, size int64, order string) (Logs, error)
ExportLogs(sf SearchFilter, w io.Writer) error
}
// Log search result
type Logs struct {
Total int64 `json:"total" description:"total number of matched results"`
Records []Record `json:"records,omitempty" description:"actual array of results"`
}
type Record struct {
Log string `json:"log,omitempty" description:"log message"`
Time string `json:"time,omitempty" description:"log timestamp"`
Namespace string `json:"namespace,omitempty" description:"namespace"`
Pod string `json:"pod,omitempty" description:"pod name"`
Container string `json:"container,omitempty" description:"container name"`
}
// Log statistics result
type Statistics struct {
Containers int64 `json:"containers" description:"total number of containers"`
Logs int64 `json:"logs" description:"total number of logs"`
}
// Log count result by interval
type Histogram struct {
Total int64 `json:"total" description:"total number of logs"`
Buckets []Bucket `json:"histograms" description:"actual array of histogram results"`
}
type Bucket struct {
Time int64 `json:"time" description:"timestamp"`
Count int64 `json:"count" description:"total number of logs at intervals"`
}
// General query conditions
type SearchFilter struct {
// xxxSearch for literal matching
// xxxfilter for fuzzy matching
// To prevent disclosing archived logs of a reopened namespace,
// NamespaceFilter records the namespace creation time.
// Any query to this namespace must begin after its creation.
NamespaceFilter map[string]time.Time
WorkloadSearch []string
WorkloadFilter []string
PodSearch []string
PodFilter []string
ContainerSearch []string
ContainerFilter []string
LogSearch []string
Starttime time.Time
Endtime time.Time
}

4
pkg/simple/client/logging/logging.go
View File

@@ -1,4 +0,0 @@
package logging
type Interface interface {
}

4
pkg/simple/client/logging/query.go
View File

@@ -1,4 +0,0 @@
package logging
type Query struct {
}