admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor logging options

Browse Source
This commit is contained in:
Jeff
2019-09-17 15:52:35 +08:00
parent fa1e62f6ac
commit f61c5e09ba
20 changed files with 444 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/simple/client/alerting/options.go Normal file
View File

@@ -0,0 +1,17 @@
package alerting
type AlertingOptions struct {
Endpoint string
}
func NewAlertingOptions() *AlertingOptions {
return &AlertingOptions{
Endpoint: "",
}
}
func (s *AlertingOptions) ApplyTo(options *AlertingOptions) {
if s.Endpoint != "" {
options.Endpoint = s.Endpoint
}
}

203
pkg/simple/client/elasticsearch/esclient.go
View File

@@ -13,13 +13,17 @@ limitations under the License.
package esclient
import (
"context"
"encoding/json"
"fmt"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
v5 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v7"
"net/http"
"strconv"
"strings"
"sync"
"time"
"github.com/json-iterator/go"
@@ -49,48 +53,109 @@ const (
fieldContainerNameKeyword = "kubernetes.container_name.keyword"
)
var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
var (
mu sync.Mutex
config *Config
client Client
const (
ElasticV5 = "5"
ElasticV6 = "6"
ElasticV7 = "7"
)
func (cfg *Config) WriteESConfigs() {
mu.Lock()
defer mu.Unlock()
var jsonIter = jsoniter.ConfigCompatibleWithStandardLibrary
config = cfg
if err := detectVersionMajor(config); err != nil {
klog.Errorln(err)
client = nil
return
}
client = NewForConfig(config)
type ElasticSearchClient struct {
client Client
}
func createQueryRequest(param QueryParameters) (int, []byte, error) {
var request Request
var mainBoolQuery BoolFilter
func NewLoggingClient(options *ElasticSearchOptions) (*ElasticSearchClient, error) {
version := "6"
esClient := &ElasticSearchClient{}
if options.Version == "" {
var err error
version, err = detectVersionMajor(options.Host)
if err != nil {
return nil, err
}
}
if options.LogstashFormat {
if options.LogstashPrefix != "" {
options.Index = options.LogstashPrefix
} else {
options.Index = "logstash"
}
}
switch version {
case ElasticV5:
esClient.client = v5.New(options.Host, options.Index)
case ElasticV6:
esClient.client = v6.New(options.Host, options.Index)
case ElasticV7:
esClient.client = v7.New(options.Host, options.Index)
default:
return nil, fmt.Errorf("unsupported elasticsearch version %s", version)
}
return esClient, nil
}
func (c *ElasticSearchClient) ES() *Client {
return &c.client
}
func detectVersionMajor(host string) (string, error) {
// Info APIs are backward compatible with versions of v5.x, v6.x and v7.x
es := v6.New(host, "")
res, err := es.Client.Info(
es.Client.Info.WithContext(context.Background()),
)
if err != nil {
return "", err
}
defer res.Body.Close()
var b map[string]interface{}
if err = json.NewDecoder(res.Body).Decode(&b); err != nil {
return "", err
}
if res.IsError() {
// Print the response status and error information.
e, _ := b["error"].(map[string]interface{})
return "", fmt.Errorf("[%s] %s: %s", res.Status(), e["type"], e["reason"])
}
// get the major version
version, _ := b["version"].(map[string]interface{})
number, _ := version["number"].(string)
if number == "" {
return "", fmt.Errorf("failed to detect elastic version number")
}
v := strings.Split(number, ".")[0]
return v, nil
}
func createQueryRequest(param v1alpha2.QueryParameters) (int, []byte, error) {
var request v1alpha2.Request
var mainBoolQuery v1alpha2.BoolFilter
if len(param.NamespaceWithCreationTime) != 0 {
var boolShould BoolShould
var boolShould v1alpha2.BoolShould
for namespace, creationTime := range param.NamespaceWithCreationTime {
var boolFilter BoolFilter
var boolFilter v1alpha2.BoolFilter
matchPhrase := MatchPhrase{MatchPhrase: map[string]string{fieldNamespaceNameKeyword: namespace}}
rangeQuery := RangeQuery{RangeSpec{TimeRange{creationTime, ""}}}
matchPhrase := v1alpha2.MatchPhrase{MatchPhrase: map[string]string{fieldNamespaceNameKeyword: namespace}}
rangeQuery := v1alpha2.RangeQuery{RangeSpec: v1alpha2.RangeSpec{TimeRange: v1alpha2.TimeRange{Gte: creationTime, Lte: ""}}}
boolFilter.Filter = append(boolFilter.Filter, matchPhrase)
boolFilter.Filter = append(boolFilter.Filter, rangeQuery)
boolShould.Should = append(boolShould.Should, BoolQuery{Bool: boolFilter})
boolShould.Should = append(boolShould.Should, v1alpha2.BoolQuery{Bool: boolFilter})
}
boolShould.MinimumShouldMatch = 1
mainBoolQuery.Filter = append(mainBoolQuery.Filter, BoolQuery{Bool: boolShould})
mainBoolQuery.Filter = append(mainBoolQuery.Filter, v1alpha2.BoolQuery{Bool: boolShould})
}
if param.WorkloadFilter != nil {
boolQuery := makeBoolShould(regexpQuery, fieldPodNameKeyword, param.WorkloadFilter)
@@ -122,15 +187,15 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
mainBoolQuery.Filter = append(mainBoolQuery.Filter, boolQuery)
}
rangeQuery := RangeQuery{RangeSpec{TimeRange{param.StartTime, param.EndTime}}}
rangeQuery := v1alpha2.RangeQuery{RangeSpec: v1alpha2.RangeSpec{TimeRange: v1alpha2.TimeRange{Gte: param.StartTime, Lte: param.EndTime}}}
mainBoolQuery.Filter = append(mainBoolQuery.Filter, rangeQuery)
var operation int
if param.Operation == "statistics" {
operation = OperationStatistics
containerAgg := AggField{"kubernetes.docker_id.keyword"}
statisticAggs := StatisticsAggs{ContainerAgg{containerAgg}}
containerAgg := v1alpha2.AggField{Field: "kubernetes.docker_id.keyword"}
statisticAggs := v1alpha2.StatisticsAggs{ContainerAgg: v1alpha2.ContainerAgg{Cardinality: containerAgg}}
request.Aggs = statisticAggs
request.Size = 0
} else if param.Operation == "histogram" {
@@ -142,7 +207,7 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
interval = "15m"
}
param.Interval = interval
request.Aggs = HistogramAggs{HistogramAgg{DateHistogram{"time", interval}}}
request.Aggs = v1alpha2.HistogramAggs{HistogramAgg: v1alpha2.HistogramAgg{DateHistogram: v1alpha2.DateHistogram{Field: "time", Interval: interval}}}
request.Size = 0
} else {
operation = OperationQuery
@@ -154,25 +219,25 @@ func createQueryRequest(param QueryParameters) (int, []byte, error) {
} else {
order = "desc"
}
request.Sorts = append(request.Sorts, Sort{Order{order}})
request.Sorts = append(request.Sorts, v1alpha2.Sort{Order: v1alpha2.Order{Order: order}})
var mainHighLight MainHighLight
mainHighLight.Fields = append(mainHighLight.Fields, LogHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, NamespaceHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, PodHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, ContainerHighLightField{})
var mainHighLight v1alpha2.MainHighLight
mainHighLight.Fields = append(mainHighLight.Fields, v1alpha2.LogHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, v1alpha2.NamespaceHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, v1alpha2.PodHighLightField{})
mainHighLight.Fields = append(mainHighLight.Fields, v1alpha2.ContainerHighLightField{})
mainHighLight.FragmentSize = 0
request.MainHighLight = mainHighLight
}
request.MainQuery = BoolQuery{mainBoolQuery}
request.MainQuery = v1alpha2.BoolQuery{Bool: mainBoolQuery}
queryRequest, err := json.Marshal(request)
return operation, queryRequest, err
}
func makeBoolShould(queryType int, field string, list []string) BoolQuery {
func makeBoolShould(queryType int, field string, list []string) v1alpha2.BoolQuery {
var should []interface{}
for _, phrase := range list {
@@ -180,18 +245,18 @@ func makeBoolShould(queryType int, field string, list []string) BoolQuery {
switch queryType {
case matchPhrase:
q = MatchPhrase{MatchPhrase: map[string]string{field: phrase}}
q = v1alpha2.MatchPhrase{MatchPhrase: map[string]string{field: phrase}}
case matchPhrasePrefix:
q = MatchPhrasePrefix{MatchPhrasePrefix: map[string]string{field: phrase}}
q = v1alpha2.MatchPhrasePrefix{MatchPhrasePrefix: map[string]string{field: phrase}}
case regexpQuery:
q = RegexpQuery{Regexp: map[string]string{field: makePodNameRegexp(phrase)}}
q = v1alpha2.RegexpQuery{Regexp: map[string]string{field: makePodNameRegexp(phrase)}}
}
should = append(should, q)
}
return BoolQuery{
Bool: BoolShould{
return v1alpha2.BoolQuery{
Bool: v1alpha2.BoolShould{
Should: should,
MinimumShouldMatch: 1,
},
@@ -244,10 +309,10 @@ func calcTimestamp(input string) int64 {
return ret
}
func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryResult {
var queryResult QueryResult
func (c *ElasticSearchClient) parseQueryResult(operation int, param v1alpha2.QueryParameters, body []byte) *v1alpha2.QueryResult {
var queryResult v1alpha2.QueryResult
var response Response
var response v1alpha2.Response
err := jsonIter.Unmarshal(body, &response)
if err != nil {
klog.Errorln(err)
@@ -273,12 +338,12 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
switch operation {
case OperationQuery:
var readResult ReadResult
readResult.Total = client.GetTotalHitCount(response.Hits.Total)
var readResult v1alpha2.ReadResult
readResult.Total = c.client.GetTotalHitCount(response.Hits.Total)
readResult.From = param.From
readResult.Size = param.Size
for _, hit := range response.Hits.Hits {
var logRecord LogRecord
var logRecord v1alpha2.LogRecord
logRecord.Time = calcTimestamp(hit.Source.Time)
logRecord.Log = hit.Source.Log
logRecord.Namespace = hit.Source.Kubernetes.Namespace
@@ -291,7 +356,7 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
queryResult.Read = &readResult
case OperationStatistics:
var statisticsResponse StatisticsResponseAggregations
var statisticsResponse v1alpha2.StatisticsResponseAggregations
err := jsonIter.Unmarshal(response.Aggregations, &statisticsResponse)
if err != nil && response.Aggregations != nil {
klog.Errorln(err)
@@ -299,17 +364,17 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
queryResult.Error = err.Error()
return &queryResult
}
queryResult.Statistics = &StatisticsResult{Containers: statisticsResponse.ContainerCount.Value, Logs: client.GetTotalHitCount(response.Hits.Total)}
queryResult.Statistics = &v1alpha2.StatisticsResult{Containers: statisticsResponse.ContainerCount.Value, Logs: c.client.GetTotalHitCount(response.Hits.Total)}
case OperationHistogram:
var histogramResult HistogramResult
histogramResult.Total = client.GetTotalHitCount(response.Hits.Total)
var histogramResult v1alpha2.HistogramResult
histogramResult.Total = c.client.GetTotalHitCount(response.Hits.Total)
histogramResult.StartTime = calcTimestamp(param.StartTime)
histogramResult.EndTime = calcTimestamp(param.EndTime)
histogramResult.Interval = param.Interval
var histogramAggregations HistogramAggregations
err := jsonIter.Unmarshal(response.Aggregations, &histogramAggregations)
var histogramAggregations v1alpha2.HistogramAggregations
err = jsonIter.Unmarshal(response.Aggregations, &histogramAggregations)
if err != nil && response.Aggregations != nil {
klog.Errorln(err)
queryResult.Status = http.StatusInternalServerError
@@ -317,7 +382,7 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
return &queryResult
}
for _, histogram := range histogramAggregations.HistogramAggregation.Histograms {
var histogramRecord HistogramRecord
var histogramRecord v1alpha2.HistogramRecord
histogramRecord.Time = histogram.Time
histogramRecord.Count = histogram.Count
@@ -332,30 +397,30 @@ func parseQueryResult(operation int, param QueryParameters, body []byte) *QueryR
return &queryResult
}
func Query(param QueryParameters) *QueryResult {
func (c *ElasticSearchClient) Query(param v1alpha2.QueryParameters) *v1alpha2.QueryResult {
var queryResult = new(QueryResult)
var queryResult = new(v1alpha2.QueryResult)
if param.NamespaceNotFound {
queryResult = new(QueryResult)
queryResult = new(v1alpha2.QueryResult)
queryResult.Status = http.StatusOK
switch param.Operation {
case "statistics":
queryResult.Statistics = new(StatisticsResult)
queryResult.Statistics = new(v1alpha2.StatisticsResult)
case "histogram":
queryResult.Histogram = &HistogramResult{
queryResult.Histogram = &v1alpha2.HistogramResult{
StartTime: calcTimestamp(param.StartTime),
EndTime: calcTimestamp(param.EndTime),
Interval: param.Interval}
default:
queryResult.Read = new(ReadResult)
queryResult.Read = new(v1alpha2.ReadResult)
}
return queryResult
}
if client == nil {
if c.client == nil {
queryResult.Status = http.StatusBadRequest
queryResult.Error = fmt.Sprintf("Invalid elasticsearch address: host=%s, port=%s", config.Host, config.Port)
queryResult.Error = "can not create elastic search client"
return queryResult
}
@@ -367,16 +432,16 @@ func Query(param QueryParameters) *QueryResult {
return queryResult
}
body, err := client.Search(query)
body, err := c.client.Search(query)
if err != nil {
klog.Errorln(err)
queryResult = new(QueryResult)
queryResult = new(v1alpha2.QueryResult)
queryResult.Status = http.StatusInternalServerError
queryResult.Error = err.Error()
return queryResult
}
queryResult = parseQueryResult(operation, param, body)
queryResult = c.parseQueryResult(operation, param, body)
return queryResult
}

65
pkg/simple/client/elasticsearch/interface.go
View File

@@ -1,72 +1,7 @@
package esclient
import (
"context"
"encoding/json"
"fmt"
v5 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch/versions/v7"
"strings"
)
const (
ElasticV5 = "5"
ElasticV6 = "6"
ElasticV7 = "7"
)
type Client interface {
// Perform Search API
Search(body []byte) ([]byte, error)
GetTotalHitCount(v interface{}) int64
}
func NewForConfig(cfg *Config) Client {
address := fmt.Sprintf("http://%s:%s", cfg.Host, cfg.Port)
index := cfg.Index
switch cfg.VersionMajor {
case ElasticV5:
return v5.New(address, index)
case ElasticV6:
return v6.New(address, index)
case ElasticV7:
return v7.New(address, index)
default:
return nil
}
}
func detectVersionMajor(cfg *Config) error {
// Info APIs are backward compatible with versions of v5.x, v6.x and v7.x
address := fmt.Sprintf("http://%s:%s", cfg.Host, cfg.Port)
es := v6.New(address, "")
res, err := es.Client.Info(
es.Client.Info.WithContext(context.Background()),
)
if err != nil {
return err
}
defer res.Body.Close()
var b map[string]interface{}
if err := json.NewDecoder(res.Body).Decode(&b); err != nil {
return err
}
if res.IsError() {
// Print the response status and error information.
e, _ := b["error"].(map[string]interface{})
return fmt.Errorf("[%s] %s: %s", res.Status(), e["type"], e["reason"])
}
// get the major version
version, _ := b["version"].(map[string]interface{})
number, _ := version["number"].(string)
if number == "" {
return fmt.Errorf("failed to detect elastic version number")
}
cfg.VersionMajor = strings.Split(number, ".")[0]
return nil
}

63
pkg/simple/client/elasticsearch/options.go Normal file
View File

@@ -0,0 +1,63 @@
package esclient
import (
"github.com/spf13/pflag"
"kubesphere.io/kubesphere/pkg/utils/reflectutils"
)
type ElasticSearchOptions struct {
Host string `json:"host,omitempty" yaml:"host,omitempty"`
LogstashFormat bool `json:"logstashFormat,omitempty" yaml:"logstashFormat,omitempty"`
Index string `json:",omitempty" yaml:",omitempty"`
LogstashPrefix string `json:"logstashPrefix,omitempty" yaml:"logstashPrefix,omitempty"`
Match string `json:",omitempty" yaml:",omitempty"`
Version string `json:",omitempty" yaml:",omitempty"`
}
func NewElasticSearchOptions() *ElasticSearchOptions {
return &ElasticSearchOptions{
Host: "",
LogstashFormat: false,
Index: "fluentbit",
LogstashPrefix: "",
Match: "kube.*",
Version: "6",
}
}
func (s *ElasticSearchOptions) ApplyTo(options *ElasticSearchOptions) {
if s.Host != "" {
reflectutils.Override(options, s)
}
}
func (s *ElasticSearchOptions) Validate() []error {
errs := []error{}
return errs
}
func (s *ElasticSearchOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.Host, "elasticsearch-host", s.Host, ""+
"ElasticSearch logging service host. KubeSphere is using elastic as log store, "+
"if this filed left blank, KubeSphere will use kubernetes builtin log API instead, and"+
" the following elastic search options will be ignored.")
fs.BoolVar(&s.LogstashFormat, "logstash-format", s.LogstashFormat, ""+
"Whether to toggle logstash format compatibility.")
fs.StringVar(&s.LogstashPrefix, "logstash-prefix", s.LogstashPrefix, ""+
"If logstash-format is enabled, the Index name is composed using a prefix and the date,"+
"e.g: If logstash-prefix is equals to 'mydata' your index will become 'mydata-YYYY.MM.DD'."+
"The last string appended belongs to the date when the data is being generated.")
fs.StringVar(&s.Match, "elasticsearch-match", s.Match, ""+
"The regex match for index, eg. kube.*")
fs.StringVar(&s.Index, "elasticsearch-index", s.Index, ""+
"Index name.")
fs.StringVar(&s.Version, "elasticsearch-version", s.Version, ""+
"ElasticSearch major version, e.g. 5/6/7, if left blank, will detect automatically."+
"Currently, minimum supported version is 5.x")
}

261
pkg/simple/client/elasticsearch/types.go
View File

@@ -1,261 +0,0 @@
package esclient
import (
"encoding/json"
)
// elasticsearch client config
type Config struct {
Host string
Port string
Index string
VersionMajor string
}
type QueryParameters struct {
// when true, indicates the provided `namespaces` or `namespace_query` doesn't match any namespace
NamespaceNotFound bool
// a map of namespace with creation time
NamespaceWithCreationTime map[string]string
// filter for literally matching
// query for fuzzy matching
WorkloadFilter []string
WorkloadQuery []string
PodFilter []string
PodQuery []string
ContainerFilter []string
ContainerQuery []string
LogQuery []string
Operation string
Interval string
StartTime string
EndTime string
Sort string
From int64
Size int64
}
// elasticsearch request body
type Request struct {
From int64 `json:"from"`
Size int64 `json:"size"`
Sorts []Sort `json:"sort,omitempty"`
MainQuery BoolQuery `json:"query"`
Aggs interface{} `json:"aggs,omitempty"`
MainHighLight MainHighLight `json:"highlight,omitempty"`
}
type Sort struct {
Order Order `json:"time"`
}
type Order struct {
Order string `json:"order"`
}
type BoolQuery struct {
Bool interface{} `json:"bool"`
}
// user filter instead of must
// filter ignores scoring
type BoolFilter struct {
Filter []interface{} `json:"filter"`
}
type BoolShould struct {
Should []interface{} `json:"should"`
MinimumShouldMatch int64 `json:"minimum_should_match"`
}
type RangeQuery struct {
RangeSpec RangeSpec `json:"range"`
}
type RangeSpec struct {
TimeRange TimeRange `json:"time"`
}
type TimeRange struct {
Gte string `json:"gte,omitempty"`
Lte string `json:"lte,omitempty"`
}
type MatchPhrase struct {
MatchPhrase map[string]string `json:"match_phrase"`
}
type MatchPhrasePrefix struct {
MatchPhrasePrefix interface{} `json:"match_phrase_prefix"`
}
type RegexpQuery struct {
Regexp interface{} `json:"regexp"`
}
type MainHighLight struct {
Fields []interface{} `json:"fields,omitempty"`
FragmentSize int `json:"fragment_size"`
}
type LogHighLightField struct {
FieldContent EmptyField `json:"log"`
}
type NamespaceHighLightField struct {
FieldContent EmptyField `json:"kubernetes.namespace_name.keyword"`
}
type PodHighLightField struct {
FieldContent EmptyField `json:"kubernetes.pod_name.keyword"`
}
type ContainerHighLightField struct {
FieldContent EmptyField `json:"kubernetes.container_name.keyword"`
}
type EmptyField struct {
}
// StatisticsAggs, the struct for `aggs` of type Request, holds a cardinality aggregation for distinct container counting
type StatisticsAggs struct {
ContainerAgg ContainerAgg `json:"containers"`
}
type ContainerAgg struct {
Cardinality AggField `json:"cardinality"`
}
type AggField struct {
Field string `json:"field"`
}
type HistogramAggs struct {
HistogramAgg HistogramAgg `json:"histogram"`
}
type HistogramAgg struct {
DateHistogram DateHistogram `json:"date_histogram"`
}
type DateHistogram struct {
Field string `json:"field"`
Interval string `json:"interval"`
}
// Fore more info, refer to https://www.elastic.co/guide/en/elasticsearch/reference/current/getting-started-search-API.html
// Response body from the elasticsearch engine
type Response struct {
Status int `json:"status"`
Workspace string `json:"workspace,omitempty"`
Shards Shards `json:"_shards"`
Hits Hits `json:"hits"`
Aggregations json.RawMessage `json:"aggregations"`
}
type Shards struct {
Total int64 `json:"total"`
Successful int64 `json:"successful"`
Skipped int64 `json:"skipped"`
Failed int64 `json:"failed"`
}
type Hits struct {
// As of ElasticSearch v7.x, hits.total is changed
Total interface{} `json:"total"`
Hits []Hit `json:"hits"`
}
type Hit struct {
Source Source `json:"_source"`
HighLight HighLight `json:"highlight"`
Sort []int64 `json:"sort"`
}
type Source struct {
Log string `json:"log"`
Time string `json:"time"`
Kubernetes Kubernetes `json:"kubernetes"`
}
type Kubernetes struct {
Namespace string `json:"namespace_name"`
Pod string `json:"pod_name"`
Container string `json:"container_name"`
Host string `json:"host"`
}
type HighLight struct {
LogHighLights []string `json:"log,omitempty" description:"log messages to highlight"`
NamespaceHighLights []string `json:"kubernetes.namespace_name.keyword,omitempty" description:"namespaces to highlight"`
PodHighLights []string `json:"kubernetes.pod_name.keyword,omitempty" description:"pods to highlight"`
ContainerHighLights []string `json:"kubernetes.container_name.keyword,omitempty" description:"containers to highlight"`
}
type LogRecord struct {
Time int64 `json:"time,omitempty" description:"log timestamp"`
Log string `json:"log,omitempty" description:"log message"`
Namespace string `json:"namespace,omitempty" description:"namespace"`
Pod string `json:"pod,omitempty" description:"pod name"`
Container string `json:"container,omitempty" description:"container name"`
Host string `json:"host,omitempty" description:"node id"`
HighLight HighLight `json:"highlight,omitempty" description:"highlighted log fragment"`
}
type ReadResult struct {
Total int64 `json:"total" description:"total number of matched results"`
From int64 `json:"from" description:"the offset from the result set"`
Size int64 `json:"size" description:"the amount of hits to be returned"`
Records []LogRecord `json:"records,omitempty" description:"actual array of results"`
}
// StatisticsResponseAggregations, the struct for `aggregations` of type Reponse, holds return results from the aggregation StatisticsAggs
type StatisticsResponseAggregations struct {
ContainerCount ContainerCount `json:"containers"`
}
type ContainerCount struct {
Value int64 `json:"value"`
}
type HistogramAggregations struct {
HistogramAggregation HistogramAggregation `json:"histogram"`
}
type HistogramAggregation struct {
Histograms []HistogramStatistics `json:"buckets"`
}
type HistogramStatistics struct {
Time int64 `json:"key"`
Count int64 `json:"doc_count"`
}
type HistogramRecord struct {
Time int64 `json:"time" description:"timestamp"`
Count int64 `json:"count" description:"total number of logs at intervals"`
}
type StatisticsResult struct {
Containers int64 `json:"containers" description:"total number of containers"`
Logs int64 `json:"logs" description:"total number of logs"`
}
type HistogramResult struct {
Total int64 `json:"total" description:"total number of logs"`
StartTime int64 `json:"start_time" description:"start time"`
EndTime int64 `json:"end_time" description:"end time"`
Interval string `json:"interval" description:"interval"`
Histograms []HistogramRecord `json:"histograms" description:"actual array of histogram results"`
}
// Wrap elasticsearch response
type QueryResult struct {
Status int `json:"status,omitempty" description:"query status"`
Error string `json:"error,omitempty" description:"debugging information"`
Read *ReadResult `json:"query,omitempty" description:"query results"`
Statistics *StatisticsResult `json:"statistics,omitempty" description:"statistics results"`
Histogram *HistogramResult `json:"histogram,omitempty" description:"histogram results"`
}

8
pkg/simple/client/elasticsearch/versions/v5/elasticsearch.go
View File

@@ -14,16 +14,16 @@ type Elastic struct {
index string
}
func New(address string, index string) Elastic {
func New(address string, index string) *Elastic {
client, _ := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
return Elastic{client: client, index: index}
return &Elastic{client: client, index: index}
}
func (e Elastic) Search(body []byte) ([]byte, error) {
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
e.client.Search.WithContext(context.Background()),
@@ -49,7 +49,7 @@ func (e Elastic) Search(body []byte) ([]byte, error) {
return ioutil.ReadAll(response.Body)
}
func (e Elastic) GetTotalHitCount(v interface{}) int64 {
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
f, _ := v.(float64)
return int64(f)
}

8
pkg/simple/client/elasticsearch/versions/v6/elasticsearch.go
View File

@@ -14,16 +14,16 @@ type Elastic struct {
index string
}
func New(address string, index string) Elastic {
func New(address string, index string) *Elastic {
client, _ := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
return Elastic{Client: client, index: index}
return &Elastic{Client: client, index: index}
}
func (e Elastic) Search(body []byte) ([]byte, error) {
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.Client.Search(
e.Client.Search.WithContext(context.Background()),
@@ -49,7 +49,7 @@ func (e Elastic) Search(body []byte) ([]byte, error) {
return ioutil.ReadAll(response.Body)
}
func (e Elastic) GetTotalHitCount(v interface{}) int64 {
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
f, _ := v.(float64)
return int64(f)
}

8
pkg/simple/client/elasticsearch/versions/v7/elasticsearch.go
View File

@@ -14,16 +14,16 @@ type Elastic struct {
index string
}
func New(address string, index string) Elastic {
func New(address string, index string) *Elastic {
client, _ := elasticsearch.NewClient(elasticsearch.Config{
Addresses: []string{address},
})
return Elastic{client: client, index: index}
return &Elastic{client: client, index: index}
}
func (e Elastic) Search(body []byte) ([]byte, error) {
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
e.client.Search.WithContext(context.Background()),
@@ -50,7 +50,7 @@ func (e Elastic) Search(body []byte) ([]byte, error) {
return ioutil.ReadAll(response.Body)
}
func (e Elastic) GetTotalHitCount(v interface{}) int64 {
func (e *Elastic) GetTotalHitCount(v interface{}) int64 {
m, _ := v.(map[string]interface{})
f, _ := m["value"].(float64)
return int64(f)

91
pkg/simple/client/factory.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
goredis "github.com/go-redis/redis"
"kubesphere.io/kubesphere/pkg/simple/client/devops"
esclient "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/kubesphere"
"kubesphere.io/kubesphere/pkg/simple/client/ldap"
@@ -25,30 +26,32 @@ func (e ClientSetNotEnabledError) Error() string {
}
type ClientSetOptions struct {
mySQLOptions *mysql.MySQLOptions
redisOptions *redis.RedisOptions
kubernetesOptions *k8s.KubernetesOptions
devopsOptions *devops.DevopsOptions
sonarqubeOptions *sonarqube.SonarQubeOptions
ldapOptions *ldap.LdapOptions
s3Options *s2is3.S3Options
openPitrixOptions *openpitrix.OpenPitrixOptions
prometheusOptions *prometheus.PrometheusOptions
kubesphereOptions *kubesphere.KubeSphereOptions
mySQLOptions *mysql.MySQLOptions
redisOptions *redis.RedisOptions
kubernetesOptions *k8s.KubernetesOptions
devopsOptions *devops.DevopsOptions
sonarqubeOptions *sonarqube.SonarQubeOptions
ldapOptions *ldap.LdapOptions
s3Options *s2is3.S3Options
openPitrixOptions *openpitrix.OpenPitrixOptions
prometheusOptions *prometheus.PrometheusOptions
kubesphereOptions *kubesphere.KubeSphereOptions
elasticSearhOptions *esclient.ElasticSearchOptions
}
func NewClientSetOptions() *ClientSetOptions {
return &ClientSetOptions{
mySQLOptions: mysql.NewMySQLOptions(),
redisOptions: redis.NewRedisOptions(),
kubernetesOptions: k8s.NewKubernetesOptions(),
ldapOptions: ldap.NewLdapOptions(),
devopsOptions: devops.NewDevopsOptions(),
sonarqubeOptions: sonarqube.NewSonarQubeOptions(),
s3Options: s2is3.NewS3Options(),
openPitrixOptions: openpitrix.NewOpenPitrixOptions(),
prometheusOptions: prometheus.NewPrometheusOptions(),
kubesphereOptions: kubesphere.NewKubeSphereOptions(),
mySQLOptions: mysql.NewMySQLOptions(),
redisOptions: redis.NewRedisOptions(),
kubernetesOptions: k8s.NewKubernetesOptions(),
ldapOptions: ldap.NewLdapOptions(),
devopsOptions: devops.NewDevopsOptions(),
sonarqubeOptions: sonarqube.NewSonarQubeOptions(),
s3Options: s2is3.NewS3Options(),
openPitrixOptions: openpitrix.NewOpenPitrixOptions(),
prometheusOptions: prometheus.NewPrometheusOptions(),
kubesphereOptions: kubesphere.NewKubeSphereOptions(),
elasticSearhOptions: esclient.NewElasticSearchOptions(),
}
}
@@ -102,6 +105,11 @@ func (c *ClientSetOptions) SetKubeSphereOptions(options *kubesphere.KubeSphereOp
return c
}
func (c *ClientSetOptions) SetElasticSearchOptions(options *esclient.ElasticSearchOptions) *ClientSetOptions {
c.elasticSearhOptions = options
return c
}
// ClientSet provide best of effort service to initialize clients,
// but there is no guarantee to return a valid client instance,
// so do validity check before use
@@ -111,15 +119,16 @@ type ClientSet struct {
mySQLClient *mysql.MySQLClient
k8sClient *k8s.KubernetesClient
ldapClient *ldap.LdapClient
devopsClient *devops.DevopsClient
sonarQubeClient *sonarqube.SonarQubeClient
redisClient *redis.RedisClient
s3Client *s2is3.S3Client
prometheusClient *prometheus.PrometheusClient
openpitrixClient *openpitrix.OpenPitrixClient
kubesphereClient *kubesphere.KubeSphereClient
k8sClient *k8s.KubernetesClient
ldapClient *ldap.LdapClient
devopsClient *devops.DevopsClient
sonarQubeClient *sonarqube.SonarQubeClient
redisClient *redis.RedisClient
s3Client *s2is3.S3Client
prometheusClient *prometheus.PrometheusClient
openpitrixClient *openpitrix.OpenPitrixClient
kubesphereClient *kubesphere.KubeSphereClient
elasticSearchClient *esclient.ElasticSearchClient
}
var mutex sync.Mutex
@@ -335,3 +344,27 @@ func (cs *ClientSet) Prometheus() (*prometheus.PrometheusClient, error) {
func (cs *ClientSet) KubeSphere() *kubesphere.KubeSphereClient {
return cs.kubesphereClient
}
func (cs *ClientSet) ElasticSearch() (*esclient.ElasticSearchClient, error) {
var err error
if cs.csoptions.elasticSearhOptions == nil || cs.csoptions.elasticSearhOptions.Host == "" {
return nil, ClientSetNotEnabledError{}
}
if cs.elasticSearchClient != nil {
return cs.elasticSearchClient, nil
} else {
mutex.Lock()
defer mutex.Unlock()
if cs.elasticSearchClient == nil {
cs.elasticSearchClient, err = esclient.NewLoggingClient(cs.csoptions.elasticSearhOptions)
if err != nil {
return nil, err
}
}
return cs.elasticSearchClient, nil
}
}

17
pkg/simple/client/notification/options.go Normal file
View File

@@ -0,0 +1,17 @@
package notification
type NotificationOptions struct {
Endpoint string
}
func NewNotificationOptions() *NotificationOptions {
return &NotificationOptions{
Endpoint: "",
}
}
func (s *NotificationOptions) ApplyTo(options *NotificationOptions) {
if s.Endpoint != "" {
options.Endpoint = s.Endpoint
}
}