From 3d3b9d5af5166397671e73141efb1c0a2a203702 Mon Sep 17 00:00:00 2001 From: wanjunlei Date: Thu, 4 Jun 2020 12:06:56 +0800 Subject: [PATCH] add response_code_filter parameter in auditing events search api --- pkg/api/auditing/v1alpha1/types.go | 6 +- pkg/kapis/tenant/v1alpha2/register.go | 3 +- pkg/models/auditing/events.go | 5 +- .../auditing/elasticsearch/elasticsearch.go | 11 +- .../elasticsearch/elasticsearch_test.go | 152 +++++++++++++++++- .../client/auditing/elasticsearch/options.go | 6 +- pkg/simple/client/auditing/interface.go | 3 +- 7 files changed, 172 insertions(+), 14 deletions(-) diff --git a/pkg/api/auditing/v1alpha1/types.go b/pkg/api/auditing/v1alpha1/types.go index 6d8357ea3..e791da8fa 100644 --- a/pkg/api/auditing/v1alpha1/types.go +++ b/pkg/api/auditing/v1alpha1/types.go @@ -45,7 +45,8 @@ type Query struct { SourceIpSearch string `json:"source_ip_search,omitempty"` ObjectRefResourceFilter string `json:"objectref_resource_filter,omitempty"` ObjectRefSubresourceFilter string `json:"objectref_subresource_filter,omitempty"` - ResponesStatusFilter string `json:"response_status_filter,omitempty"` + ResponseCodeFilter string `json:"response_code_filter,omitempty"` + ResponseStatusFilter string `json:"response_status_filter,omitempty"` StartTime *time.Time `json:"start_time,omitempty"` EndTime *time.Time `json:"end_time,omitempty"` @@ -74,7 +75,8 @@ func ParseQueryParameter(req *restful.Request) (*Query, error) { q.GroupSearch = req.QueryParameter("group_search") q.ObjectRefResourceFilter = req.QueryParameter("objectref_resource_filter") q.ObjectRefSubresourceFilter = req.QueryParameter("objectref_subresource_filter") - q.ResponesStatusFilter = req.QueryParameter("response_status_filter") + q.ResponseCodeFilter = req.QueryParameter("response_code_filter") + q.ResponseStatusFilter = req.QueryParameter("response_status_filter") if tstr := req.QueryParameter("start_time"); tstr != "" { sec, err := strconv.ParseInt(tstr, 10, 64) diff --git a/pkg/kapis/tenant/v1alpha2/register.go b/pkg/kapis/tenant/v1alpha2/register.go index a8ab022d6..e9d100d60 100644 --- a/pkg/kapis/tenant/v1alpha2/register.go +++ b/pkg/kapis/tenant/v1alpha2/register.go @@ -172,7 +172,8 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s Param(ws.QueryParameter("source_ip_search", "A comma-separated list of keywords. This field performs fuzzy matching on 'SourceIPs'. For example, the following value limits the query to SourceIPs which contains 127.0 *OR* 192.168.: `127.0,192.168.`.")). Param(ws.QueryParameter("objectref_resource_filter", "A comma-separated list of resource. This field restricts the query to specified ip. This field restricts the query to specified `ObjectRef.Resource`.")). Param(ws.QueryParameter("objectref_subresource_filter", "A comma-separated list of subresource. This field restricts the query to specified subresource. This field restricts the query to specified `ObjectRef.Subresource`.")). - Param(ws.QueryParameter("response_status_filter", "A comma-separated list of response status code. This field restricts the query to specified response status code. This field restricts the query to specified `ResponseStatus.Code`.")). + Param(ws.QueryParameter("response_code_filter", "A comma-separated list of response status code. This field restricts the query to specified response status code. This field restricts the query to specified `ResponseStatus.code`.")). + Param(ws.QueryParameter("response_status_filter", "A comma-separated list of response status. This field restricts the query to specified response status. This field restricts the query to specified `ResponseStatus.status`.")). Param(ws.QueryParameter("start_time", "Start time of query (limits `RequestReceivedTimestamp`). The format is a string representing seconds since the epoch, eg. 1136214245.")). Param(ws.QueryParameter("end_time", "End time of query (limits `RequestReceivedTimestamp`). The format is a string representing seconds since the epoch, eg. 1136214245.")). Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to `histogram`. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m")). diff --git a/pkg/models/auditing/events.go b/pkg/models/auditing/events.go index 1088b750f..9e7a2ac70 100644 --- a/pkg/models/auditing/events.go +++ b/pkg/models/auditing/events.go @@ -48,6 +48,7 @@ func (eo *eventsOperator) Events(queryParam *v1alpha1.Query, SourceIpFuzzy: stringutils.Split(queryParam.SourceIpSearch, ","), ObjectRefResources: stringutils.Split(queryParam.ObjectRefResourceFilter, ","), ObjectRefSubresources: stringutils.Split(queryParam.ObjectRefSubresourceFilter, ","), + ResponseStatus: stringutils.Split(queryParam.ResponseStatusFilter, ","), StartTime: queryParam.StartTime, EndTime: queryParam.EndTime, } @@ -55,14 +56,14 @@ func (eo *eventsOperator) Events(queryParam *v1alpha1.Query, MutateFilterFunc(filter) } - cs := stringutils.Split(queryParam.ResponesStatusFilter, ",") + cs := stringutils.Split(queryParam.ResponseCodeFilter, ",") for _, c := range cs { code, err := strconv.ParseInt(c, 10, 64) if err != nil { continue } - filter.ResponseStatus = append(filter.ResponseStatus, int32(code)) + filter.ResponseCodes = append(filter.ResponseCodes, int32(code)) } var ar v1alpha1.APIResponse diff --git a/pkg/simple/client/auditing/elasticsearch/elasticsearch.go b/pkg/simple/client/auditing/elasticsearch/elasticsearch.go index 74dbdbf6b..ee7e972e6 100644 --- a/pkg/simple/client/auditing/elasticsearch/elasticsearch.go +++ b/pkg/simple/client/auditing/elasticsearch/elasticsearch.go @@ -361,10 +361,10 @@ func parseToQueryPart(f *auditing.Filter) interface{} { } } - if f.ResponseStatus != nil && len(f.ResponseStatus) > 0 { + if f.ResponseCodes != nil && len(f.ResponseCodes) > 0 { bi := BoolBody{MinimumShouldMatch: &mini} - for _, v := range f.ResponseStatus { + for _, v := range f.ResponseCodes { bi.Should = append(bi.Should, map[string]interface{}{ "term": map[string]int32{"ResponseStatus.code": v}, }) @@ -373,6 +373,13 @@ func parseToQueryPart(f *auditing.Filter) interface{} { b.Filter = append(b.Filter, map[string]interface{}{"bool": bi}) } + if len(f.ResponseStatus) > 0 { + if bi := shouldBoolbody("match_phrase", "ResponseStatus.status", + f.ResponseStatus, nil); bi != nil { + b.Filter = append(b.Filter, map[string]interface{}{"bool": bi}) + } + } + if f.StartTime != nil || f.EndTime != nil { m := make(map[string]*time.Time) if f.StartTime != nil { diff --git a/pkg/simple/client/auditing/elasticsearch/elasticsearch_test.go b/pkg/simple/client/auditing/elasticsearch/elasticsearch_test.go index e647f0828..05251203f 100644 --- a/pkg/simple/client/auditing/elasticsearch/elasticsearch_test.go +++ b/pkg/simple/client/auditing/elasticsearch/elasticsearch_test.go @@ -171,6 +171,18 @@ func TestParseToQueryPart(t *testing.T) { "minimum_should_match": 1 } }, + { + "bool": { + "should": [ + { + "match_phrase_prefix": { + "ObjectRef.Name.keyword": "istio" + } + } + ], + "minimum_should_match": 1 + } + }, { "bool": { "should": [ @@ -183,10 +195,131 @@ func TestParseToQueryPart(t *testing.T) { "minimum_should_match": 1 } }, + { + "bool": { + "should": [ + { + "match_phrase": { + "Verb": "create" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "Level": "Metadata" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "wildcard": { + "SourceIPs": "*192.168*" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "User.Username.keyword": "system:serviceaccount:kubesphere-system:kubesphere" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "wildcard": { + "User.Username": "*system:serviceaccount*" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "wildcard": { + "User.Groups": "*system:serviceaccounts*" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase_prefix": { + "ObjectRef.Resource.keyword": "devops" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase_prefix": { + "ObjectRef.Subresource.keyword": "pipeline" + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "term": { + "ResponseStatus.code": 404 + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match_phrase": { + "ResponseStatus.status": "Failure" + } + } + ], + "minimum_should_match": 1 + } + }, { "range": { "RequestReceivedTimestamp": { - "gte": "2019-12-01T01:01:01.000000001Z" + "gte": "2019-12-01T01:01:01.000000001Z", + "lte": "2020-01-01T01:01:01.000000001Z" } } } @@ -196,13 +329,26 @@ func TestParseToQueryPart(t *testing.T) { ` nsCreateTime := time.Date(2020, time.Month(1), 1, 1, 1, 1, 1, time.UTC) startTime := nsCreateTime.AddDate(0, -1, 0) + endTime := nsCreateTime.AddDate(0, 0, 0) filter := &auditing.Filter{ ObjectRefNamespaceMap: map[string]time.Time{ "kubesphere-system": nsCreateTime, }, - ObjectRefNameFuzzy: []string{"istio"}, - StartTime: &startTime, + ObjectRefNames: []string{"istio"}, + ObjectRefNameFuzzy: []string{"istio"}, + Levels: []string{"Metadata"}, + Verbs: []string{"create"}, + Users: []string{"system:serviceaccount:kubesphere-system:kubesphere"}, + UserFuzzy: []string{"system:serviceaccount"}, + GroupFuzzy: []string{"system:serviceaccounts"}, + SourceIpFuzzy: []string{"192.168"}, + ObjectRefResources: []string{"devops"}, + ObjectRefSubresources: []string{"pipeline"}, + ResponseCodes: []int32{404}, + ResponseStatus: []string{"Failure"}, + StartTime: &startTime, + EndTime: &endTime, } qp := parseToQueryPart(filter) diff --git a/pkg/simple/client/auditing/elasticsearch/options.go b/pkg/simple/client/auditing/elasticsearch/options.go index 3f35a9c9c..7809c25a5 100644 --- a/pkg/simple/client/auditing/elasticsearch/options.go +++ b/pkg/simple/client/auditing/elasticsearch/options.go @@ -47,15 +47,15 @@ func (s *Options) Validate() []error { } func (s *Options) AddFlags(fs *pflag.FlagSet, c *Options) { - fs.StringVar(&s.Host, "elasticsearch-host", c.Host, ""+ + fs.StringVar(&s.Host, "auditing-elasticsearch-host", c.Host, ""+ "Elasticsearch service host. KubeSphere is using elastic as auditing store, "+ "if this filed left blank, KubeSphere will use kubernetes builtin event API instead, and"+ " the following elastic search options will be ignored.") - fs.StringVar(&s.IndexPrefix, "index-prefix", c.IndexPrefix, ""+ + fs.StringVar(&s.IndexPrefix, "auditing-index-prefix", c.IndexPrefix, ""+ "Index name prefix. KubeSphere will retrieve auditing against indices matching the prefix.") - fs.StringVar(&s.Version, "elasticsearch-version", c.Version, ""+ + fs.StringVar(&s.Version, "auditing-elasticsearch-version", c.Version, ""+ "Elasticsearch major version, e.g. 5/6/7, if left blank, will detect automatically."+ "Currently, minimum supported version is 5.x") } diff --git a/pkg/simple/client/auditing/interface.go b/pkg/simple/client/auditing/interface.go index 02bf6d30d..4e8ded20d 100644 --- a/pkg/simple/client/auditing/interface.go +++ b/pkg/simple/client/auditing/interface.go @@ -38,7 +38,8 @@ type Filter struct { SourceIpFuzzy []string ObjectRefResources []string ObjectRefSubresources []string - ResponseStatus []int32 + ResponseCodes []int32 + ResponseStatus []string StartTime *time.Time EndTime *time.Time }