diff --git a/pkg/api/logging/v1alpha2/types.go b/pkg/api/logging/v1alpha2/types.go index 30a64ac68..a63e9f46d 100644 --- a/pkg/api/logging/v1alpha2/types.go +++ b/pkg/api/logging/v1alpha2/types.go @@ -1,9 +1,103 @@ package v1alpha2 -import "kubesphere.io/kubesphere/pkg/simple/client/logging" +import ( + "github.com/emicklei/go-restful" + "kubesphere.io/kubesphere/pkg/simple/client/logging" + "strconv" + "time" +) + +const ( + OperationStatistics = "statistics" + OperationHistogram = "histogram" + OperationQuery = "query" + OperationExport = "export" + + DefaultInterval = "15m" + DefaultSize = 10 + OrderAscending = "asc" + OrderDescending = "desc" +) type APIResponse struct { Logs *logging.Logs `json:"query,omitempty" description:"query results"` Statistics *logging.Statistics `json:"statistics,omitempty" description:"statistics results"` Histogram *logging.Histogram `json:"histogram,omitempty" description:"histogram results"` } + +type Query struct { + Operation string + WorkspaceFilter string + WorkspaceSearch string + NamespaceFilter string + NamespaceSearch string + WorkloadFilter string + WorkloadSearch string + PodFilter string + PodSearch string + ContainerFilter string + ContainerSearch string + LogSearch string + StartTime time.Time + EndTime time.Time + Interval string + Sort string + From int64 + Size int64 +} + +func ParseQueryParameter(req *restful.Request) (*Query, error) { + var q Query + q.Operation = req.QueryParameter("operation") + q.WorkspaceFilter = req.QueryParameter("workspaces") + q.WorkspaceSearch = req.QueryParameter("workspace_query") + q.NamespaceFilter = req.QueryParameter("namespaces") + q.NamespaceSearch = req.QueryParameter("namespace_query") + q.WorkloadFilter = req.QueryParameter("workloads") + q.WorkloadSearch = req.QueryParameter("workload_query") + q.PodFilter = req.QueryParameter("pods") + q.PodSearch = req.QueryParameter("pod_query") + q.ContainerFilter = req.QueryParameter("containers") + q.ContainerSearch = req.QueryParameter("container_query") + q.LogSearch = req.QueryParameter("log_query") + + if q.Operation == "" { + q.Operation = OperationQuery + } + + if tstr := req.QueryParameter("start_time"); tstr != "" { + sec, err := strconv.ParseInt(tstr, 10, 64) + if err != nil { + return nil, err + } + q.StartTime = time.Unix(sec, 0) + } + if tstr := req.QueryParameter("end_time"); tstr != "" { + sec, err := strconv.ParseInt(tstr, 10, 64) + if err != nil { + return nil, err + } + q.EndTime = time.Unix(sec, 0) + } + + switch q.Operation { + case OperationHistogram: + q.Interval = req.QueryParameter("interval") + if q.Interval == "" { + q.Interval = DefaultInterval + } + case OperationQuery: + q.From, _ = strconv.ParseInt(req.QueryParameter("from"), 10, 64) + size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64) + if err != nil { + size = DefaultSize + } + q.Size = size + q.Sort = req.QueryParameter("sort") + if q.Sort != OrderAscending { + q.Sort = OrderDescending + } + } + + return &q, nil +} diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 242847d91..83ec0a061 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -49,7 +49,6 @@ import ( devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2" devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3" iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2" - loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2" monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2" notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1" @@ -165,7 +164,6 @@ func (s *APIServer) PrepareRun() error { func (s *APIServer) installKubeSphereAPIs() { urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config)) urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory)) - urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient)) urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient)) urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient)) urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost)) @@ -173,7 +171,7 @@ func (s *APIServer) installKubeSphereAPIs() { urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory, s.KubernetesClient.Master())) urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(), - s.KubernetesClient.KubeSphere(), s.EventsClient)) + s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient)) urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config())) urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container, s.InformerFactory.KubernetesSharedInformerFactory(), diff --git a/pkg/kapis/logging/group.go b/pkg/kapis/logging/group.go deleted file mode 100644 index 4cc500d07..000000000 --- a/pkg/kapis/logging/group.go +++ /dev/null @@ -1,18 +0,0 @@ -/* -Copyright 2019 The KubeSphere authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package logging contains logging API versions -package logging diff --git a/pkg/kapis/logging/v1alpha2/handler.go b/pkg/kapis/logging/v1alpha2/handler.go deleted file mode 100644 index 585099466..000000000 --- a/pkg/kapis/logging/v1alpha2/handler.go +++ /dev/null @@ -1,156 +0,0 @@ -package v1alpha2 - -import ( - "github.com/emicklei/go-restful" - "kubesphere.io/kubesphere/pkg/api" - "kubesphere.io/kubesphere/pkg/models/logging" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging" - util "kubesphere.io/kubesphere/pkg/utils/stringutils" - "strconv" - "strings" - "time" -) - -const ( - LevelCluster = iota - LevelContainer - - // query type, default to `query` - TypeStat = "statistics" - TypeHist = "histogram" - TypeExport = "export" - - Ascending = "asc" - Descending = "desc" -) - -type handler struct { - k k8s.Client - lo logging.LoggingOperator -} - -func newHandler(k k8s.Client, l loggingclient.Interface) *handler { - return &handler{k, logging.NewLoggingOperator(l)} -} - -func (h handler) handleClusterQuery(req *restful.Request, resp *restful.Response) { - h.get(req, LevelCluster, resp) -} - -func (h handler) handleContainerQuery(req *restful.Request, resp *restful.Response) { - h.get(req, LevelContainer, resp) -} - -func (h handler) get(req *restful.Request, lvl int, resp *restful.Response) { - typ := req.QueryParameter("type") - - noHit, sf, err := h.newSearchFilter(req, lvl) - if err != nil { - api.HandleBadRequest(resp, nil, err) - } - if noHit { - handleNoHit(typ, resp) - return - } - - switch typ { - case TypeStat: - res, err := h.lo.GetCurrentStats(sf) - if err != nil { - api.HandleInternalError(resp, nil, err) - } - resp.WriteAsJson(res) - case TypeHist: - interval := req.QueryParameter("interval") - res, err := h.lo.CountLogsByInterval(sf, interval) - if err != nil { - api.HandleInternalError(resp, nil, err) - } - resp.WriteAsJson(res) - case TypeExport: - resp.Header().Set(restful.HEADER_ContentType, "text/plain") - resp.Header().Set("Content-Disposition", "attachment") - err := h.lo.ExportLogs(sf, resp.ResponseWriter) - if err != nil { - api.HandleInternalError(resp, nil, err) - } - default: - from, _ := strconv.ParseInt(req.QueryParameter("from"), 10, 64) - size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64) - if err != nil { - size = 10 - } - order := req.QueryParameter("sort") - if order != Ascending { - order = Descending - } - res, err := h.lo.SearchLogs(sf, from, size, order) - if err != nil { - api.HandleInternalError(resp, nil, err) - } - resp.WriteAsJson(res) - } -} - -func (h handler) newSearchFilter(req *restful.Request, level int) (bool, loggingclient.SearchFilter, error) { - var sf loggingclient.SearchFilter - - switch level { - case LevelCluster: - sf.NamespaceFilter = h.intersect( - util.Split(req.QueryParameter("namespaces"), ","), - util.Split(strings.ToLower(req.QueryParameter("namespace_query")), ","), - util.Split(req.QueryParameter("workspaces"), ","), - util.Split(strings.ToLower(req.QueryParameter("workspace_query")), ",")) - sf.WorkloadFilter = util.Split(req.QueryParameter("workloads"), ",") - sf.WorkloadSearch = util.Split(req.QueryParameter("workload_query"), ",") - sf.PodFilter = util.Split(req.QueryParameter("pods"), ",") - sf.PodSearch = util.Split(req.QueryParameter("pod_query"), ",") - sf.ContainerFilter = util.Split(req.QueryParameter("containers"), ",") - sf.ContainerSearch = util.Split(req.QueryParameter("container_query"), ",") - case LevelContainer: - sf.NamespaceFilter = h.withCreationTime(req.PathParameter("namespace")) - sf.PodFilter = []string{req.PathParameter("pod")} - sf.ContainerFilter = []string{req.PathParameter("container")} - } - - sf.LogSearch = util.Split(req.QueryParameter("log_query"), ",") - - var err error - now := time.Now() - // If time is not given, set it to now. - if req.QueryParameter("start_time") == "" { - sf.Starttime = now - } else { - sf.Starttime, err = time.Parse(time.RFC3339, req.QueryParameter("start_time")) - if err != nil { - return false, sf, err - } - } - if req.QueryParameter("end_time") == "" { - sf.Endtime = now - } else { - sf.Endtime, err = time.Parse(time.RFC3339, req.QueryParameter("end_time")) - if err != nil { - return false, sf, err - } - } - - return len(sf.NamespaceFilter) == 0, sf, nil -} - -func handleNoHit(typ string, resp *restful.Response) { - switch typ { - case TypeStat: - resp.WriteAsJson(new(loggingclient.Statistics)) - case TypeHist: - resp.WriteAsJson(new(loggingclient.Histogram)) - case TypeExport: - resp.Header().Set(restful.HEADER_ContentType, "text/plain") - resp.Header().Set("Content-Disposition", "attachment") - resp.Write(nil) - default: - resp.WriteAsJson(new(loggingclient.Logs)) - } -} diff --git a/pkg/kapis/logging/v1alpha2/helper.go b/pkg/kapis/logging/v1alpha2/helper.go deleted file mode 100644 index 1acb22a9e..000000000 --- a/pkg/kapis/logging/v1alpha2/helper.go +++ /dev/null @@ -1,56 +0,0 @@ -package v1alpha2 - -import ( - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/utils/stringutils" - "strings" - "time" -) - -func (h handler) intersect(nsFilter []string, nsSearch []string, wsFilter []string, wsSearch []string) map[string]time.Time { - nsList, err := h.k.Kubernetes().CoreV1().Namespaces().List(v1.ListOptions{}) - if err != nil { - klog.Errorf("failed to list namespace, error: %s", err) - return nil - } - - inner := make(map[string]time.Time) - - // if no search condition is set on both namespace and workspace, - // then return all namespaces - if nsSearch == nil && nsFilter == nil && wsSearch == nil && wsFilter == nil { - for _, ns := range nsList.Items { - inner[ns.Name] = ns.CreationTimestamp.Time - } - } else { - for _, ns := range nsList.Items { - if stringutils.StringIn(ns.Name, nsFilter) || - stringutils.StringIn(ns.Annotations[constants.WorkspaceLabelKey], wsFilter) || - containsIn(ns.Name, nsSearch) || - containsIn(ns.Annotations[constants.WorkspaceLabelKey], wsSearch) { - inner[ns.Name] = ns.CreationTimestamp.Time - } - } - } - - return inner -} - -func containsIn(str string, subStrs []string) bool { - for _, sub := range subStrs { - if strings.Contains(str, sub) { - return true - } - } - return false -} - -func (h handler) withCreationTime(name string) map[string]time.Time { - ns, err := h.k.Kubernetes().CoreV1().Namespaces().Get(name, v1.GetOptions{}) - if err == nil { - return map[string]time.Time{name: ns.CreationTimestamp.Time} - } - return nil -} diff --git a/pkg/kapis/logging/v1alpha2/register.go b/pkg/kapis/logging/v1alpha2/register.go deleted file mode 100644 index efd268d70..000000000 --- a/pkg/kapis/logging/v1alpha2/register.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package v1alpha2 - -import ( - "github.com/emicklei/go-restful" - "github.com/emicklei/go-restful-openapi" - "k8s.io/apimachinery/pkg/runtime/schema" - "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2" - "kubesphere.io/kubesphere/pkg/apiserver/runtime" - "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - "kubesphere.io/kubesphere/pkg/simple/client/logging" - "net/http" -) - -const ( - GroupName = "logging.kubesphere.io" - RespOK = "ok" -) - -var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"} - -func AddToContainer(c *restful.Container, k8sClient k8s.Client, loggingClient logging.Interface) error { - ws := runtime.NewWebService(GroupVersion) - - h := newHandler(k8sClient, loggingClient) - - ws.Route(ws.GET("/cluster"). - To(h.handleClusterQuery). - Doc("Query logs against the cluster."). - Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). - Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)). - Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)). - Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)). - Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)). - Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)). - Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)). - Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)). - Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)). - Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)). - Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}). - Writes(v1alpha2.APIResponse{}). - Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON, "text/plain") - - ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}"). - To(h.handleContainerQuery). - Doc("Query logs against the specific container."). - Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). - Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). - Param(ws.PathParameter("container", "Container name.").DataType("string").Required(true)). - Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). - Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)). - Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)). - Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)). - Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)). - Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}). - Writes(v1alpha2.APIResponse{}). - Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})). - Consumes(restful.MIME_JSON, restful.MIME_XML). - Produces(restful.MIME_JSON, restful.MIME_OCTET) - - c.Add(ws) - return nil -} diff --git a/pkg/kapis/tenant/v1alpha2/handler.go b/pkg/kapis/tenant/v1alpha2/handler.go index 162bd3138..e267eb9be 100644 --- a/pkg/kapis/tenant/v1alpha2/handler.go +++ b/pkg/kapis/tenant/v1alpha2/handler.go @@ -9,6 +9,7 @@ import ( "k8s.io/klog" "kubesphere.io/kubesphere/pkg/api" eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1" + loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2" tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2" "kubesphere.io/kubesphere/pkg/apiserver/query" "kubesphere.io/kubesphere/pkg/apiserver/request" @@ -17,16 +18,17 @@ import ( "kubesphere.io/kubesphere/pkg/models/tenant" servererr "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/simple/client/events" + "kubesphere.io/kubesphere/pkg/simple/client/logging" ) type tenantHandler struct { tenant tenant.Interface } -func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) *tenantHandler { +func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) *tenantHandler { return &tenantHandler{ - tenant: tenant.New(factory, k8sclient, ksclient, evtsClient), + tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient), } } @@ -245,3 +247,38 @@ func (h *tenantHandler) Events(req *restful.Request, resp *restful.Response) { resp.WriteEntity(result) } + +func (h *tenantHandler) QueryLogs(req *restful.Request, resp *restful.Response) { + user, ok := request.UserFrom(req.Request.Context()) + if !ok { + err := fmt.Errorf("cannot obtain user info") + klog.Errorln(err) + api.HandleForbidden(resp, req, err) + return + } + queryParam, err := loggingv1alpha2.ParseQueryParameter(req) + if err != nil { + klog.Errorln(err) + api.HandleInternalError(resp, req, err) + return + } + + if queryParam.Operation == loggingv1alpha2.OperationExport { + resp.Header().Set(restful.HEADER_ContentType, "text/plain") + resp.Header().Set("Content-Disposition", "attachment") + err := h.tenant.ExportLogs(user, queryParam, resp) + if err != nil { + klog.Errorln(err) + api.HandleInternalError(resp, req, err) + return + } + } else { + result, err := h.tenant.QueryLogs(user, queryParam) + if err != nil { + klog.Errorln(err) + api.HandleInternalError(resp, req, err) + return + } + resp.WriteAsJson(result) + } +} diff --git a/pkg/kapis/tenant/v1alpha2/register.go b/pkg/kapis/tenant/v1alpha2/register.go index 0c85f64a4..4a924abb5 100644 --- a/pkg/kapis/tenant/v1alpha2/register.go +++ b/pkg/kapis/tenant/v1alpha2/register.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes" "kubesphere.io/kubesphere/pkg/api" eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1" + loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2" tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2" "kubesphere.io/kubesphere/pkg/apiserver/runtime" kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned" @@ -32,6 +33,7 @@ import ( "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/simple/client/events" + "kubesphere.io/kubesphere/pkg/simple/client/logging" "net/http" ) @@ -41,9 +43,9 @@ const ( var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"} -func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) error { +func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) error { ws := runtime.NewWebService(GroupVersion) - handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient) + handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient) ws.Route(ws.POST("/workspaces"). To(handler.CreateWorkspace). @@ -95,7 +97,7 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s ws.Route(ws.GET("/events"). To(handler.Events). Doc("Query events against the cluster"). - Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")). + Param(ws.QueryParameter("operation", "Operation type. This can be one of three types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")). Param(ws.QueryParameter("workspace_filter", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`.")). Param(ws.QueryParameter("workspace_search", "A comma-separated list of keywords. Differing from **workspace_filter**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.")). Param(ws.QueryParameter("involved_object_namespace_filter", "A comma-separated list of namespaces. This field restricts the query to specified `involvedObject.namespace`.")). @@ -117,6 +119,33 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s Writes(eventsv1alpha1.APIResponse{}). Returns(http.StatusOK, api.StatusOK, eventsv1alpha1.APIResponse{})) + ws.Route(ws.GET("/logs"). + To(handler.QueryLogs). + Doc("Query logs against the cluster."). + Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). + Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)). + Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). + Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)). + Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). + Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)). + Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). + Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)). + Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). + Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)). + Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). + Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)). + Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)). + Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)). + Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)). + Param(ws.QueryParameter("sort", "Sort order. One of asc, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)). + Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)). + Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}). + Writes(loggingv1alpha2.APIResponse{}). + Returns(http.StatusOK, api.StatusOK, loggingv1alpha2.APIResponse{})). + Consumes(restful.MIME_JSON, restful.MIME_XML). + Produces(restful.MIME_JSON, "text/plain") + c.Add(ws) return nil } diff --git a/pkg/models/tenant/tenant.go b/pkg/models/tenant/tenant.go index db77803b9..959523714 100644 --- a/pkg/models/tenant/tenant.go +++ b/pkg/models/tenant/tenant.go @@ -18,6 +18,7 @@ package tenant import ( "fmt" + "io" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -27,9 +28,9 @@ import ( "k8s.io/klog" "kubesphere.io/kubesphere/pkg/api" eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1" + loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2" clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1" tenantv1alpha1 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" - tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2" "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory" @@ -38,9 +39,11 @@ import ( "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models/events" "kubesphere.io/kubesphere/pkg/models/iam/am" + "kubesphere.io/kubesphere/pkg/models/logging" resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3" resourcesv1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource" eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events" + loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging" "kubesphere.io/kubesphere/pkg/utils/stringutils" "strings" "time" @@ -57,6 +60,8 @@ type Interface interface { ListWorkspaceClusters(workspace string) (*api.ListResult, error) Events(user user.Info, queryParam *eventsv1alpha1.Query) (*eventsv1alpha1.APIResponse, error) + QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error) + ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error } type tenantOperator struct { @@ -66,9 +71,10 @@ type tenantOperator struct { ksclient kubesphere.Interface resourceGetter *resourcesv1alpha3.ResourceGetter events events.Interface + lo logging.LoggingOperator } -func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client) Interface { +func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Interface) Interface { amOperator := am.NewReadOnlyOperator(informers) authorizer := authorizerfactory.NewRBACAuthorizer(amOperator) return &tenantOperator{ @@ -78,6 +84,7 @@ func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ks k8sclient: k8sclient, ksclient: ksclient, events: events.NewEventsOperator(evtsClient), + lo: logging.NewLoggingOperator(loggingClient), } } @@ -406,6 +413,129 @@ func (t *tenantOperator) Events(user user.Info, queryParam *eventsv1alpha1.Query }) } +func (t *tenantOperator) QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error) { + iNamespaces, err := t.listIntersectedNamespaces(user, + stringutils.Split(query.WorkspaceFilter, ","), + stringutils.Split(query.WorkspaceSearch, ","), + stringutils.Split(query.NamespaceFilter, ","), + stringutils.Split(query.NamespaceSearch, ",")) + if err != nil { + klog.Error(err) + return nil, err + } + + namespaceCreateTimeMap := make(map[string]time.Time) + for _, ns := range iNamespaces { + podLogs := authorizer.AttributesRecord{ + User: user, + Verb: "get", + APIGroup: "", + APIVersion: "v1", + Namespace: ns.Name, + Resource: "pods", + Subresource: "log", + ResourceRequest: true, + } + decision, _, err := t.authorizer.Authorize(podLogs) + if err != nil { + klog.Error(err) + return nil, err + } + if decision == authorizer.DecisionAllow { + namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time + } + } + + sf := loggingclient.SearchFilter{ + NamespaceFilter: namespaceCreateTimeMap, + WorkloadSearch: stringutils.Split(query.WorkloadSearch, ","), + WorkloadFilter: stringutils.Split(query.WorkloadFilter, ","), + PodSearch: stringutils.Split(query.PodSearch, ","), + PodFilter: stringutils.Split(query.PodFilter, ","), + ContainerSearch: stringutils.Split(query.ContainerSearch, ","), + ContainerFilter: stringutils.Split(query.ContainerFilter, ","), + LogSearch: stringutils.Split(query.LogSearch, ","), + Starttime: query.StartTime, + Endtime: query.EndTime, + } + + var ar loggingv1alpha2.APIResponse + switch query.Operation { + case loggingv1alpha2.OperationStatistics: + if len(namespaceCreateTimeMap) == 0 { + ar.Statistics = &loggingclient.Statistics{} + } else { + ar, err = t.lo.GetCurrentStats(sf) + } + case loggingv1alpha2.OperationHistogram: + if len(namespaceCreateTimeMap) == 0 { + ar.Histogram = &loggingclient.Histogram{} + } else { + ar, err = t.lo.CountLogsByInterval(sf, query.Interval) + } + default: + if len(namespaceCreateTimeMap) == 0 { + ar.Logs = &loggingclient.Logs{} + } else { + ar, err = t.lo.SearchLogs(sf, query.From, query.Size, query.Sort) + } + } + return &ar, err +} + +func (t *tenantOperator) ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error { + iNamespaces, err := t.listIntersectedNamespaces(user, + stringutils.Split(query.WorkspaceFilter, ","), + stringutils.Split(query.WorkspaceSearch, ","), + stringutils.Split(query.NamespaceFilter, ","), + stringutils.Split(query.NamespaceSearch, ",")) + if err != nil { + klog.Error(err) + return err + } + + namespaceCreateTimeMap := make(map[string]time.Time) + for _, ns := range iNamespaces { + podLogs := authorizer.AttributesRecord{ + User: user, + Verb: "get", + APIGroup: "", + APIVersion: "v1", + Namespace: ns.Name, + Resource: "pods", + Subresource: "log", + ResourceRequest: true, + } + decision, _, err := t.authorizer.Authorize(podLogs) + if err != nil { + klog.Error(err) + return err + } + if decision == authorizer.DecisionAllow { + namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time + } + } + + sf := loggingclient.SearchFilter{ + NamespaceFilter: namespaceCreateTimeMap, + WorkloadSearch: stringutils.Split(query.WorkloadSearch, ","), + WorkloadFilter: stringutils.Split(query.WorkloadFilter, ","), + PodSearch: stringutils.Split(query.PodSearch, ","), + PodFilter: stringutils.Split(query.PodFilter, ","), + ContainerSearch: stringutils.Split(query.ContainerSearch, ","), + ContainerFilter: stringutils.Split(query.ContainerFilter, ","), + LogSearch: stringutils.Split(query.LogSearch, ","), + Starttime: query.StartTime, + Endtime: query.EndTime, + } + + if len(namespaceCreateTimeMap) == 0 { + return nil + } else { + return t.lo.ExportLogs(sf, writer) + } +} + func contains(objects []runtime.Object, object runtime.Object) bool { for _, item := range objects { if item == object { diff --git a/pkg/models/tenant/tenent_test.go b/pkg/models/tenant/tenent_test.go index 4df46198b..765c437b3 100644 --- a/pkg/models/tenant/tenent_test.go +++ b/pkg/models/tenant/tenent_test.go @@ -328,5 +328,5 @@ func prepare() Interface { RoleBindings().Informer().GetIndexer().Add(roleBinding) } - return New(fakeInformerFactory, nil, nil, nil) + return New(fakeInformerFactory, nil, nil, nil, nil) } diff --git a/pkg/simple/client/logging/elasticsearch/api_body.go b/pkg/simple/client/logging/elasticsearch/api_body.go index 72b9d6fed..aea26dc72 100644 --- a/pkg/simple/client/logging/elasticsearch/api_body.go +++ b/pkg/simple/client/logging/elasticsearch/api_body.go @@ -2,6 +2,7 @@ package elasticsearch import ( "fmt" + "github.com/json-iterator/go" "k8s.io/klog" "kubesphere.io/kubesphere/pkg/simple/client/logging" "time" @@ -13,6 +14,9 @@ const ( replicaSetSuffixMaxLength = 11 // max 10 characters + 1 hyphen ) +// TODO: elastic/go-elasticsearch is working on Query DSL support. +// See https://github.com/elastic/go-elasticsearch/issues/42. +// We need refactor our query body builder when that is ready. type bodyBuilder struct { Body } @@ -22,75 +26,9 @@ func newBodyBuilder() *bodyBuilder { } func (bb *bodyBuilder) bytes() ([]byte, error) { - return json.Marshal(bb.Body) + return jsoniter.Marshal(bb.Body) } -// The mainBody func builds api body for query. -// TODO: Should use an elegant pakcage for building query body, but `elastic/go-elasticsearch` doesn't provide it currently. -// -// Example: -// GET kapis/logging.kubesphere.io/v1alpha2/cluster?start_time=0&end_time=156576063993&namespaces=kubesphere-system&pod_query=ks-apiserver -// ----- -//{ -// "from":0, -// "size":10, -// "sort":[ -// { -// "time": "desc" -// } -// ], -// "query":{ -// "bool":{ -// "filter":[ -// { -// "bool":{ -// "should":[ -// { -// "bool":{ -// "filter":[ -// { -// "match_phrase":{ -// "kubernetes.namespace_name.keyword":"kubesphere-system" -// } -// }, -// { -// "range":{ -// "time":{ -// "gte":"1572315987000" -// } -// } -// } -// ] -// } -// } -// ], -// "minimum_should_match":1 -// } -// }, -// { -// "bool":{ -// "should":[ -// { -// "match_phrase_prefix":{ -// "kubernetes.pod_name":"ks-apiserver" -// } -// } -// ], -// "minimum_should_match":1 -// } -// }, -// { -// "range":{ -// "time":{ -// "gte":"0", -// "lte":"156576063993" -// } -// } -// } -// ] -// } -// } -//} func (bb *bodyBuilder) mainBool(sf logging.SearchFilter) *bodyBuilder { var ms []Match @@ -207,10 +145,6 @@ func (bb *bodyBuilder) cardinalityAggregation() *bodyBuilder { } func (bb *bodyBuilder) dateHistogramAggregation(interval string) *bodyBuilder { - if interval == "" { - interval = "15m" - } - bb.Body.Aggs = &Aggs{ DateHistogramAggregation: &DateHistogramAggregation{ &DateHistogram{ @@ -232,12 +166,8 @@ func (bb *bodyBuilder) size(n int64) *bodyBuilder { return bb } -func (bb *bodyBuilder) sort(o string) *bodyBuilder { - if o != "asc" { - o = "desc" - } - - bb.Sorts = []map[string]string{{"time": o}} +func (bb *bodyBuilder) sort(order string) *bodyBuilder { + bb.Sorts = []map[string]string{{"time": order}} return bb } @@ -268,7 +198,7 @@ func podNameRegexp(workloadName string) string { func parseResponse(body []byte) (Response, error) { var res Response - err := json.Unmarshal(body, &res) + err := jsoniter.Unmarshal(body, &res) if err != nil { klog.Error(err) return Response{}, err diff --git a/pkg/simple/client/logging/elasticsearch/api_body_test.go b/pkg/simple/client/logging/elasticsearch/api_body_test.go index 6e160f358..8e13a6ccc 100644 --- a/pkg/simple/client/logging/elasticsearch/api_body_test.go +++ b/pkg/simple/client/logging/elasticsearch/api_body_test.go @@ -1,50 +1,117 @@ package elasticsearch import ( + "fmt" "github.com/google/go-cmp/cmp" "kubesphere.io/kubesphere/pkg/simple/client/logging" + "reflect" "testing" + "time" ) -func TestCardinalityAggregation(t *testing.T) { - var test = struct { - description string - searchFilter logging.SearchFilter - expected *bodyBuilder +func TestMainBool(t *testing.T) { + var tests = []struct { + filter logging.SearchFilter + expected string }{ - description: "add cardinality aggregation", - searchFilter: logging.SearchFilter{ - LogSearch: []string{"info"}, + { + filter: logging.SearchFilter{ + NamespaceFilter: map[string]time.Time{ + "default": time.Unix(1589981934, 0), + }, + }, + expected: "api_body_1.json", }, - expected: &bodyBuilder{Body{ - Query: &Query{ - Bool: Bool{ - Filter: []Match{ - { - Bool: &Bool{ - Should: []Match{ - { - MatchPhrasePrefix: map[string]string{"log": "info"}, - }, - }, - MinimumShouldMatch: 1, - }, - }, - }, - }, + { + filter: logging.SearchFilter{ + WorkloadFilter: []string{"mysql"}, + Starttime: time.Unix(1589980934, 0), + Endtime: time.Unix(1589981934, 0), }, - Aggs: &Aggs{ - CardinalityAggregation: &CardinalityAggregation{ - Cardinality: &Cardinality{Field: "kubernetes.docker_id.keyword"}, - }, + expected: "api_body_2.json", + }, + { + filter: logging.SearchFilter{ + PodFilter: []string{"mysql"}, + PodSearch: []string{"mysql-a8w3s-10945j"}, + LogSearch: []string{"info"}, }, - }}, + expected: "api_body_3.json", + }, + { + filter: logging.SearchFilter{ + ContainerFilter: []string{"mysql-1"}, + ContainerSearch: []string{"mysql-3"}, + }, + expected: "api_body_4.json", + }, } - t.Run(test.description, func(t *testing.T) { - body := newBodyBuilder().mainBool(test.searchFilter).cardinalityAggregation() - if diff := cmp.Diff(body, test.expected); diff != "" { - t.Fatalf("%T differ (-got, +want): %s", test.expected, diff) - } - }) + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + var expected Body + err := JsonFromFile(test.expected, &expected) + if err != nil { + t.Fatal(err) + } + + result := newBodyBuilder().mainBool(test.filter).Body + + if diff := cmp.Diff(result, expected); diff != "" { + fmt.Printf("%T differ (-got, +want): %s", expected, diff) + } + }) + } +} + +func TestCardinalityAggregation(t *testing.T) { + var tests = []struct { + expected string + }{ + { + expected: "api_body_5.json", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + var expected Body + err := JsonFromFile(test.expected, &expected) + if err != nil { + t.Fatal(err) + } + + result := newBodyBuilder().cardinalityAggregation().Body + + if !reflect.DeepEqual(result, expected) { + t.Fatalf("expected: %v, but got %v", expected, result) + } + }) + } +} + +func TestDateHistogramAggregation(t *testing.T) { + var tests = []struct { + expected string + }{ + { + expected: "api_body_6.json", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + var expected Body + err := JsonFromFile(test.expected, &expected) + if err != nil { + t.Fatal(err) + } + + result := newBodyBuilder().dateHistogramAggregation("15m").Body + + if !reflect.DeepEqual(result, expected) { + t.Fatalf("expected: %v, but got %v", expected, result) + } + }) + } } diff --git a/pkg/simple/client/logging/elasticsearch/elasticsearch.go b/pkg/simple/client/logging/elasticsearch/elasticsearch.go index 30a3b2619..0b3d22fe3 100644 --- a/pkg/simple/client/logging/elasticsearch/elasticsearch.go +++ b/pkg/simple/client/logging/elasticsearch/elasticsearch.go @@ -4,12 +4,12 @@ import ( "bytes" "context" "fmt" - jsoniter "github.com/json-iterator/go" + "github.com/json-iterator/go" "io" "kubesphere.io/kubesphere/pkg/simple/client/logging" - v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5" - v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6" - v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7" "kubesphere.io/kubesphere/pkg/utils/stringutils" "strings" ) @@ -20,8 +20,6 @@ const ( ElasticV7 = "7" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary - // Elasticsearch implement logging interface type Elasticsearch struct { c client @@ -29,8 +27,7 @@ type Elasticsearch struct { // versioned es client interface type client interface { - // Perform Search API - Search(body []byte) ([]byte, error) + Search(body []byte, scroll bool) ([]byte, error) Scroll(id string) ([]byte, error) ClearScroll(id string) GetTotalHitCount(v interface{}) int64 @@ -83,11 +80,10 @@ func detectVersionMajor(host string) (string, error) { if err != nil { return "", err } - defer res.Body.Close() var b map[string]interface{} - if err = json.NewDecoder(res.Body).Decode(&b); err != nil { + if err = jsoniter.NewDecoder(res.Body).Decode(&b); err != nil { return "", err } if res.IsError() { @@ -116,7 +112,7 @@ func (es Elasticsearch) GetCurrentStats(sf logging.SearchFilter) (logging.Statis return logging.Statistics{}, err } - b, err := es.c.Search(body) + b, err := es.c.Search(body, true) if err != nil { return logging.Statistics{}, err } @@ -142,7 +138,7 @@ func (es Elasticsearch) CountLogsByInterval(sf logging.SearchFilter, interval st return logging.Histogram{}, err } - b, err := es.c.Search(body) + b, err := es.c.Search(body, false) if err != nil { return logging.Histogram{}, err } @@ -174,7 +170,7 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string return logging.Logs{}, err } - b, err := es.c.Search(body) + b, err := es.c.Search(body, false) if err != nil { return logging.Logs{}, err } @@ -200,33 +196,50 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error { var id string - var from int64 = 0 - var size int64 = 1000 + var data []string - res, err := es.SearchLogs(sf, from, size, "desc") - defer es.ClearScroll(id) + // Initial Search + body, err := newBodyBuilder(). + mainBool(sf). + from(0). + size(1000). + sort("desc"). + bytes() if err != nil { return err } - if res.Records == nil || len(res.Records) == 0 { + b, err := es.c.Search(body, true) + defer es.ClearScroll(id) + if err != nil { + return err + } + res, err := parseResponse(b) + if err != nil { + return err + } + + id = res.ScrollId + for _, hit := range res.AllHits { + data = append(data, hit.Log) + } + if len(data) == 0 { return nil } // limit to retrieve max 100k records for i := 0; i < 100; i++ { - res, id, err = es.scroll(id) + data, id, err = es.scroll(id) if err != nil { return err } - - if res.Records == nil || len(res.Records) == 0 { + if len(data) == 0 { return nil } output := new(bytes.Buffer) - for _, r := range res.Records { - output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(r.Log))) + for _, l := range data { + output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(l))) } _, err = io.Copy(w, output) if err != nil { @@ -236,24 +249,22 @@ func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error { return nil } -func (es *Elasticsearch) scroll(id string) (logging.Logs, string, error) { +func (es *Elasticsearch) scroll(id string) ([]string, string, error) { b, err := es.c.Scroll(id) if err != nil { - return logging.Logs{}, id, err + return nil, id, err } res, err := parseResponse(b) if err != nil { - return logging.Logs{}, id, err + return nil, id, err } - var l logging.Logs + var data []string for _, hit := range res.AllHits { - l.Records = append(l.Records, logging.Record{ - Log: hit.Log, - }) + data = append(data, hit.Log) } - return l, res.ScrollId, nil + return data, res.ScrollId, nil } func (es *Elasticsearch) ClearScroll(id string) { diff --git a/pkg/simple/client/logging/elasticsearch/elasticsearch_test.go b/pkg/simple/client/logging/elasticsearch/elasticsearch_test.go index 2907ee159..5c43370c3 100644 --- a/pkg/simple/client/logging/elasticsearch/elasticsearch_test.go +++ b/pkg/simple/client/logging/elasticsearch/elasticsearch_test.go @@ -1,70 +1,46 @@ package elasticsearch import ( + "fmt" "github.com/google/go-cmp/cmp" + "github.com/json-iterator/go" + "io/ioutil" "kubesphere.io/kubesphere/pkg/simple/client/logging" - v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5" - v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6" - v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6" + "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7" "net/http" "net/http/httptest" "testing" - "time" ) -func MockElasticsearchService(pattern string, fakeResp string) *httptest.Server { - mux := http.NewServeMux() - mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) { - res.Write([]byte(fakeResp)) - }) - return httptest.NewServer(mux) -} - func TestDetectVersionMajor(t *testing.T) { var tests = []struct { - description string - fakeResp string - expected string - expectedError bool + fakeResp string + expected string }{ { - description: "detect es 6.x version number", - fakeResp: `{ - "name" : "elasticsearch-logging-data-0", - "cluster_name" : "elasticsearch", - "cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg", - "version" : { - "number" : "6.7.0", - "build_flavor" : "oss", - "build_type" : "docker", - "build_hash" : "8453f77", - "build_date" : "2019-03-21T15:32:29.844721Z", - "build_snapshot" : false, - "lucene_version" : "7.7.0", - "minimum_wire_compatibility_version" : "5.6.0", - "minimum_index_compatibility_version" : "5.0.0" - }, - "tagline" : "You Know, for Search" -}`, - expected: ElasticV6, - expectedError: false, + fakeResp: "es6_detect_version_major_200.json", + expected: ElasticV6, + }, + { + fakeResp: "es7_detect_version_major_200.json", + expected: ElasticV7, }, } - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - es := MockElasticsearchService("/", test.fakeResp) + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + es := mockElasticsearchService("/", test.fakeResp, http.StatusOK) defer es.Close() - v, err := detectVersionMajor(es.URL) - if err == nil && test.expectedError { - t.Fatalf("expected error while got nothing") - } else if err != nil && !test.expectedError { + result, err := detectVersionMajor(es.URL) + if err != nil { t.Fatal(err) } - if v != test.expected { - t.Fatalf("expected get version %s, but got %s", test.expected, v) + if diff := cmp.Diff(result, test.expected); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", test.expected, diff) } }) } @@ -72,297 +48,202 @@ func TestDetectVersionMajor(t *testing.T) { func TestGetCurrentStats(t *testing.T) { var tests = []struct { - description string - searchFilter logging.SearchFilter - fakeVersion string - fakeResp string - expected logging.Statistics - expectedError bool + fakeVersion string + fakeResp string + fakeCode int + expected logging.Statistics + expectedErr string }{ { - description: "[es 6.x] run as admin", - searchFilter: logging.SearchFilter{}, - fakeVersion: ElasticV6, - fakeResp: `{ - "took": 171, - "timed_out": false, - "_shards": { - "total": 10, - "successful": 10, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 241222, - "max_score": 1.0, - "hits": [ - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "Hn1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:29.015Z", - "log": " value: \"hostpath\"\n", - "time": "2020-02-28T19:25:29.015492329Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "I31GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:33.103Z", - "log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n", - "time": "2020-02-28T19:25:33.103075891Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "JX1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:33.113Z", - "log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n", - "time": "2020-02-28T19:25:33.113110332Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "Kn1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:34.168Z", - "log": " value: \"hostpath\"\n", - "time": "2020-02-28T19:25:34.168983384Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "LH1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:34.168Z", - "log": " value: \"/var/openebs/local/\"\n", - "time": "2020-02-28T19:25:34.168997393Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "NX1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:25:42.868Z", - "log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n", - "time": "2020-02-28T19:25:42.868578188Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "Q31GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:26:13.881Z", - "log": "- name: BasePath\n", - "time": "2020-02-28T19:26:13.881180681Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "S31GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:26:14.597Z", - "log": " value: \"/var/openebs/local/\"\n", - "time": "2020-02-28T19:26:14.597702238Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "TH1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:26:14.597Z", - "log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n", - "time": "2020-02-28T19:26:14.597708432Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - }, - { - "_index": "ks-logstash-log-2020.02.28", - "_type": "flb_type", - "_id": "UX1GjXABMO5aQxyNsyxy", - "_score": 1.0, - "_source": { - "@timestamp": "2020-02-28T19:26:15.920Z", - "log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n", - "time": "2020-02-28T19:26:15.920650572Z", - "kubernetes": { - "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", - "namespace_name": "kube-system", - "host": "ks-allinone", - "container_name": "openebs-localpv-provisioner", - "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", - "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" - } - } - } - ] - }, - "aggregations": { - "container_count": { - "value": 93 - } - } -}`, + fakeVersion: ElasticV6, + fakeResp: "es6_get_current_stats_200.json", + fakeCode: http.StatusOK, expected: logging.Statistics{ Containers: 93, Logs: 241222, }, - expectedError: false, }, { - description: "[es 6.x] index not found", - searchFilter: logging.SearchFilter{ - NamespaceFilter: map[string]time.Time{ - "workspace-1-project-a": time.Unix(1582000000, 0), - "workspace-1-project-b": time.Unix(1582333333, 0), - }, - }, fakeVersion: ElasticV6, - fakeResp: `{ - "error": { - "root_cause": [ - { - "type": "index_not_found_exception", - "reason": "no such index", - "resource.type": "index_or_alias", - "resource.id": "ks-lsdfsdfsdfs", - "index_uuid": "_na_", - "index": "ks-lsdfsdfsdfs" - } - ], - "type": "index_not_found_exception", - "reason": "no such index", - "resource.type": "index_or_alias", - "resource.id": "ks-lsdfsdfsdfs", - "index_uuid": "_na_", - "index": "ks-lsdfsdfsdfs" - }, - "status": 404 -}`, + fakeResp: "es6_get_current_stats_404.json", + fakeCode: http.StatusNotFound, + expectedErr: "type: index_not_found_exception, reason: no such index", + }, + { + fakeVersion: ElasticV7, + fakeResp: "es7_get_current_stats_200.json", + fakeCode: http.StatusOK, expected: logging.Statistics{ - Containers: 0, - Logs: 0, + Containers: 48, + Logs: 9726, }, - expectedError: true, + }, + { + fakeVersion: ElasticV7, + fakeResp: "es7_get_current_stats_404.json", + fakeCode: http.StatusNotFound, + expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-2020.05.2]", }, } - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - es := MockElasticsearchService("/", test.fakeResp) - defer es.Close() + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode) + defer srv.Close() - clientv5 := Elasticsearch{c: v5.New(es.URL, "ks-logstash-log")} - clientv6 := Elasticsearch{c: v6.New(es.URL, "ks-logstash-log")} - clientv7 := Elasticsearch{c: v7.New(es.URL, "ks-logstash-log")} + es := newElasticsearchClient(srv, test.fakeVersion) - var stats logging.Statistics - var err error - switch test.fakeVersion { - case ElasticV5: - stats, err = clientv5.GetCurrentStats(test.searchFilter) - case ElasticV6: - stats, err = clientv6.GetCurrentStats(test.searchFilter) - case ElasticV7: - stats, err = clientv7.GetCurrentStats(test.searchFilter) + result, err := es.GetCurrentStats(logging.SearchFilter{}) + if test.expectedErr != "" { + if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff) + } } - - if err != nil && !test.expectedError { - t.Fatal(err) - } else if diff := cmp.Diff(stats, test.expected); diff != "" { + if diff := cmp.Diff(result, test.expected); diff != "" { t.Fatalf("%T differ (-got, +want): %s", test.expected, diff) } }) } } + +func TestCountLogsByInterval(t *testing.T) { + var tests = []struct { + fakeVersion string + fakeResp string + fakeCode int + expected logging.Histogram + expectedErr string + }{ + { + fakeVersion: ElasticV7, + fakeResp: "es7_count_logs_by_interval_200.json", + fakeCode: http.StatusOK, + expected: logging.Histogram{ + Total: 10000, + Buckets: []logging.Bucket{ + { + Time: 1589644800000, + Count: 410, + }, + { + Time: 1589646600000, + Count: 7465, + }, + { + Time: 1589648400000, + Count: 12790, + }, + }, + }, + }, + { + fakeVersion: ElasticV7, + fakeResp: "es7_count_logs_by_interval_400.json", + fakeCode: http.StatusBadRequest, + expectedErr: "type: search_phase_execution_exception, reason: all shards failed", + }, + { + fakeVersion: ElasticV7, + fakeResp: "es7_count_logs_by_interval_404.json", + fakeCode: http.StatusNotFound, + expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-20]", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode) + defer srv.Close() + + es := newElasticsearchClient(srv, test.fakeVersion) + + result, err := es.CountLogsByInterval(logging.SearchFilter{}, "15m") + if test.expectedErr != "" { + if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff) + } + } + if diff := cmp.Diff(result, test.expected); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", test.expected, diff) + } + }) + } +} + +func TestSearchLogs(t *testing.T) { + var tests = []struct { + fakeVersion string + fakeResp string + fakeCode int + expected string + expectedErr string + }{ + { + fakeVersion: ElasticV7, + fakeResp: "es7_search_logs_200.json", + fakeCode: http.StatusOK, + expected: "es7_search_logs_200_result.json", + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + var expected logging.Logs + err := JsonFromFile(test.expected, &expected) + if err != nil { + t.Fatal(err) + } + + srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode) + defer srv.Close() + + es := newElasticsearchClient(srv, test.fakeVersion) + + result, err := es.SearchLogs(logging.SearchFilter{}, 0, 10, "asc") + if test.expectedErr != "" { + if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff) + } + } + if diff := cmp.Diff(result, expected); diff != "" { + t.Fatalf("%T differ (-got, +want): %s", expected, diff) + } + }) + } +} + +func mockElasticsearchService(pattern, fakeResp string, fakeCode int) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) { + b, _ := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", fakeResp)) + res.WriteHeader(fakeCode) + res.Write(b) + }) + return httptest.NewServer(mux) +} + +func newElasticsearchClient(srv *httptest.Server, version string) Elasticsearch { + var es Elasticsearch + switch version { + case ElasticV5: + es = Elasticsearch{c: v5.New(srv.URL, "ks-logstash-log")} + case ElasticV6: + es = Elasticsearch{c: v6.New(srv.URL, "ks-logstash-log")} + case ElasticV7: + es = Elasticsearch{c: v7.New(srv.URL, "ks-logstash-log")} + } + return es +} + +func JsonFromFile(expectedFile string, expectedJsonPtr interface{}) error { + json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile)) + if err != nil { + return err + } + err = jsoniter.Unmarshal(json, expectedJsonPtr) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json new file mode 100644 index 000000000..7a70dc874 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json @@ -0,0 +1,33 @@ +{ + "query":{ + "bool":{ + "filter":[ + { + "bool":{ + "should":[ + { + "bool":{ + "filter":[ + { + "match_phrase":{ + "kubernetes.namespace_name.keyword":"default" + } + }, + { + "range":{ + "time":{ + "gte":"2020-05-20T21:38:54+08:00" + } + } + } + ] + } + } + ], + "minimum_should_match":1 + } + } + ] + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json new file mode 100644 index 000000000..a6de9e65d --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json @@ -0,0 +1,28 @@ +{ + "query":{ + "bool":{ + "filter":[ + { + "bool":{ + "should":[ + { + "regexp":{ + "kubernetes.pod_name.keyword":"mysql-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|mysql-[0-9]+|mysql-[a-z0-9]{5}" + } + } + ], + "minimum_should_match":1 + } + }, + { + "range":{ + "time":{ + "gte":"2020-05-20T21:22:14+08:00", + "lte":"2020-05-20T21:38:54+08:00" + } + } + } + ] + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json new file mode 100644 index 000000000..9ec247fb2 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json @@ -0,0 +1,44 @@ +{ + "query":{ + "bool":{ + "filter":[ + { + "bool":{ + "should":[ + { + "match_phrase":{ + "kubernetes.pod_name.keyword":"mysql" + } + } + ], + "minimum_should_match":1 + } + }, + { + "bool":{ + "should":[ + { + "match_phrase_prefix":{ + "kubernetes.pod_name":"mysql-a8w3s-10945j" + } + } + ], + "minimum_should_match":1 + } + }, + { + "bool":{ + "should":[ + { + "match_phrase_prefix":{ + "log":"info" + } + } + ], + "minimum_should_match":1 + } + } + ] + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json new file mode 100644 index 000000000..b8180e000 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json @@ -0,0 +1,32 @@ +{ + "query":{ + "bool":{ + "filter":[ + { + "bool":{ + "should":[ + { + "match_phrase":{ + "kubernetes.container_name.keyword":"mysql-1" + } + } + ], + "minimum_should_match":1 + } + }, + { + "bool":{ + "should":[ + { + "match_phrase_prefix":{ + "kubernetes.container_name":"mysql-3" + } + } + ], + "minimum_should_match":1 + } + } + ] + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json new file mode 100644 index 000000000..f46804d52 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json @@ -0,0 +1,9 @@ +{ + "aggs":{ + "container_count":{ + "cardinality":{ + "field":"kubernetes.docker_id.keyword" + } + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json b/pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json new file mode 100644 index 000000000..9da99500b --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json @@ -0,0 +1,10 @@ +{ + "aggs":{ + "log_count_over_time":{ + "date_histogram":{ + "field":"time", + "interval":"15m" + } + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json new file mode 100644 index 000000000..f0ffed604 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json @@ -0,0 +1,17 @@ +{ + "name" : "elasticsearch-logging-data-0", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg", + "version" : { + "number" : "6.7.0", + "build_flavor" : "oss", + "build_type" : "docker", + "build_hash" : "8453f77", + "build_date" : "2019-03-21T15:32:29.844721Z", + "build_snapshot" : false, + "lucene_version" : "7.7.0", + "minimum_wire_compatibility_version" : "5.6.0", + "minimum_index_compatibility_version" : "5.0.0" + }, + "tagline" : "You Know, for Search" +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json new file mode 100644 index 000000000..db91c786c --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json @@ -0,0 +1,211 @@ +{ + "took": 171, + "timed_out": false, + "_shards": { + "total": 10, + "successful": 10, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 241222, + "max_score": 1.0, + "hits": [ + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "Hn1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:29.015Z", + "log": " value: \"hostpath\"\n", + "time": "2020-02-28T19:25:29.015492329Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "I31GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:33.103Z", + "log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n", + "time": "2020-02-28T19:25:33.103075891Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "JX1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:33.113Z", + "log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n", + "time": "2020-02-28T19:25:33.113110332Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "Kn1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:34.168Z", + "log": " value: \"hostpath\"\n", + "time": "2020-02-28T19:25:34.168983384Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "LH1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:34.168Z", + "log": " value: \"/var/openebs/local/\"\n", + "time": "2020-02-28T19:25:34.168997393Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "NX1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:25:42.868Z", + "log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n", + "time": "2020-02-28T19:25:42.868578188Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "Q31GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:26:13.881Z", + "log": "- name: BasePath\n", + "time": "2020-02-28T19:26:13.881180681Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "S31GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:26:14.597Z", + "log": " value: \"/var/openebs/local/\"\n", + "time": "2020-02-28T19:26:14.597702238Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "TH1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:26:14.597Z", + "log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n", + "time": "2020-02-28T19:26:14.597708432Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + }, + { + "_index": "ks-logstash-log-2020.02.28", + "_type": "flb_type", + "_id": "UX1GjXABMO5aQxyNsyxy", + "_score": 1.0, + "_source": { + "@timestamp": "2020-02-28T19:26:15.920Z", + "log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n", + "time": "2020-02-28T19:26:15.920650572Z", + "kubernetes": { + "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc", + "namespace_name": "kube-system", + "host": "ks-allinone", + "container_name": "openebs-localpv-provisioner", + "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39", + "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806" + } + } + } + ] + }, + "aggregations": { + "container_count": { + "value": 93 + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json b/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json new file mode 100644 index 000000000..a930e366c --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json @@ -0,0 +1,21 @@ +{ + "error": { + "root_cause": [ + { + "type": "index_not_found_exception", + "reason": "no such index", + "resource.type": "index_or_alias", + "resource.id": "ks-lsdfsdfsdfs", + "index_uuid": "_na_", + "index": "ks-lsdfsdfsdfs" + } + ], + "type": "index_not_found_exception", + "reason": "no such index", + "resource.type": "index_or_alias", + "resource.id": "ks-lsdfsdfsdfs", + "index_uuid": "_na_", + "index": "ks-lsdfsdfsdfs" + }, + "status": 404 +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json new file mode 100644 index 000000000..ab014e8be --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json @@ -0,0 +1,230 @@ +{ + "took": 23, + "timed_out": false, + "_shards": { + "total": 2, + "successful": 2, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 10000, + "relation": "gte" + }, + "max_score": 1.0, + "hits": [ + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tRt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.608Z", + "log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.608962452Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tht2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.670Z", + "log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.670430525Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "txt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.731Z", + "log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.731865428Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "jxt2MXIBlcWZ594bpIVN", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:07.648Z", + "log": "ls: cannot access '/calico-secrets': No such file or directory\n", + "time": "2020-05-16T16:00:07.64848716Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kBt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.164Z", + "log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n", + "time": "2020-05-16T16:00:08.164003996Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kRt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "CNI plugin version: v3.7.3\n", + "time": "2020-05-16T16:00:08.350585959Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kht2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "/host/secondary-bin-dir is non-writeable, skipping\n", + "time": "2020-05-16T16:00:08.350625112Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kxt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n", + "time": "2020-05-16T16:00:08.350692011Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "lBt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.454Z", + "log": "CNI config: {\n", + "time": "2020-05-16T16:00:08.454417144Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "lRt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.454Z", + "log": " \"name\": \"cni0\",\n", + "time": "2020-05-16T16:00:08.454452649Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + } + ] + }, + "aggregations": { + "log_count_over_time": { + "buckets": [ + { + "key_as_string": "2020-05-16T16:00:00.000Z", + "key": 1589644800000, + "doc_count": 410 + }, + { + "key_as_string": "2020-05-16T16:30:00.000Z", + "key": 1589646600000, + "doc_count": 7465 + }, + { + "key_as_string": "2020-05-16T17:00:00.000Z", + "key": 1589648400000, + "doc_count": 12790 + } + ] + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json new file mode 100644 index 000000000..a430ad038 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json @@ -0,0 +1,47 @@ +{ + "error": { + "root_cause": [ + { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]" + }, + { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]" + } + ], + "type": "search_phase_execution_exception", + "reason": "all shards failed", + "phase": "query", + "grouped": true, + "failed_shards": [ + { + "shard": 0, + "index": "ks-logstash-log-2020.05.16", + "node": "Zr2OFlfeSJmK_W3Re4UBlg", + "reason": { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]" + } + }, + { + "shard": 0, + "index": "ks-logstash-log-2020.05.20", + "node": "pbGNYbV3QUuV5yJAgxbp3g", + "reason": { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]" + } + } + ], + "caused_by": { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]", + "caused_by": { + "type": "illegal_argument_exception", + "reason": "Unable to parse interval [30m0s]" + } + } + }, + "status": 400 +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json new file mode 100644 index 000000000..4d5275003 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json @@ -0,0 +1,21 @@ +{ + "error": { + "root_cause": [ + { + "type": "index_not_found_exception", + "reason": "no such index [ks-logstash-log-20]", + "resource.type": "index_or_alias", + "resource.id": "ks-logstash-log-20", + "index_uuid": "_na_", + "index": "ks-logstash-log-20" + } + ], + "type": "index_not_found_exception", + "reason": "no such index [ks-logstash-log-20]", + "resource.type": "index_or_alias", + "resource.id": "ks-logstash-log-20", + "index_uuid": "_na_", + "index": "ks-logstash-log-20" + }, + "status": 404 +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json new file mode 100644 index 000000000..b2fc43429 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json @@ -0,0 +1,17 @@ +{ + "name" : "elasticsearch-master-2", + "cluster_name" : "elasticsearch", + "cluster_uuid" : "_A-3shR0R0i-2M9CzOWP8g", + "version" : { + "number" : "7.7.0", + "build_flavor" : "default", + "build_type" : "docker", + "build_hash" : "81a1e9eda8e6183f5237786246f6dced26a10eaf", + "build_date" : "2020-05-12T02:01:37.602180Z", + "build_snapshot" : false, + "lucene_version" : "8.5.1", + "minimum_wire_compatibility_version" : "6.8.0", + "minimum_index_compatibility_version" : "6.0.0-beta1" + }, + "tagline" : "You Know, for Search" +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json new file mode 100644 index 000000000..b4b55c731 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json @@ -0,0 +1,214 @@ +{ + "took": 1455, + "timed_out": false, + "_shards": { + "total": 2, + "successful": 2, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 9726, + "relation": "eq" + }, + "max_score": 1.0, + "hits": [ + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tRt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.608Z", + "log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.608962452Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tht2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.670Z", + "log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.670430525Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "txt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.731Z", + "log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.731865428Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "jxt2MXIBlcWZ594bpIVN", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:07.648Z", + "log": "ls: cannot access '/calico-secrets': No such file or directory\n", + "time": "2020-05-16T16:00:07.64848716Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kBt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.164Z", + "log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n", + "time": "2020-05-16T16:00:08.164003996Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kRt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "CNI plugin version: v3.7.3\n", + "time": "2020-05-16T16:00:08.350585959Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kht2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "/host/secondary-bin-dir is non-writeable, skipping\n", + "time": "2020-05-16T16:00:08.350625112Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "kxt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.350Z", + "log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n", + "time": "2020-05-16T16:00:08.350692011Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "lBt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.454Z", + "log": "CNI config: {\n", + "time": "2020-05-16T16:00:08.454417144Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "lRt2MXIBlcWZ594bpIV0", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:08.454Z", + "log": " \"name\": \"cni0\",\n", + "time": "2020-05-16T16:00:08.454452649Z", + "kubernetes": { + "pod_name": "calico-node-gc7pp", + "namespace_name": "kube-system", + "host": "master0", + "container_name": "install-cni", + "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a", + "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6" + } + } + } + ] + }, + "aggregations": { + "container_count": { + "value": 48 + } + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json new file mode 100644 index 000000000..df509db55 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json @@ -0,0 +1,21 @@ +{ + "error": { + "root_cause": [ + { + "type": "index_not_found_exception", + "reason": "no such index [ks-logstash-log-2020.05.2]", + "resource.type": "index_or_alias", + "resource.id": "ks-logstash-log-2020.05.2", + "index_uuid": "_na_", + "index": "ks-logstash-log-2020.05.2" + } + ], + "type": "index_not_found_exception", + "reason": "no such index [ks-logstash-log-2020.05.2]", + "resource.type": "index_or_alias", + "resource.id": "ks-logstash-log-2020.05.2", + "index_uuid": "_na_", + "index": "ks-logstash-log-2020.05.2" + }, + "status": 404 +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json new file mode 100644 index 000000000..62806771c --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json @@ -0,0 +1,76 @@ +{ + "took": 772, + "timed_out": false, + "_shards": { + "total": 2, + "successful": 2, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 10000, + "relation": "gte" + }, + "max_score": 1.0, + "hits": [ + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tRt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.608Z", + "log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.608962452Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "tht2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.670Z", + "log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n", + "time": "2020-05-16T16:00:42.670430525Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "kubesphere-system", + "host": "master0", + "container_name": "config-init", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + }, + { + "_index": "ks-logstash-log-2020.05.16", + "_type": "flb_type", + "_id": "txt2MXIBlcWZ594bqIUO", + "_score": 1.0, + "_source": { + "@timestamp": "2020-05-16T16:00:42.731Z", + "log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n", + "time": "2020-05-16T16:00:42.731865428Z", + "kubernetes": { + "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj", + "namespace_name": "istio-system", + "host": "node0", + "container_name": "mixer", + "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0", + "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336" + } + } + } + ] + } +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json b/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json new file mode 100644 index 000000000..94313c347 --- /dev/null +++ b/pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json @@ -0,0 +1,29 @@ +{ + "total": 10000, + "records": [ + { + "time": "2020-05-16T16:00:42.608962452Z", + "log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n", + "namespace": "kubesphere-system", + "pod": "redis-ha-haproxy-ffb8d889d-8x9kj", + "container": "config-init", + "host": "master0" + }, + { + "time": "2020-05-16T16:00:42.670430525Z", + "log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n", + "namespace": "kubesphere-system", + "pod": "redis-ha-haproxy-ffb8d889d-8x9kj", + "container": "config-init", + "host": "master0" + }, + { + "time": "2020-05-16T16:00:42.731865428Z", + "log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n", + "namespace": "istio-system", + "pod": "redis-ha-haproxy-ffb8d889d-8x9kj", + "container": "mixer", + "host": "node0" + } + ] +} \ No newline at end of file diff --git a/pkg/simple/client/logging/elasticsearch/versions/v5/v5.go b/pkg/simple/client/logging/elasticsearch/versions/v5/v5.go index e53a0c03c..b3344df9a 100644 --- a/pkg/simple/client/logging/elasticsearch/versions/v5/v5.go +++ b/pkg/simple/client/logging/elasticsearch/versions/v5/v5.go @@ -29,12 +29,17 @@ func New(address string, index string) *Elastic { return &Elastic{client: client, index: index} } -func (e *Elastic) Search(body []byte) ([]byte, error) { - response, err := e.client.Search( +func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) { + opts := []func(*esapi.SearchRequest){ e.client.Search.WithContext(context.Background()), e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)), e.client.Search.WithBody(bytes.NewBuffer(body)), - e.client.Search.WithScroll(time.Minute)) + } + if scroll { + opts = append(opts, e.client.Search.WithScroll(time.Minute)) + } + + response, err := e.client.Search(opts...) if err != nil { return nil, err } diff --git a/pkg/simple/client/logging/elasticsearch/versions/v6/v6.go b/pkg/simple/client/logging/elasticsearch/versions/v6/v6.go index 8c584aadb..198acdb7b 100644 --- a/pkg/simple/client/logging/elasticsearch/versions/v6/v6.go +++ b/pkg/simple/client/logging/elasticsearch/versions/v6/v6.go @@ -29,12 +29,17 @@ func New(address string, index string) *Elastic { return &Elastic{Client: client, index: index} } -func (e *Elastic) Search(body []byte) ([]byte, error) { - response, err := e.Client.Search( +func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) { + opts := []func(*esapi.SearchRequest){ e.Client.Search.WithContext(context.Background()), e.Client.Search.WithIndex(fmt.Sprintf("%s*", e.index)), e.Client.Search.WithBody(bytes.NewBuffer(body)), - e.Client.Search.WithScroll(time.Minute)) + } + if scroll { + opts = append(opts, e.Client.Search.WithScroll(time.Minute)) + } + + response, err := e.Client.Search(opts...) if err != nil { return nil, err } diff --git a/pkg/simple/client/logging/elasticsearch/versions/v7/v7.go b/pkg/simple/client/logging/elasticsearch/versions/v7/v7.go index 1051ffc66..4ff3d1e70 100644 --- a/pkg/simple/client/logging/elasticsearch/versions/v7/v7.go +++ b/pkg/simple/client/logging/elasticsearch/versions/v7/v7.go @@ -29,13 +29,17 @@ func New(address string, index string) *Elastic { return &Elastic{client: client, index: index} } -func (e *Elastic) Search(body []byte) ([]byte, error) { - response, err := e.client.Search( +func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) { + opts := []func(*esapi.SearchRequest){ e.client.Search.WithContext(context.Background()), e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)), - e.client.Search.WithTrackTotalHits(true), e.client.Search.WithBody(bytes.NewBuffer(body)), - e.client.Search.WithScroll(time.Minute)) + } + if scroll { + opts = append(opts, e.client.Search.WithScroll(time.Minute)) + } + + response, err := e.client.Search(opts...) if err != nil { return nil, err } diff --git a/pkg/simple/client/logging/interface.go b/pkg/simple/client/logging/interface.go index 69a20909d..0ada45cb0 100644 --- a/pkg/simple/client/logging/interface.go +++ b/pkg/simple/client/logging/interface.go @@ -6,13 +6,9 @@ import ( ) type Interface interface { - // Current stats about log store, eg. total number of logs and containers GetCurrentStats(sf SearchFilter) (Statistics, error) - CountLogsByInterval(sf SearchFilter, interval string) (Histogram, error) - SearchLogs(sf SearchFilter, from, size int64, order string) (Logs, error) - ExportLogs(sf SearchFilter, w io.Writer) error } diff --git a/tools/cmd/doc-gen/main.go b/tools/cmd/doc-gen/main.go index 32843a7d9..b4d7cdb6c 100644 --- a/tools/cmd/doc-gen/main.go +++ b/tools/cmd/doc-gen/main.go @@ -37,7 +37,6 @@ import ( "kubesphere.io/kubesphere/pkg/informers" devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2" iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2" - loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2" monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3" networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2" openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1" @@ -113,13 +112,12 @@ func generateSwaggerJson() []byte { urlruntime.Must(devopsv1alpha2.AddToContainer(container, informerFactory.KubeSphereSharedInformerFactory(), &fake.Devops{}, nil, clientsets.KubeSphere(), fakes3.NewFakeS3())) urlruntime.Must(iamv1alpha2.AddToContainer(container, im.NewOperator(clientsets.KubeSphere(), informerFactory), am.NewReadOnlyOperator(informerFactory), authoptions.NewAuthenticateOptions())) - urlruntime.Must(loggingv1alpha2.AddToContainer(container, clientsets, nil)) urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil, informerFactory, nil)) urlruntime.Must(openpitrixv1.AddToContainer(container, informerFactory, nil)) urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes())) urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory)) urlruntime.Must(resourcesv1alpha3.AddToContainer(container, informerFactory)) - urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil)) + urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil)) urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil)) urlruntime.Must(metricsv1alpha2.AddToContainer(container)) urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))