logging: integrate new IAM

Signed-off-by: huanggze <loganhuang@yunify.com>

pkg/api/logging/v1alpha2/types.go

@@ -1,9 +1,103 @@
 package v1alpha2
 
-import "kubesphere.io/kubesphere/pkg/simple/client/logging"
+import (
+	"github.com/emicklei/go-restful"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging"
+	"strconv"
+	"time"
+)
+
+const (
+	OperationStatistics = "statistics"
+	OperationHistogram  = "histogram"
+	OperationQuery      = "query"
+	OperationExport     = "export"
+
+	DefaultInterval = "15m"
+	DefaultSize     = 10
+	OrderAscending  = "asc"
+	OrderDescending = "desc"
+)
 
 type APIResponse struct {
 	Logs       *logging.Logs       `json:"query,omitempty" description:"query results"`
 	Statistics *logging.Statistics `json:"statistics,omitempty" description:"statistics results"`
 	Histogram  *logging.Histogram  `json:"histogram,omitempty" description:"histogram results"`
 }
+
+type Query struct {
+	Operation       string
+	WorkspaceFilter string
+	WorkspaceSearch string
+	NamespaceFilter string
+	NamespaceSearch string
+	WorkloadFilter  string
+	WorkloadSearch  string
+	PodFilter       string
+	PodSearch       string
+	ContainerFilter string
+	ContainerSearch string
+	LogSearch       string
+	StartTime       time.Time
+	EndTime         time.Time
+	Interval        string
+	Sort            string
+	From            int64
+	Size            int64
+}
+
+func ParseQueryParameter(req *restful.Request) (*Query, error) {
+	var q Query
+	q.Operation = req.QueryParameter("operation")
+	q.WorkspaceFilter = req.QueryParameter("workspaces")
+	q.WorkspaceSearch = req.QueryParameter("workspace_query")
+	q.NamespaceFilter = req.QueryParameter("namespaces")
+	q.NamespaceSearch = req.QueryParameter("namespace_query")
+	q.WorkloadFilter = req.QueryParameter("workloads")
+	q.WorkloadSearch = req.QueryParameter("workload_query")
+	q.PodFilter = req.QueryParameter("pods")
+	q.PodSearch = req.QueryParameter("pod_query")
+	q.ContainerFilter = req.QueryParameter("containers")
+	q.ContainerSearch = req.QueryParameter("container_query")
+	q.LogSearch = req.QueryParameter("log_query")
+
+	if q.Operation == "" {
+		q.Operation = OperationQuery
+	}
+
+	if tstr := req.QueryParameter("start_time"); tstr != "" {
+		sec, err := strconv.ParseInt(tstr, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		q.StartTime = time.Unix(sec, 0)
+	}
+	if tstr := req.QueryParameter("end_time"); tstr != "" {
+		sec, err := strconv.ParseInt(tstr, 10, 64)
+		if err != nil {
+			return nil, err
+		}
+		q.EndTime = time.Unix(sec, 0)
+	}
+
+	switch q.Operation {
+	case OperationHistogram:
+		q.Interval = req.QueryParameter("interval")
+		if q.Interval == "" {
+			q.Interval = DefaultInterval
+		}
+	case OperationQuery:
+		q.From, _ = strconv.ParseInt(req.QueryParameter("from"), 10, 64)
+		size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64)
+		if err != nil {
+			size = DefaultSize
+		}
+		q.Size = size
+		q.Sort = req.QueryParameter("sort")
+		if q.Sort != OrderAscending {
+			q.Sort = OrderDescending
+		}
+	}
+
+	return &q, nil
+}

pkg/apiserver/apiserver.go

@@ -33,7 +33,6 @@ import (
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
 	iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
-	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
 	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
 	networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
 	notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1"
@@ -149,7 +148,6 @@ func (s *APIServer) PrepareRun() error {
 func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
 	urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory))
-	urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient))
 	urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient))
 	urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient))
 	urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
@@ -157,7 +155,7 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory,
 		s.KubernetesClient.Master()))
 	urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
-		s.KubernetesClient.KubeSphere(), s.EventsClient))
+		s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient))
 	urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config()))
 	urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
 		s.InformerFactory.KubernetesSharedInformerFactory(),

pkg/kapis/logging/group.go

@@ -1,18 +0,0 @@
-/*
-Copyright 2019 The KubeSphere authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package logging contains logging API versions
-package logging

pkg/kapis/logging/v1alpha2/handler.go

@@ -1,156 +0,0 @@
-package v1alpha2
-
-import (
-	"github.com/emicklei/go-restful"
-	"kubesphere.io/kubesphere/pkg/api"
-	"kubesphere.io/kubesphere/pkg/models/logging"
-	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
-	loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging"
-	util "kubesphere.io/kubesphere/pkg/utils/stringutils"
-	"strconv"
-	"strings"
-	"time"
-)
-
-const (
-	LevelCluster = iota
-	LevelContainer
-
-	// query type, default to `query`
-	TypeStat   = "statistics"
-	TypeHist   = "histogram"
-	TypeExport = "export"
-
-	Ascending  = "asc"
-	Descending = "desc"
-)
-
-type handler struct {
-	k  k8s.Client
-	lo logging.LoggingOperator
-}
-
-func newHandler(k k8s.Client, l loggingclient.Interface) *handler {
-	return &handler{k, logging.NewLoggingOperator(l)}
-}
-
-func (h handler) handleClusterQuery(req *restful.Request, resp *restful.Response) {
-	h.get(req, LevelCluster, resp)
-}
-
-func (h handler) handleContainerQuery(req *restful.Request, resp *restful.Response) {
-	h.get(req, LevelContainer, resp)
-}
-
-func (h handler) get(req *restful.Request, lvl int, resp *restful.Response) {
-	typ := req.QueryParameter("type")
-
-	noHit, sf, err := h.newSearchFilter(req, lvl)
-	if err != nil {
-		api.HandleBadRequest(resp, nil, err)
-	}
-	if noHit {
-		handleNoHit(typ, resp)
-		return
-	}
-
-	switch typ {
-	case TypeStat:
-		res, err := h.lo.GetCurrentStats(sf)
-		if err != nil {
-			api.HandleInternalError(resp, nil, err)
-		}
-		resp.WriteAsJson(res)
-	case TypeHist:
-		interval := req.QueryParameter("interval")
-		res, err := h.lo.CountLogsByInterval(sf, interval)
-		if err != nil {
-			api.HandleInternalError(resp, nil, err)
-		}
-		resp.WriteAsJson(res)
-	case TypeExport:
-		resp.Header().Set(restful.HEADER_ContentType, "text/plain")
-		resp.Header().Set("Content-Disposition", "attachment")
-		err := h.lo.ExportLogs(sf, resp.ResponseWriter)
-		if err != nil {
-			api.HandleInternalError(resp, nil, err)
-		}
-	default:
-		from, _ := strconv.ParseInt(req.QueryParameter("from"), 10, 64)
-		size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64)
-		if err != nil {
-			size = 10
-		}
-		order := req.QueryParameter("sort")
-		if order != Ascending {
-			order = Descending
-		}
-		res, err := h.lo.SearchLogs(sf, from, size, order)
-		if err != nil {
-			api.HandleInternalError(resp, nil, err)
-		}
-		resp.WriteAsJson(res)
-	}
-}
-
-func (h handler) newSearchFilter(req *restful.Request, level int) (bool, loggingclient.SearchFilter, error) {
-	var sf loggingclient.SearchFilter
-
-	switch level {
-	case LevelCluster:
-		sf.NamespaceFilter = h.intersect(
-			util.Split(req.QueryParameter("namespaces"), ","),
-			util.Split(strings.ToLower(req.QueryParameter("namespace_query")), ","),
-			util.Split(req.QueryParameter("workspaces"), ","),
-			util.Split(strings.ToLower(req.QueryParameter("workspace_query")), ","))
-		sf.WorkloadFilter = util.Split(req.QueryParameter("workloads"), ",")
-		sf.WorkloadSearch = util.Split(req.QueryParameter("workload_query"), ",")
-		sf.PodFilter = util.Split(req.QueryParameter("pods"), ",")
-		sf.PodSearch = util.Split(req.QueryParameter("pod_query"), ",")
-		sf.ContainerFilter = util.Split(req.QueryParameter("containers"), ",")
-		sf.ContainerSearch = util.Split(req.QueryParameter("container_query"), ",")
-	case LevelContainer:
-		sf.NamespaceFilter = h.withCreationTime(req.PathParameter("namespace"))
-		sf.PodFilter = []string{req.PathParameter("pod")}
-		sf.ContainerFilter = []string{req.PathParameter("container")}
-	}
-
-	sf.LogSearch = util.Split(req.QueryParameter("log_query"), ",")
-
-	var err error
-	now := time.Now()
-	// If time is not given, set it to now.
-	if req.QueryParameter("start_time") == "" {
-		sf.Starttime = now
-	} else {
-		sf.Starttime, err = time.Parse(time.RFC3339, req.QueryParameter("start_time"))
-		if err != nil {
-			return false, sf, err
-		}
-	}
-	if req.QueryParameter("end_time") == "" {
-		sf.Endtime = now
-	} else {
-		sf.Endtime, err = time.Parse(time.RFC3339, req.QueryParameter("end_time"))
-		if err != nil {
-			return false, sf, err
-		}
-	}
-
-	return len(sf.NamespaceFilter) == 0, sf, nil
-}
-
-func handleNoHit(typ string, resp *restful.Response) {
-	switch typ {
-	case TypeStat:
-		resp.WriteAsJson(new(loggingclient.Statistics))
-	case TypeHist:
-		resp.WriteAsJson(new(loggingclient.Histogram))
-	case TypeExport:
-		resp.Header().Set(restful.HEADER_ContentType, "text/plain")
-		resp.Header().Set("Content-Disposition", "attachment")
-		resp.Write(nil)
-	default:
-		resp.WriteAsJson(new(loggingclient.Logs))
-	}
-}

pkg/kapis/logging/v1alpha2/helper.go

@@ -1,56 +0,0 @@
-package v1alpha2
-
-import (
-	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/klog"
-	"kubesphere.io/kubesphere/pkg/constants"
-	"kubesphere.io/kubesphere/pkg/utils/stringutils"
-	"strings"
-	"time"
-)
-
-func (h handler) intersect(nsFilter []string, nsSearch []string, wsFilter []string, wsSearch []string) map[string]time.Time {
-	nsList, err := h.k.Kubernetes().CoreV1().Namespaces().List(v1.ListOptions{})
-	if err != nil {
-		klog.Errorf("failed to list namespace, error: %s", err)
-		return nil
-	}
-
-	inner := make(map[string]time.Time)
-
-	// if no search condition is set on both namespace and workspace,
-	// then return all namespaces
-	if nsSearch == nil && nsFilter == nil && wsSearch == nil && wsFilter == nil {
-		for _, ns := range nsList.Items {
-			inner[ns.Name] = ns.CreationTimestamp.Time
-		}
-	} else {
-		for _, ns := range nsList.Items {
-			if stringutils.StringIn(ns.Name, nsFilter) ||
-				stringutils.StringIn(ns.Annotations[constants.WorkspaceLabelKey], wsFilter) ||
-				containsIn(ns.Name, nsSearch) ||
-				containsIn(ns.Annotations[constants.WorkspaceLabelKey], wsSearch) {
-				inner[ns.Name] = ns.CreationTimestamp.Time
-			}
-		}
-	}
-
-	return inner
-}
-
-func containsIn(str string, subStrs []string) bool {
-	for _, sub := range subStrs {
-		if strings.Contains(str, sub) {
-			return true
-		}
-	}
-	return false
-}
-
-func (h handler) withCreationTime(name string) map[string]time.Time {
-	ns, err := h.k.Kubernetes().CoreV1().Namespaces().Get(name, v1.GetOptions{})
-	if err == nil {
-		return map[string]time.Time{name: ns.CreationTimestamp.Time}
-	}
-	return nil
-}

pkg/kapis/logging/v1alpha2/register.go

@@ -1,93 +0,0 @@
-/*
-
- Copyright 2019 The KubeSphere Authors.
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
-
-*/
-package v1alpha2
-
-import (
-	"github.com/emicklei/go-restful"
-	"github.com/emicklei/go-restful-openapi"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
-	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
-	"kubesphere.io/kubesphere/pkg/constants"
-	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
-	"kubesphere.io/kubesphere/pkg/simple/client/logging"
-	"net/http"
-)
-
-const (
-	GroupName = "logging.kubesphere.io"
-	RespOK    = "ok"
-)
-
-var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
-
-func AddToContainer(c *restful.Container, k8sClient k8s.Client, loggingClient logging.Interface) error {
-	ws := runtime.NewWebService(GroupVersion)
-
-	h := newHandler(k8sClient, loggingClient)
-
-	ws.Route(ws.GET("/cluster").
-		To(h.handleClusterQuery).
-		Doc("Query logs against the cluster.").
-		Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
-		Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)).
-		Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)).
-		Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)).
-		Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)).
-		Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)).
-		Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
-		Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
-		Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
-		Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
-		Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
-		Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
-		Writes(v1alpha2.APIResponse{}).
-		Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})).
-		Consumes(restful.MIME_JSON, restful.MIME_XML).
-		Produces(restful.MIME_JSON, "text/plain")
-
-	ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}").
-		To(h.handleContainerQuery).
-		Doc("Query logs against the specific container.").
-		Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
-		Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)).
-		Param(ws.PathParameter("container", "Container name.").DataType("string").Required(true)).
-		Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
-		Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
-		Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
-		Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
-		Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
-		Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
-		Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
-		Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
-		Writes(v1alpha2.APIResponse{}).
-		Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})).
-		Consumes(restful.MIME_JSON, restful.MIME_XML).
-		Produces(restful.MIME_JSON, restful.MIME_OCTET)
-
-	c.Add(ws)
-	return nil
-}

pkg/kapis/tenant/v1alpha2/handler.go

@@ -9,6 +9,7 @@ import (
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/api"
 	eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
+	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
 	tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
 	"kubesphere.io/kubesphere/pkg/apiserver/query"
 	"kubesphere.io/kubesphere/pkg/apiserver/request"
@@ -17,16 +18,17 @@ import (
 	"kubesphere.io/kubesphere/pkg/models/tenant"
 	servererr "kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging"
 )
 
 type tenantHandler struct {
 	tenant tenant.Interface
 }
 
-func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) *tenantHandler {
+func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) *tenantHandler {
 
 	return &tenantHandler{
-		tenant: tenant.New(factory, k8sclient, ksclient, evtsClient),
+		tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient),
 	}
 }
 
@@ -245,3 +247,38 @@ func (h *tenantHandler) Events(req *restful.Request, resp *restful.Response) {
 	resp.WriteEntity(result)
 
 }
+
+func (h *tenantHandler) QueryLogs(req *restful.Request, resp *restful.Response) {
+	user, ok := request.UserFrom(req.Request.Context())
+	if !ok {
+		err := fmt.Errorf("cannot obtain user info")
+		klog.Errorln(err)
+		api.HandleForbidden(resp, req, err)
+		return
+	}
+	queryParam, err := loggingv1alpha2.ParseQueryParameter(req)
+	if err != nil {
+		klog.Errorln(err)
+		api.HandleInternalError(resp, req, err)
+		return
+	}
+
+	if queryParam.Operation == loggingv1alpha2.OperationExport {
+		resp.Header().Set(restful.HEADER_ContentType, "text/plain")
+		resp.Header().Set("Content-Disposition", "attachment")
+		err := h.tenant.ExportLogs(user, queryParam, resp)
+		if err != nil {
+			klog.Errorln(err)
+			api.HandleInternalError(resp, req, err)
+			return
+		}
+	} else {
+		result, err := h.tenant.QueryLogs(user, queryParam)
+		if err != nil {
+			klog.Errorln(err)
+			api.HandleInternalError(resp, req, err)
+			return
+		}
+		resp.WriteAsJson(result)
+	}
+}

pkg/kapis/tenant/v1alpha2/register.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"kubesphere.io/kubesphere/pkg/api"
 	eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
+	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
 	tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
 	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
 	kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
@@ -33,6 +34,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/models"
 	"kubesphere.io/kubesphere/pkg/server/errors"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging"
 	"net/http"
 )
 
@@ -42,9 +44,9 @@ const (
 
 var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
 
-func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) error {
+func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) error {
 	ws := runtime.NewWebService(GroupVersion)
-	handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient)
+	handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient)
 
 	ws.Route(ws.POST("/workspaces").
 		To(handler.CreateWorkspace).
@@ -96,7 +98,7 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s
 	ws.Route(ws.GET("/events").
 		To(handler.Events).
 		Doc("Query events against the cluster").
-		Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")).
+		Param(ws.QueryParameter("operation", "Operation type. This can be one of three types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")).
 		Param(ws.QueryParameter("workspace_filter", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`.")).
 		Param(ws.QueryParameter("workspace_search", "A comma-separated list of keywords. Differing from **workspace_filter**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.")).
 		Param(ws.QueryParameter("involved_object_namespace_filter", "A comma-separated list of namespaces. This field restricts the query to specified `involvedObject.namespace`.")).
@@ -118,6 +120,33 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s
 		Writes(eventsv1alpha1.APIResponse{}).
 		Returns(http.StatusOK, api.StatusOK, eventsv1alpha1.APIResponse{}))
 
+	ws.Route(ws.GET("/logs").
+		To(handler.QueryLogs).
+		Doc("Query logs against the cluster.").
+		Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)).
+		Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)).
+		Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)).
+		Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)).
+		Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
+		Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
+		Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)).
+		Param(ws.QueryParameter("sort", "Sort order. One of asc, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
+		Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
+		Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
+		Writes(loggingv1alpha2.APIResponse{}).
+		Returns(http.StatusOK, api.StatusOK, loggingv1alpha2.APIResponse{})).
+		Consumes(restful.MIME_JSON, restful.MIME_XML).
+		Produces(restful.MIME_JSON, "text/plain")
+
 	c.Add(ws)
 	return nil
 }

pkg/models/tenant/tenant.go

@@ -19,6 +19,7 @@ package tenant
 
 import (
 	"fmt"
+	"io"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -28,9 +29,9 @@ import (
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/api"
 	eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
+	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
 	clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1"
 	tenantv1alpha1 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1"
-
 	tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
 	"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
@@ -39,9 +40,11 @@ import (
 	"kubesphere.io/kubesphere/pkg/informers"
 	"kubesphere.io/kubesphere/pkg/models/events"
 	"kubesphere.io/kubesphere/pkg/models/iam/am"
+	"kubesphere.io/kubesphere/pkg/models/logging"
 	resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
 	resourcesv1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
 	eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events"
+	loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging"
 	"kubesphere.io/kubesphere/pkg/utils/stringutils"
 	"strings"
 	"time"
@@ -58,6 +61,8 @@ type Interface interface {
 	ListWorkspaceClusters(workspace string) (*api.ListResult, error)
 
 	Events(user user.Info, queryParam *eventsv1alpha1.Query) (*eventsv1alpha1.APIResponse, error)
+	QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error)
+	ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error
 }
 
 type tenantOperator struct {
@@ -67,9 +72,10 @@ type tenantOperator struct {
 	ksclient       kubesphere.Interface
 	resourceGetter *resourcesv1alpha3.ResourceGetter
 	events         events.Interface
+	lo             logging.LoggingOperator
 }
 
-func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client) Interface {
+func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Interface) Interface {
 	amOperator := am.NewReadOnlyOperator(informers)
 	authorizer := authorizerfactory.NewRBACAuthorizer(amOperator)
 	return &tenantOperator{
@@ -79,6 +85,7 @@ func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ks
 		k8sclient:      k8sclient,
 		ksclient:       ksclient,
 		events:         events.NewEventsOperator(evtsClient),
+		lo:             logging.NewLoggingOperator(loggingClient),
 	}
 }
 
@@ -407,6 +414,129 @@ func (t *tenantOperator) Events(user user.Info, queryParam *eventsv1alpha1.Query
 	})
 }
 
+func (t *tenantOperator) QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error) {
+	iNamespaces, err := t.listIntersectedNamespaces(user,
+		stringutils.Split(query.WorkspaceFilter, ","),
+		stringutils.Split(query.WorkspaceSearch, ","),
+		stringutils.Split(query.NamespaceFilter, ","),
+		stringutils.Split(query.NamespaceSearch, ","))
+	if err != nil {
+		klog.Error(err)
+		return nil, err
+	}
+
+	namespaceCreateTimeMap := make(map[string]time.Time)
+	for _, ns := range iNamespaces {
+		podLogs := authorizer.AttributesRecord{
+			User:            user,
+			Verb:            "get",
+			APIGroup:        "",
+			APIVersion:      "v1",
+			Namespace:       ns.Name,
+			Resource:        "pods",
+			Subresource:     "log",
+			ResourceRequest: true,
+		}
+		decision, _, err := t.authorizer.Authorize(podLogs)
+		if err != nil {
+			klog.Error(err)
+			return nil, err
+		}
+		if decision == authorizer.DecisionAllow {
+			namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time
+		}
+	}
+
+	sf := loggingclient.SearchFilter{
+		NamespaceFilter: namespaceCreateTimeMap,
+		WorkloadSearch:  stringutils.Split(query.WorkloadSearch, ","),
+		WorkloadFilter:  stringutils.Split(query.WorkloadFilter, ","),
+		PodSearch:       stringutils.Split(query.PodSearch, ","),
+		PodFilter:       stringutils.Split(query.PodFilter, ","),
+		ContainerSearch: stringutils.Split(query.ContainerSearch, ","),
+		ContainerFilter: stringutils.Split(query.ContainerFilter, ","),
+		LogSearch:       stringutils.Split(query.LogSearch, ","),
+		Starttime:       query.StartTime,
+		Endtime:         query.EndTime,
+	}
+
+	var ar loggingv1alpha2.APIResponse
+	switch query.Operation {
+	case loggingv1alpha2.OperationStatistics:
+		if len(namespaceCreateTimeMap) == 0 {
+			ar.Statistics = &loggingclient.Statistics{}
+		} else {
+			ar, err = t.lo.GetCurrentStats(sf)
+		}
+	case loggingv1alpha2.OperationHistogram:
+		if len(namespaceCreateTimeMap) == 0 {
+			ar.Histogram = &loggingclient.Histogram{}
+		} else {
+			ar, err = t.lo.CountLogsByInterval(sf, query.Interval)
+		}
+	default:
+		if len(namespaceCreateTimeMap) == 0 {
+			ar.Logs = &loggingclient.Logs{}
+		} else {
+			ar, err = t.lo.SearchLogs(sf, query.From, query.Size, query.Sort)
+		}
+	}
+	return &ar, err
+}
+
+func (t *tenantOperator) ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error {
+	iNamespaces, err := t.listIntersectedNamespaces(user,
+		stringutils.Split(query.WorkspaceFilter, ","),
+		stringutils.Split(query.WorkspaceSearch, ","),
+		stringutils.Split(query.NamespaceFilter, ","),
+		stringutils.Split(query.NamespaceSearch, ","))
+	if err != nil {
+		klog.Error(err)
+		return err
+	}
+
+	namespaceCreateTimeMap := make(map[string]time.Time)
+	for _, ns := range iNamespaces {
+		podLogs := authorizer.AttributesRecord{
+			User:            user,
+			Verb:            "get",
+			APIGroup:        "",
+			APIVersion:      "v1",
+			Namespace:       ns.Name,
+			Resource:        "pods",
+			Subresource:     "log",
+			ResourceRequest: true,
+		}
+		decision, _, err := t.authorizer.Authorize(podLogs)
+		if err != nil {
+			klog.Error(err)
+			return err
+		}
+		if decision == authorizer.DecisionAllow {
+			namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time
+		}
+	}
+
+	sf := loggingclient.SearchFilter{
+		NamespaceFilter: namespaceCreateTimeMap,
+		WorkloadSearch:  stringutils.Split(query.WorkloadSearch, ","),
+		WorkloadFilter:  stringutils.Split(query.WorkloadFilter, ","),
+		PodSearch:       stringutils.Split(query.PodSearch, ","),
+		PodFilter:       stringutils.Split(query.PodFilter, ","),
+		ContainerSearch: stringutils.Split(query.ContainerSearch, ","),
+		ContainerFilter: stringutils.Split(query.ContainerFilter, ","),
+		LogSearch:       stringutils.Split(query.LogSearch, ","),
+		Starttime:       query.StartTime,
+		Endtime:         query.EndTime,
+	}
+
+	if len(namespaceCreateTimeMap) == 0 {
+		return nil
+	} else {
+		return t.lo.ExportLogs(sf, writer)
+	}
+}
+
 func contains(objects []runtime.Object, object runtime.Object) bool {
 	for _, item := range objects {
 		if item == object {

pkg/models/tenant/tenent_test.go

@@ -330,5 +330,5 @@ func prepare() Interface {
 			RoleBindings().Informer().GetIndexer().Add(roleBinding)
 	}
 
-	return New(fakeInformerFactory, nil, nil, nil)
+	return New(fakeInformerFactory, nil, nil, nil, nil)
 }

pkg/simple/client/logging/elasticsearch/api_body.go

@@ -2,6 +2,7 @@ package elasticsearch
 
 import (
 	"fmt"
+	"github.com/json-iterator/go"
 	"k8s.io/klog"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
 	"time"
@@ -13,6 +14,9 @@ const (
 	replicaSetSuffixMaxLength = 11 // max 10 characters + 1 hyphen
 )
 
+// TODO: elastic/go-elasticsearch is working on Query DSL support.
+//  See https://github.com/elastic/go-elasticsearch/issues/42.
+//  We need refactor our query body builder when that is ready.
 type bodyBuilder struct {
 	Body
 }
@@ -22,75 +26,9 @@ func newBodyBuilder() *bodyBuilder {
 }
 
 func (bb *bodyBuilder) bytes() ([]byte, error) {
-	return json.Marshal(bb.Body)
+	return jsoniter.Marshal(bb.Body)
 }
 
-// The mainBody func builds api body for query.
-// TODO: Should use an elegant pakcage for building query body, but `elastic/go-elasticsearch` doesn't provide it currently.
-//
-// Example:
-// GET kapis/logging.kubesphere.io/v1alpha2/cluster?start_time=0&end_time=156576063993&namespaces=kubesphere-system&pod_query=ks-apiserver
-// -----
-//{
-//    "from":0,
-//    "size":10,
-//    "sort":[
-//        {
-//            "time": "desc"
-//        }
-//    ],
-//    "query":{
-//        "bool":{
-//            "filter":[
-//                {
-//                    "bool":{
-//                        "should":[
-//                            {
-//                                "bool":{
-//                                    "filter":[
-//                                        {
-//                                            "match_phrase":{
-//                                                "kubernetes.namespace_name.keyword":"kubesphere-system"
-//                                            }
-//                                        },
-//                                        {
-//                                            "range":{
-//                                                "time":{
-//                                                    "gte":"1572315987000"
-//                                                }
-//                                            }
-//                                        }
-//                                    ]
-//                                }
-//                            }
-//                        ],
-//                        "minimum_should_match":1
-//                    }
-//                },
-//                {
-//                    "bool":{
-//                        "should":[
-//                            {
-//                                "match_phrase_prefix":{
-//                                    "kubernetes.pod_name":"ks-apiserver"
-//                                }
-//                            }
-//                        ],
-//                        "minimum_should_match":1
-//                    }
-//                },
-//                {
-//                    "range":{
-//                        "time":{
-//                            "gte":"0",
-//                            "lte":"156576063993"
-//                        }
-//                    }
-//                }
-//            ]
-//        }
-//    }
-//}
 func (bb *bodyBuilder) mainBool(sf logging.SearchFilter) *bodyBuilder {
 	var ms []Match
 
@@ -207,10 +145,6 @@ func (bb *bodyBuilder) cardinalityAggregation() *bodyBuilder {
 }
 
 func (bb *bodyBuilder) dateHistogramAggregation(interval string) *bodyBuilder {
-	if interval == "" {
-		interval = "15m"
-	}
-
 	bb.Body.Aggs = &Aggs{
 		DateHistogramAggregation: &DateHistogramAggregation{
 			&DateHistogram{
@@ -232,12 +166,8 @@ func (bb *bodyBuilder) size(n int64) *bodyBuilder {
 	return bb
 }
 
-func (bb *bodyBuilder) sort(o string) *bodyBuilder {
+func (bb *bodyBuilder) sort(order string) *bodyBuilder {
-	if o != "asc" {
+	bb.Sorts = []map[string]string{{"time": order}}
-		o = "desc"
-	}
-
-	bb.Sorts = []map[string]string{{"time": o}}
 	return bb
 }
 
@@ -268,7 +198,7 @@ func podNameRegexp(workloadName string) string {
 
 func parseResponse(body []byte) (Response, error) {
 	var res Response
-	err := json.Unmarshal(body, &res)
+	err := jsoniter.Unmarshal(body, &res)
 	if err != nil {
 		klog.Error(err)
 		return Response{}, err

pkg/simple/client/logging/elasticsearch/api_body_test.go

@@ -1,50 +1,117 @@
 package elasticsearch
 
 import (
+	"fmt"
 	"github.com/google/go-cmp/cmp"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
+	"reflect"
 	"testing"
+	"time"
 )
 
-func TestCardinalityAggregation(t *testing.T) {
+func TestMainBool(t *testing.T) {
-	var test = struct {
+	var tests = []struct {
-		description  string
+		filter   logging.SearchFilter
-		searchFilter logging.SearchFilter
+		expected string
-		expected     *bodyBuilder
 	}{
-		description: "add cardinality aggregation",
+		{
-		searchFilter: logging.SearchFilter{
+			filter: logging.SearchFilter{
-			LogSearch: []string{"info"},
+				NamespaceFilter: map[string]time.Time{
+					"default": time.Unix(1589981934, 0),
+				},
+			},
+			expected: "api_body_1.json",
 		},
-		expected: &bodyBuilder{Body{
+		{
-			Query: &Query{
+			filter: logging.SearchFilter{
-				Bool: Bool{
+				WorkloadFilter: []string{"mysql"},
-					Filter: []Match{
+				Starttime:      time.Unix(1589980934, 0),
-						{
+				Endtime:        time.Unix(1589981934, 0),
-							Bool: &Bool{
-								Should: []Match{
-									{
-										MatchPhrasePrefix: map[string]string{"log": "info"},
-									},
-								},
-								MinimumShouldMatch: 1,
-							},
-						},
-					},
-				},
 			},
-			Aggs: &Aggs{
+			expected: "api_body_2.json",
-				CardinalityAggregation: &CardinalityAggregation{
+		},
-					Cardinality: &Cardinality{Field: "kubernetes.docker_id.keyword"},
+		{
-				},
+			filter: logging.SearchFilter{
+				PodFilter: []string{"mysql"},
+				PodSearch: []string{"mysql-a8w3s-10945j"},
+				LogSearch: []string{"info"},
 			},
-		}},
+			expected: "api_body_3.json",
+		},
+		{
+			filter: logging.SearchFilter{
+				ContainerFilter: []string{"mysql-1"},
+				ContainerSearch: []string{"mysql-3"},
+			},
+			expected: "api_body_4.json",
+		},
 	}
 
-	t.Run(test.description, func(t *testing.T) {
+	for i, test := range tests {
-		body := newBodyBuilder().mainBool(test.searchFilter).cardinalityAggregation()
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-		if diff := cmp.Diff(body, test.expected); diff != "" {
+			var expected Body
-			t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
+			err := JsonFromFile(test.expected, &expected)
-		}
+			if err != nil {
-	})
+				t.Fatal(err)
+			}
+
+			result := newBodyBuilder().mainBool(test.filter).Body
+
+			if diff := cmp.Diff(result, expected); diff != "" {
+				fmt.Printf("%T differ (-got, +want): %s", expected, diff)
+			}
+		})
+	}
+}
+
+func TestCardinalityAggregation(t *testing.T) {
+	var tests = []struct {
+		expected string
+	}{
+		{
+			expected: "api_body_5.json",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var expected Body
+			err := JsonFromFile(test.expected, &expected)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			result := newBodyBuilder().cardinalityAggregation().Body
+
+			if !reflect.DeepEqual(result, expected) {
+				t.Fatalf("expected: %v, but got %v", expected, result)
+			}
+		})
+	}
+}
+
+func TestDateHistogramAggregation(t *testing.T) {
+	var tests = []struct {
+		expected string
+	}{
+		{
+			expected: "api_body_6.json",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var expected Body
+			err := JsonFromFile(test.expected, &expected)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			result := newBodyBuilder().dateHistogramAggregation("15m").Body
+
+			if !reflect.DeepEqual(result, expected) {
+				t.Fatalf("expected: %v, but got %v", expected, result)
+			}
+		})
+	}
 }

pkg/simple/client/logging/elasticsearch/elasticsearch.go

@@ -4,12 +4,12 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	jsoniter "github.com/json-iterator/go"
+	"github.com/json-iterator/go"
 	"io"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
-	v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
-	v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
-	v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
 	"kubesphere.io/kubesphere/pkg/utils/stringutils"
 	"strings"
 )
@@ -20,8 +20,6 @@ const (
 	ElasticV7 = "7"
 )
 
-var json = jsoniter.ConfigCompatibleWithStandardLibrary
-
 // Elasticsearch implement logging interface
 type Elasticsearch struct {
 	c client
@@ -29,8 +27,7 @@ type Elasticsearch struct {
 
 // versioned es client interface
 type client interface {
-	// Perform Search API
+	Search(body []byte, scroll bool) ([]byte, error)
-	Search(body []byte) ([]byte, error)
 	Scroll(id string) ([]byte, error)
 	ClearScroll(id string)
 	GetTotalHitCount(v interface{}) int64
@@ -83,11 +80,10 @@ func detectVersionMajor(host string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
 	defer res.Body.Close()
 
 	var b map[string]interface{}
-	if err = json.NewDecoder(res.Body).Decode(&b); err != nil {
+	if err = jsoniter.NewDecoder(res.Body).Decode(&b); err != nil {
 		return "", err
 	}
 	if res.IsError() {
@@ -116,7 +112,7 @@ func (es Elasticsearch) GetCurrentStats(sf logging.SearchFilter) (logging.Statis
 		return logging.Statistics{}, err
 	}
 
-	b, err := es.c.Search(body)
+	b, err := es.c.Search(body, true)
 	if err != nil {
 		return logging.Statistics{}, err
 	}
@@ -142,7 +138,7 @@ func (es Elasticsearch) CountLogsByInterval(sf logging.SearchFilter, interval st
 		return logging.Histogram{}, err
 	}
 
-	b, err := es.c.Search(body)
+	b, err := es.c.Search(body, false)
 	if err != nil {
 		return logging.Histogram{}, err
 	}
@@ -174,7 +170,7 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
 		return logging.Logs{}, err
 	}
 
-	b, err := es.c.Search(body)
+	b, err := es.c.Search(body, false)
 	if err != nil {
 		return logging.Logs{}, err
 	}
@@ -200,33 +196,50 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
 
 func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
 	var id string
-	var from int64 = 0
+	var data []string
-	var size int64 = 1000
 
-	res, err := es.SearchLogs(sf, from, size, "desc")
+	// Initial Search
-	defer es.ClearScroll(id)
+	body, err := newBodyBuilder().
+		mainBool(sf).
+		from(0).
+		size(1000).
+		sort("desc").
+		bytes()
 	if err != nil {
 		return err
 	}
 
-	if res.Records == nil || len(res.Records) == 0 {
+	b, err := es.c.Search(body, true)
+	defer es.ClearScroll(id)
+	if err != nil {
+		return err
+	}
+	res, err := parseResponse(b)
+	if err != nil {
+		return err
+	}
+
+	id = res.ScrollId
+	for _, hit := range res.AllHits {
+		data = append(data, hit.Log)
+	}
+	if len(data) == 0 {
 		return nil
 	}
 
 	// limit to retrieve max 100k records
 	for i := 0; i < 100; i++ {
-		res, id, err = es.scroll(id)
+		data, id, err = es.scroll(id)
 		if err != nil {
 			return err
 		}
-
+		if len(data) == 0 {
-		if res.Records == nil || len(res.Records) == 0 {
 			return nil
 		}
 
 		output := new(bytes.Buffer)
-		for _, r := range res.Records {
+		for _, l := range data {
-			output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(r.Log)))
+			output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(l)))
 		}
 		_, err = io.Copy(w, output)
 		if err != nil {
@@ -236,24 +249,22 @@ func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
 	return nil
 }
 
-func (es *Elasticsearch) scroll(id string) (logging.Logs, string, error) {
+func (es *Elasticsearch) scroll(id string) ([]string, string, error) {
 	b, err := es.c.Scroll(id)
 	if err != nil {
-		return logging.Logs{}, id, err
+		return nil, id, err
 	}
 
 	res, err := parseResponse(b)
 	if err != nil {
-		return logging.Logs{}, id, err
+		return nil, id, err
 	}
 
-	var l logging.Logs
+	var data []string
 	for _, hit := range res.AllHits {
-		l.Records = append(l.Records, logging.Record{
+		data = append(data, hit.Log)
-			Log: hit.Log,
-		})
 	}
-	return l, res.ScrollId, nil
+	return data, res.ScrollId, nil
 }
 
 func (es *Elasticsearch) ClearScroll(id string) {

pkg/simple/client/logging/elasticsearch/elasticsearch_test.go

@@ -1,70 +1,46 @@
 package elasticsearch
 
 import (
+	"fmt"
 	"github.com/google/go-cmp/cmp"
+	"github.com/json-iterator/go"
+	"io/ioutil"
 	"kubesphere.io/kubesphere/pkg/simple/client/logging"
-	v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
-	v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
-	v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
+	"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
 	"net/http"
 	"net/http/httptest"
 	"testing"
-	"time"
 )
 
-func MockElasticsearchService(pattern string, fakeResp string) *httptest.Server {
-	mux := http.NewServeMux()
-	mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
-		res.Write([]byte(fakeResp))
-	})
-	return httptest.NewServer(mux)
-}
-
 func TestDetectVersionMajor(t *testing.T) {
 	var tests = []struct {
-		description   string
+		fakeResp string
-		fakeResp      string
+		expected string
-		expected      string
-		expectedError bool
 	}{
 		{
-			description: "detect es 6.x version number",
+			fakeResp: "es6_detect_version_major_200.json",
-			fakeResp: `{
+			expected: ElasticV6,
-  "name" : "elasticsearch-logging-data-0",
+		},
-  "cluster_name" : "elasticsearch",
+		{
-  "cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
+			fakeResp: "es7_detect_version_major_200.json",
-  "version" : {
+			expected: ElasticV7,
-    "number" : "6.7.0",
-    "build_flavor" : "oss",
-    "build_type" : "docker",
-    "build_hash" : "8453f77",
-    "build_date" : "2019-03-21T15:32:29.844721Z",
-    "build_snapshot" : false,
-    "lucene_version" : "7.7.0",
-    "minimum_wire_compatibility_version" : "5.6.0",
-    "minimum_index_compatibility_version" : "5.0.0"
-  },
-  "tagline" : "You Know, for Search"
-}`,
-			expected:      ElasticV6,
-			expectedError: false,
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
-		t.Run(test.description, func(t *testing.T) {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			es := MockElasticsearchService("/", test.fakeResp)
+			es := mockElasticsearchService("/", test.fakeResp, http.StatusOK)
 			defer es.Close()
 
-			v, err := detectVersionMajor(es.URL)
+			result, err := detectVersionMajor(es.URL)
-			if err == nil && test.expectedError {
+			if err != nil {
-				t.Fatalf("expected error while got nothing")
-			} else if err != nil && !test.expectedError {
 				t.Fatal(err)
 			}
 
-			if v != test.expected {
+			if diff := cmp.Diff(result, test.expected); diff != "" {
-				t.Fatalf("expected get version %s, but got %s", test.expected, v)
+				t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
 			}
 		})
 	}
@@ -72,297 +48,202 @@ func TestDetectVersionMajor(t *testing.T) {
 
 func TestGetCurrentStats(t *testing.T) {
 	var tests = []struct {
-		description   string
+		fakeVersion string
-		searchFilter  logging.SearchFilter
+		fakeResp    string
-		fakeVersion   string
+		fakeCode    int
-		fakeResp      string
+		expected    logging.Statistics
-		expected      logging.Statistics
+		expectedErr string
-		expectedError bool
 	}{
 		{
-			description:  "[es 6.x] run as admin",
+			fakeVersion: ElasticV6,
-			searchFilter: logging.SearchFilter{},
+			fakeResp:    "es6_get_current_stats_200.json",
-			fakeVersion:  ElasticV6,
+			fakeCode:    http.StatusOK,
-			fakeResp: `{
-    "took": 171,
-    "timed_out": false,
-    "_shards": {
-        "total": 10,
-        "successful": 10,
-        "skipped": 0,
-        "failed": 0
-    },
-    "hits": {
-        "total": 241222,
-        "max_score": 1.0,
-        "hits": [
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "Hn1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:29.015Z",
-                    "log": "  value: \"hostpath\"\n",
-                    "time": "2020-02-28T19:25:29.015492329Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "I31GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:33.103Z",
-                    "log": "I0228 19:25:33.102631       1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
-                    "time": "2020-02-28T19:25:33.103075891Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "JX1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:33.113Z",
-                    "log": "I0228 19:25:33.112200       1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
-                    "time": "2020-02-28T19:25:33.113110332Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "Kn1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:34.168Z",
-                    "log": "  value: \"hostpath\"\n",
-                    "time": "2020-02-28T19:25:34.168983384Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "LH1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:34.168Z",
-                    "log": "  value: \"/var/openebs/local/\"\n",
-                    "time": "2020-02-28T19:25:34.168997393Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "NX1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:25:42.868Z",
-                    "log": "I0228 19:25:42.868413       1 config.go:83] SC local has config:- name: StorageType\n",
-                    "time": "2020-02-28T19:25:42.868578188Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "Q31GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:26:13.881Z",
-                    "log": "- name: BasePath\n",
-                    "time": "2020-02-28T19:26:13.881180681Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "S31GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:26:14.597Z",
-                    "log": "  value: \"/var/openebs/local/\"\n",
-                    "time": "2020-02-28T19:26:14.597702238Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "TH1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:26:14.597Z",
-                    "log": "I0228 19:26:14.597007       1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
-                    "time": "2020-02-28T19:26:14.597708432Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            },
-            {
-                "_index": "ks-logstash-log-2020.02.28",
-                "_type": "flb_type",
-                "_id": "UX1GjXABMO5aQxyNsyxy",
-                "_score": 1.0,
-                "_source": {
-                    "@timestamp": "2020-02-28T19:26:15.920Z",
-                    "log": "I0228 19:26:15.915071       1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
-                    "time": "2020-02-28T19:26:15.920650572Z",
-                    "kubernetes": {
-                        "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
-                        "namespace_name": "kube-system",
-                        "host": "ks-allinone",
-                        "container_name": "openebs-localpv-provisioner",
-                        "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
-                        "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
-                    }
-                }
-            }
-        ]
-    },
-    "aggregations": {
-        "container_count": {
-            "value": 93
-        }
-    }
-}`,
 			expected: logging.Statistics{
 				Containers: 93,
 				Logs:       241222,
 			},
-			expectedError: false,
 		},
 		{
-			description: "[es 6.x] index not found",
-			searchFilter: logging.SearchFilter{
-				NamespaceFilter: map[string]time.Time{
-					"workspace-1-project-a": time.Unix(1582000000, 0),
-					"workspace-1-project-b": time.Unix(1582333333, 0),
-				},
-			},
 			fakeVersion: ElasticV6,
-			fakeResp: `{
+			fakeResp:    "es6_get_current_stats_404.json",
-   "error": {
+			fakeCode:    http.StatusNotFound,
-       "root_cause": [
+			expectedErr: "type: index_not_found_exception, reason: no such index",
-           {
+		},
-               "type": "index_not_found_exception",
+		{
-               "reason": "no such index",
+			fakeVersion: ElasticV7,
-               "resource.type": "index_or_alias",
+			fakeResp:    "es7_get_current_stats_200.json",
-               "resource.id": "ks-lsdfsdfsdfs",
+			fakeCode:    http.StatusOK,
-               "index_uuid": "_na_",
-               "index": "ks-lsdfsdfsdfs"
-           }
-       ],
-       "type": "index_not_found_exception",
-       "reason": "no such index",
-       "resource.type": "index_or_alias",
-       "resource.id": "ks-lsdfsdfsdfs",
-       "index_uuid": "_na_",
-       "index": "ks-lsdfsdfsdfs"
-   },
-   "status": 404
-}`,
 			expected: logging.Statistics{
-				Containers: 0,
+				Containers: 48,
-				Logs:       0,
+				Logs:       9726,
 			},
-			expectedError: true,
+		},
+		{
+			fakeVersion: ElasticV7,
+			fakeResp:    "es7_get_current_stats_404.json",
+			fakeCode:    http.StatusNotFound,
+			expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-2020.05.2]",
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
-		t.Run(test.description, func(t *testing.T) {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			es := MockElasticsearchService("/", test.fakeResp)
+			srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
-			defer es.Close()
+			defer srv.Close()
 
-			clientv5 := Elasticsearch{c: v5.New(es.URL, "ks-logstash-log")}
+			es := newElasticsearchClient(srv, test.fakeVersion)
-			clientv6 := Elasticsearch{c: v6.New(es.URL, "ks-logstash-log")}
-			clientv7 := Elasticsearch{c: v7.New(es.URL, "ks-logstash-log")}
 
-			var stats logging.Statistics
+			result, err := es.GetCurrentStats(logging.SearchFilter{})
-			var err error
+			if test.expectedErr != "" {
-			switch test.fakeVersion {
+				if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
-			case ElasticV5:
+					t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
-				stats, err = clientv5.GetCurrentStats(test.searchFilter)
+				}
-			case ElasticV6:
-				stats, err = clientv6.GetCurrentStats(test.searchFilter)
-			case ElasticV7:
-				stats, err = clientv7.GetCurrentStats(test.searchFilter)
 			}
-
+			if diff := cmp.Diff(result, test.expected); diff != "" {
-			if err != nil && !test.expectedError {
-				t.Fatal(err)
-			} else if diff := cmp.Diff(stats, test.expected); diff != "" {
 				t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
 			}
 		})
 	}
 }
+
+func TestCountLogsByInterval(t *testing.T) {
+	var tests = []struct {
+		fakeVersion string
+		fakeResp    string
+		fakeCode    int
+		expected    logging.Histogram
+		expectedErr string
+	}{
+		{
+			fakeVersion: ElasticV7,
+			fakeResp:    "es7_count_logs_by_interval_200.json",
+			fakeCode:    http.StatusOK,
+			expected: logging.Histogram{
+				Total: 10000,
+				Buckets: []logging.Bucket{
+					{
+						Time:  1589644800000,
+						Count: 410,
+					},
+					{
+						Time:  1589646600000,
+						Count: 7465,
+					},
+					{
+						Time:  1589648400000,
+						Count: 12790,
+					},
+				},
+			},
+		},
+		{
+			fakeVersion: ElasticV7,
+			fakeResp:    "es7_count_logs_by_interval_400.json",
+			fakeCode:    http.StatusBadRequest,
+			expectedErr: "type: search_phase_execution_exception, reason: all shards failed",
+		},
+		{
+			fakeVersion: ElasticV7,
+			fakeResp:    "es7_count_logs_by_interval_404.json",
+			fakeCode:    http.StatusNotFound,
+			expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-20]",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
+			defer srv.Close()
+
+			es := newElasticsearchClient(srv, test.fakeVersion)
+
+			result, err := es.CountLogsByInterval(logging.SearchFilter{}, "15m")
+			if test.expectedErr != "" {
+				if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
+					t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
+				}
+			}
+			if diff := cmp.Diff(result, test.expected); diff != "" {
+				t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
+			}
+		})
+	}
+}
+
+func TestSearchLogs(t *testing.T) {
+	var tests = []struct {
+		fakeVersion string
+		fakeResp    string
+		fakeCode    int
+		expected    string
+		expectedErr string
+	}{
+		{
+			fakeVersion: ElasticV7,
+			fakeResp:    "es7_search_logs_200.json",
+			fakeCode:    http.StatusOK,
+			expected:    "es7_search_logs_200_result.json",
+		},
+	}
+
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			var expected logging.Logs
+			err := JsonFromFile(test.expected, &expected)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
+			defer srv.Close()
+
+			es := newElasticsearchClient(srv, test.fakeVersion)
+
+			result, err := es.SearchLogs(logging.SearchFilter{}, 0, 10, "asc")
+			if test.expectedErr != "" {
+				if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
+					t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
+				}
+			}
+			if diff := cmp.Diff(result, expected); diff != "" {
+				t.Fatalf("%T differ (-got, +want): %s", expected, diff)
+			}
+		})
+	}
+}
+
+func mockElasticsearchService(pattern, fakeResp string, fakeCode int) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
+		b, _ := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", fakeResp))
+		res.WriteHeader(fakeCode)
+		res.Write(b)
+	})
+	return httptest.NewServer(mux)
+}
+
+func newElasticsearchClient(srv *httptest.Server, version string) Elasticsearch {
+	var es Elasticsearch
+	switch version {
+	case ElasticV5:
+		es = Elasticsearch{c: v5.New(srv.URL, "ks-logstash-log")}
+	case ElasticV6:
+		es = Elasticsearch{c: v6.New(srv.URL, "ks-logstash-log")}
+	case ElasticV7:
+		es = Elasticsearch{c: v7.New(srv.URL, "ks-logstash-log")}
+	}
+	return es
+}
+
+func JsonFromFile(expectedFile string, expectedJsonPtr interface{}) error {
+	json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile))
+	if err != nil {
+		return err
+	}
+	err = jsoniter.Unmarshal(json, expectedJsonPtr)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json

@@ -0,0 +1,33 @@
+{
+  "query":{
+    "bool":{
+      "filter":[
+        {
+          "bool":{
+            "should":[
+              {
+                "bool":{
+                  "filter":[
+                    {
+                      "match_phrase":{
+                        "kubernetes.namespace_name.keyword":"default"
+                      }
+                    },
+                    {
+                      "range":{
+                        "time":{
+                          "gte":"2020-05-20T21:38:54+08:00"
+                        }
+                      }
+                    }
+                  ]
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        }
+      ]
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json

@@ -0,0 +1,28 @@
+{
+  "query":{
+    "bool":{
+      "filter":[
+        {
+          "bool":{
+            "should":[
+              {
+                "regexp":{
+                  "kubernetes.pod_name.keyword":"mysql-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|mysql-[0-9]+|mysql-[a-z0-9]{5}"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        },
+        {
+          "range":{
+            "time":{
+              "gte":"2020-05-20T21:22:14+08:00",
+              "lte":"2020-05-20T21:38:54+08:00"
+            }
+          }
+        }
+      ]
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json

@@ -0,0 +1,44 @@
+{
+  "query":{
+    "bool":{
+      "filter":[
+        {
+          "bool":{
+            "should":[
+              {
+                "match_phrase":{
+                  "kubernetes.pod_name.keyword":"mysql"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        },
+        {
+          "bool":{
+            "should":[
+              {
+                "match_phrase_prefix":{
+                  "kubernetes.pod_name":"mysql-a8w3s-10945j"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        },
+        {
+          "bool":{
+            "should":[
+              {
+                "match_phrase_prefix":{
+                  "log":"info"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        }
+      ]
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json

@@ -0,0 +1,32 @@
+{
+  "query":{
+    "bool":{
+      "filter":[
+        {
+          "bool":{
+            "should":[
+              {
+                "match_phrase":{
+                  "kubernetes.container_name.keyword":"mysql-1"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        },
+        {
+          "bool":{
+            "should":[
+              {
+                "match_phrase_prefix":{
+                  "kubernetes.container_name":"mysql-3"
+                }
+              }
+            ],
+            "minimum_should_match":1
+          }
+        }
+      ]
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json

@@ -0,0 +1,9 @@
+{
+  "aggs":{
+    "container_count":{
+      "cardinality":{
+        "field":"kubernetes.docker_id.keyword"
+      }
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json

@@ -0,0 +1,10 @@
+{
+  "aggs":{
+    "log_count_over_time":{
+      "date_histogram":{
+        "field":"time",
+        "interval":"15m"
+      }
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json

@@ -0,0 +1,17 @@
+{
+  "name" : "elasticsearch-logging-data-0",
+  "cluster_name" : "elasticsearch",
+  "cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
+  "version" : {
+    "number" : "6.7.0",
+    "build_flavor" : "oss",
+    "build_type" : "docker",
+    "build_hash" : "8453f77",
+    "build_date" : "2019-03-21T15:32:29.844721Z",
+    "build_snapshot" : false,
+    "lucene_version" : "7.7.0",
+    "minimum_wire_compatibility_version" : "5.6.0",
+    "minimum_index_compatibility_version" : "5.0.0"
+  },
+  "tagline" : "You Know, for Search"
+}

pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json

@@ -0,0 +1,211 @@
+{
+  "took": 171,
+  "timed_out": false,
+  "_shards": {
+    "total": 10,
+    "successful": 10,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": 241222,
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "Hn1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:29.015Z",
+          "log": "  value: \"hostpath\"\n",
+          "time": "2020-02-28T19:25:29.015492329Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "I31GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:33.103Z",
+          "log": "I0228 19:25:33.102631       1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
+          "time": "2020-02-28T19:25:33.103075891Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "JX1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:33.113Z",
+          "log": "I0228 19:25:33.112200       1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
+          "time": "2020-02-28T19:25:33.113110332Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "Kn1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:34.168Z",
+          "log": "  value: \"hostpath\"\n",
+          "time": "2020-02-28T19:25:34.168983384Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "LH1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:34.168Z",
+          "log": "  value: \"/var/openebs/local/\"\n",
+          "time": "2020-02-28T19:25:34.168997393Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "NX1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:25:42.868Z",
+          "log": "I0228 19:25:42.868413       1 config.go:83] SC local has config:- name: StorageType\n",
+          "time": "2020-02-28T19:25:42.868578188Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "Q31GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:26:13.881Z",
+          "log": "- name: BasePath\n",
+          "time": "2020-02-28T19:26:13.881180681Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "S31GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:26:14.597Z",
+          "log": "  value: \"/var/openebs/local/\"\n",
+          "time": "2020-02-28T19:26:14.597702238Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "TH1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:26:14.597Z",
+          "log": "I0228 19:26:14.597007       1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
+          "time": "2020-02-28T19:26:14.597708432Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.02.28",
+        "_type": "flb_type",
+        "_id": "UX1GjXABMO5aQxyNsyxy",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-02-28T19:26:15.920Z",
+          "log": "I0228 19:26:15.915071       1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
+          "time": "2020-02-28T19:26:15.920650572Z",
+          "kubernetes": {
+            "pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
+            "namespace_name": "kube-system",
+            "host": "ks-allinone",
+            "container_name": "openebs-localpv-provisioner",
+            "docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
+            "container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
+          }
+        }
+      }
+    ]
+  },
+  "aggregations": {
+    "container_count": {
+      "value": 93
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json

@@ -0,0 +1,21 @@
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "index_not_found_exception",
+        "reason": "no such index",
+        "resource.type": "index_or_alias",
+        "resource.id": "ks-lsdfsdfsdfs",
+        "index_uuid": "_na_",
+        "index": "ks-lsdfsdfsdfs"
+      }
+    ],
+    "type": "index_not_found_exception",
+    "reason": "no such index",
+    "resource.type": "index_or_alias",
+    "resource.id": "ks-lsdfsdfsdfs",
+    "index_uuid": "_na_",
+    "index": "ks-lsdfsdfsdfs"
+  },
+  "status": 404
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json

@@ -0,0 +1,230 @@
+{
+  "took": 23,
+  "timed_out": false,
+  "_shards": {
+    "total": 2,
+    "successful": 2,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 10000,
+      "relation": "gte"
+    },
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tRt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.608Z",
+          "log": "10.233.30.76    redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.608962452Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tht2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.670Z",
+          "log": "10.233.30.204   redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.670430525Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "txt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.731Z",
+          "log": "10.233.56.100   redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.731865428Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "jxt2MXIBlcWZ594bpIVN",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:07.648Z",
+          "log": "ls: cannot access '/calico-secrets': No such file or directory\n",
+          "time": "2020-05-16T16:00:07.64848716Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kBt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.164Z",
+          "log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
+          "time": "2020-05-16T16:00:08.164003996Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kRt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "CNI plugin version: v3.7.3\n",
+          "time": "2020-05-16T16:00:08.350585959Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kht2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "/host/secondary-bin-dir is non-writeable, skipping\n",
+          "time": "2020-05-16T16:00:08.350625112Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kxt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
+          "time": "2020-05-16T16:00:08.350692011Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "lBt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.454Z",
+          "log": "CNI config: {\n",
+          "time": "2020-05-16T16:00:08.454417144Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "lRt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.454Z",
+          "log": "  \"name\": \"cni0\",\n",
+          "time": "2020-05-16T16:00:08.454452649Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      }
+    ]
+  },
+  "aggregations": {
+    "log_count_over_time": {
+      "buckets": [
+        {
+          "key_as_string": "2020-05-16T16:00:00.000Z",
+          "key": 1589644800000,
+          "doc_count": 410
+        },
+        {
+          "key_as_string": "2020-05-16T16:30:00.000Z",
+          "key": 1589646600000,
+          "doc_count": 7465
+        },
+        {
+          "key_as_string": "2020-05-16T17:00:00.000Z",
+          "key": 1589648400000,
+          "doc_count": 12790
+        }
+      ]
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json

@@ -0,0 +1,47 @@
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "illegal_argument_exception",
+        "reason": "Unable to parse interval [30m0s]"
+      },
+      {
+        "type": "illegal_argument_exception",
+        "reason": "Unable to parse interval [30m0s]"
+      }
+    ],
+    "type": "search_phase_execution_exception",
+    "reason": "all shards failed",
+    "phase": "query",
+    "grouped": true,
+    "failed_shards": [
+      {
+        "shard": 0,
+        "index": "ks-logstash-log-2020.05.16",
+        "node": "Zr2OFlfeSJmK_W3Re4UBlg",
+        "reason": {
+          "type": "illegal_argument_exception",
+          "reason": "Unable to parse interval [30m0s]"
+        }
+      },
+      {
+        "shard": 0,
+        "index": "ks-logstash-log-2020.05.20",
+        "node": "pbGNYbV3QUuV5yJAgxbp3g",
+        "reason": {
+          "type": "illegal_argument_exception",
+          "reason": "Unable to parse interval [30m0s]"
+        }
+      }
+    ],
+    "caused_by": {
+      "type": "illegal_argument_exception",
+      "reason": "Unable to parse interval [30m0s]",
+      "caused_by": {
+        "type": "illegal_argument_exception",
+        "reason": "Unable to parse interval [30m0s]"
+      }
+    }
+  },
+  "status": 400
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json

@@ -0,0 +1,21 @@
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "index_not_found_exception",
+        "reason": "no such index [ks-logstash-log-20]",
+        "resource.type": "index_or_alias",
+        "resource.id": "ks-logstash-log-20",
+        "index_uuid": "_na_",
+        "index": "ks-logstash-log-20"
+      }
+    ],
+    "type": "index_not_found_exception",
+    "reason": "no such index [ks-logstash-log-20]",
+    "resource.type": "index_or_alias",
+    "resource.id": "ks-logstash-log-20",
+    "index_uuid": "_na_",
+    "index": "ks-logstash-log-20"
+  },
+  "status": 404
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json

@@ -0,0 +1,17 @@
+{
+  "name" : "elasticsearch-master-2",
+  "cluster_name" : "elasticsearch",
+  "cluster_uuid" : "_A-3shR0R0i-2M9CzOWP8g",
+  "version" : {
+    "number" : "7.7.0",
+    "build_flavor" : "default",
+    "build_type" : "docker",
+    "build_hash" : "81a1e9eda8e6183f5237786246f6dced26a10eaf",
+    "build_date" : "2020-05-12T02:01:37.602180Z",
+    "build_snapshot" : false,
+    "lucene_version" : "8.5.1",
+    "minimum_wire_compatibility_version" : "6.8.0",
+    "minimum_index_compatibility_version" : "6.0.0-beta1"
+  },
+  "tagline" : "You Know, for Search"
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json

@@ -0,0 +1,214 @@
+{
+  "took": 1455,
+  "timed_out": false,
+  "_shards": {
+    "total": 2,
+    "successful": 2,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 9726,
+      "relation": "eq"
+    },
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tRt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.608Z",
+          "log": "10.233.30.76    redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.608962452Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tht2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.670Z",
+          "log": "10.233.30.204   redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.670430525Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "txt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.731Z",
+          "log": "10.233.56.100   redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.731865428Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "jxt2MXIBlcWZ594bpIVN",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:07.648Z",
+          "log": "ls: cannot access '/calico-secrets': No such file or directory\n",
+          "time": "2020-05-16T16:00:07.64848716Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kBt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.164Z",
+          "log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
+          "time": "2020-05-16T16:00:08.164003996Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kRt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "CNI plugin version: v3.7.3\n",
+          "time": "2020-05-16T16:00:08.350585959Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kht2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "/host/secondary-bin-dir is non-writeable, skipping\n",
+          "time": "2020-05-16T16:00:08.350625112Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "kxt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.350Z",
+          "log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
+          "time": "2020-05-16T16:00:08.350692011Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "lBt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.454Z",
+          "log": "CNI config: {\n",
+          "time": "2020-05-16T16:00:08.454417144Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "lRt2MXIBlcWZ594bpIV0",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:08.454Z",
+          "log": "  \"name\": \"cni0\",\n",
+          "time": "2020-05-16T16:00:08.454452649Z",
+          "kubernetes": {
+            "pod_name": "calico-node-gc7pp",
+            "namespace_name": "kube-system",
+            "host": "master0",
+            "container_name": "install-cni",
+            "docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
+            "container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
+          }
+        }
+      }
+    ]
+  },
+  "aggregations": {
+    "container_count": {
+      "value": 48
+    }
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json

@@ -0,0 +1,21 @@
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "index_not_found_exception",
+        "reason": "no such index [ks-logstash-log-2020.05.2]",
+        "resource.type": "index_or_alias",
+        "resource.id": "ks-logstash-log-2020.05.2",
+        "index_uuid": "_na_",
+        "index": "ks-logstash-log-2020.05.2"
+      }
+    ],
+    "type": "index_not_found_exception",
+    "reason": "no such index [ks-logstash-log-2020.05.2]",
+    "resource.type": "index_or_alias",
+    "resource.id": "ks-logstash-log-2020.05.2",
+    "index_uuid": "_na_",
+    "index": "ks-logstash-log-2020.05.2"
+  },
+  "status": 404
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json

@@ -0,0 +1,76 @@
+{
+  "took": 772,
+  "timed_out": false,
+  "_shards": {
+    "total": 2,
+    "successful": 2,
+    "skipped": 0,
+    "failed": 0
+  },
+  "hits": {
+    "total": {
+      "value": 10000,
+      "relation": "gte"
+    },
+    "max_score": 1.0,
+    "hits": [
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tRt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.608Z",
+          "log": "10.233.30.76    redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.608962452Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "tht2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.670Z",
+          "log": "10.233.30.204   redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
+          "time": "2020-05-16T16:00:42.670430525Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "kubesphere-system",
+            "host": "master0",
+            "container_name": "config-init",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      },
+      {
+        "_index": "ks-logstash-log-2020.05.16",
+        "_type": "flb_type",
+        "_id": "txt2MXIBlcWZ594bqIUO",
+        "_score": 1.0,
+        "_source": {
+          "@timestamp": "2020-05-16T16:00:42.731Z",
+          "log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
+          "time": "2020-05-16T16:00:42.731865428Z",
+          "kubernetes": {
+            "pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
+            "namespace_name": "istio-system",
+            "host": "node0",
+            "container_name": "mixer",
+            "docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
+            "container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
+          }
+        }
+      }
+    ]
+  }
+}

pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json

@@ -0,0 +1,29 @@
+{
+  "total": 10000,
+  "records": [
+    {
+      "time": "2020-05-16T16:00:42.608962452Z",
+      "log": "10.233.30.76    redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
+      "namespace": "kubesphere-system",
+      "pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
+      "container": "config-init",
+      "host": "master0"
+    },
+    {
+      "time": "2020-05-16T16:00:42.670430525Z",
+      "log": "10.233.30.204   redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
+      "namespace": "kubesphere-system",
+      "pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
+      "container": "config-init",
+      "host": "master0"
+    },
+    {
+      "time": "2020-05-16T16:00:42.731865428Z",
+      "log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
+      "namespace": "istio-system",
+      "pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
+      "container": "mixer",
+      "host": "node0"
+    }
+  ]
+}

pkg/simple/client/logging/elasticsearch/versions/v5/v5.go

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
 	return &Elastic{client: client, index: index}
 }
 
-func (e *Elastic) Search(body []byte) ([]byte, error) {
+func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
-	response, err := e.client.Search(
+	opts := []func(*esapi.SearchRequest){
 		e.client.Search.WithContext(context.Background()),
 		e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
 		e.client.Search.WithBody(bytes.NewBuffer(body)),
-		e.client.Search.WithScroll(time.Minute))
+	}
+	if scroll {
+		opts = append(opts, e.client.Search.WithScroll(time.Minute))
+	}
+
+	response, err := e.client.Search(opts...)
 	if err != nil {
 		return nil, err
 	}

pkg/simple/client/logging/elasticsearch/versions/v6/v6.go

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
 	return &Elastic{Client: client, index: index}
 }
 
-func (e *Elastic) Search(body []byte) ([]byte, error) {
+func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
-	response, err := e.Client.Search(
+	opts := []func(*esapi.SearchRequest){
 		e.Client.Search.WithContext(context.Background()),
 		e.Client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
 		e.Client.Search.WithBody(bytes.NewBuffer(body)),
-		e.Client.Search.WithScroll(time.Minute))
+	}
+	if scroll {
+		opts = append(opts, e.Client.Search.WithScroll(time.Minute))
+	}
+
+	response, err := e.Client.Search(opts...)
 	if err != nil {
 		return nil, err
 	}

pkg/simple/client/logging/elasticsearch/versions/v7/v7.go

@@ -29,13 +29,17 @@ func New(address string, index string) *Elastic {
 	return &Elastic{client: client, index: index}
 }
 
-func (e *Elastic) Search(body []byte) ([]byte, error) {
+func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
-	response, err := e.client.Search(
+	opts := []func(*esapi.SearchRequest){
 		e.client.Search.WithContext(context.Background()),
 		e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
-		e.client.Search.WithTrackTotalHits(true),
 		e.client.Search.WithBody(bytes.NewBuffer(body)),
-		e.client.Search.WithScroll(time.Minute))
+	}
+	if scroll {
+		opts = append(opts, e.client.Search.WithScroll(time.Minute))
+	}
+
+	response, err := e.client.Search(opts...)
 	if err != nil {
 		return nil, err
 	}

pkg/simple/client/logging/interface.go

@@ -6,13 +6,9 @@ import (
 )
 
 type Interface interface {
-	// Current stats about log store, eg. total number of logs and containers
 	GetCurrentStats(sf SearchFilter) (Statistics, error)
-
 	CountLogsByInterval(sf SearchFilter, interval string) (Histogram, error)
-
 	SearchLogs(sf SearchFilter, from, size int64, order string) (Logs, error)
-
 	ExportLogs(sf SearchFilter, w io.Writer) error
 }
 

tools/cmd/doc-gen/main.go

@@ -38,7 +38,6 @@ import (
 	"kubesphere.io/kubesphere/pkg/informers"
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
-	loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
 	monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
 	networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
 	openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
@@ -114,13 +113,12 @@ func generateSwaggerJson() []byte {
 
 	urlruntime.Must(devopsv1alpha2.AddToContainer(container, informerFactory.KubeSphereSharedInformerFactory(), &fake.Devops{}, nil, clientsets.KubeSphere(), fakes3.NewFakeS3()))
 	urlruntime.Must(iamv1alpha2.AddToContainer(container, im.NewOperator(clientsets.KubeSphere(), informerFactory), am.NewReadOnlyOperator(informerFactory), authoptions.NewAuthenticateOptions()))
-	urlruntime.Must(loggingv1alpha2.AddToContainer(container, clientsets, nil))
 	urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil, informerFactory, nil))
 	urlruntime.Must(openpitrixv1.AddToContainer(container, informerFactory, nil))
 	urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes()))
 	urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory))
 	urlruntime.Must(resourcesv1alpha3.AddToContainer(container, informerFactory))
-	urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil))
+	urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil))
 	urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil))
 	urlruntime.Must(metricsv1alpha2.AddToContainer(container))
 	urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))