admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2100 from huanggze/logging

Browse Source 
logging: integrate new IAM
This commit is contained in:
KubeSphere CI Bot
2020-05-25 21:43:34 +08:00
committed by GitHub
parent c104516ce0 9b6f38d857
commit b8f6a97340
36 changed files with 1735 additions and 813 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
pkg/api/logging/v1alpha2/types.go
View File

@@ -1,9 +1,103 @@
package v1alpha2
import "kubesphere.io/kubesphere/pkg/simple/client/logging"
import (
"github.com/emicklei/go-restful"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"strconv"
"time"
)
const (
OperationStatistics = "statistics"
OperationHistogram = "histogram"
OperationQuery = "query"
OperationExport = "export"
DefaultInterval = "15m"
DefaultSize = 10
OrderAscending = "asc"
OrderDescending = "desc"
)
type APIResponse struct {
Logs *logging.Logs `json:"query,omitempty" description:"query results"`
Statistics *logging.Statistics `json:"statistics,omitempty" description:"statistics results"`
Histogram *logging.Histogram `json:"histogram,omitempty" description:"histogram results"`
}
type Query struct {
Operation string
WorkspaceFilter string
WorkspaceSearch string
NamespaceFilter string
NamespaceSearch string
WorkloadFilter string
WorkloadSearch string
PodFilter string
PodSearch string
ContainerFilter string
ContainerSearch string
LogSearch string
StartTime time.Time
EndTime time.Time
Interval string
Sort string
From int64
Size int64
}
func ParseQueryParameter(req *restful.Request) (*Query, error) {
var q Query
q.Operation = req.QueryParameter("operation")
q.WorkspaceFilter = req.QueryParameter("workspaces")
q.WorkspaceSearch = req.QueryParameter("workspace_query")
q.NamespaceFilter = req.QueryParameter("namespaces")
q.NamespaceSearch = req.QueryParameter("namespace_query")
q.WorkloadFilter = req.QueryParameter("workloads")
q.WorkloadSearch = req.QueryParameter("workload_query")
q.PodFilter = req.QueryParameter("pods")
q.PodSearch = req.QueryParameter("pod_query")
q.ContainerFilter = req.QueryParameter("containers")
q.ContainerSearch = req.QueryParameter("container_query")
q.LogSearch = req.QueryParameter("log_query")
if q.Operation == "" {
q.Operation = OperationQuery
}
if tstr := req.QueryParameter("start_time"); tstr != "" {
sec, err := strconv.ParseInt(tstr, 10, 64)
if err != nil {
return nil, err
}
q.StartTime = time.Unix(sec, 0)
}
if tstr := req.QueryParameter("end_time"); tstr != "" {
sec, err := strconv.ParseInt(tstr, 10, 64)
if err != nil {
return nil, err
}
q.EndTime = time.Unix(sec, 0)
}
switch q.Operation {
case OperationHistogram:
q.Interval = req.QueryParameter("interval")
if q.Interval == "" {
q.Interval = DefaultInterval
}
case OperationQuery:
q.From, _ = strconv.ParseInt(req.QueryParameter("from"), 10, 64)
size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64)
if err != nil {
size = DefaultSize
}
q.Size = size
q.Sort = req.QueryParameter("sort")
if q.Sort != OrderAscending {
q.Sort = OrderDescending
}
}
return &q, nil
}

4
pkg/apiserver/apiserver.go
View File

@@ -49,7 +49,6 @@ import (
devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
notificationv1 "kubesphere.io/kubesphere/pkg/kapis/notification/v1"
@@ -165,7 +164,6 @@ func (s *APIServer) PrepareRun() error {
func (s *APIServer) installKubeSphereAPIs() {
urlruntime.Must(configv1alpha2.AddToContainer(s.container, s.Config))
urlruntime.Must(resourcev1alpha3.AddToContainer(s.container, s.InformerFactory))
urlruntime.Must(loggingv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.LoggingClient))
urlruntime.Must(monitoringv1alpha3.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.MonitoringClient, s.InformerFactory, s.OpenpitrixClient))
urlruntime.Must(openpitrixv1.AddToContainer(s.container, s.InformerFactory, s.OpenpitrixClient))
urlruntime.Must(networkv1alpha2.AddToContainer(s.container, s.Config.NetworkOptions.WeaveScopeHost))
@@ -173,7 +171,7 @@ func (s *APIServer) installKubeSphereAPIs() {
urlruntime.Must(resourcesv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.InformerFactory,
s.KubernetesClient.Master()))
urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.InformerFactory, s.KubernetesClient.Kubernetes(),
s.KubernetesClient.KubeSphere(), s.EventsClient))
s.KubernetesClient.KubeSphere(), s.EventsClient, s.LoggingClient))
urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config()))
urlruntime.Must(clusterkapisv1alpha1.AddToContainer(s.container,
s.InformerFactory.KubernetesSharedInformerFactory(),

18
pkg/kapis/logging/group.go
View File

@@ -1,18 +0,0 @@
/*
Copyright 2019 The KubeSphere authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package logging contains logging API versions
package logging

156
pkg/kapis/logging/v1alpha2/handler.go
View File

@@ -1,156 +0,0 @@
package v1alpha2
import (
"github.com/emicklei/go-restful"
"kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/models/logging"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging"
util "kubesphere.io/kubesphere/pkg/utils/stringutils"
"strconv"
"strings"
"time"
)
const (
LevelCluster = iota
LevelContainer
// query type, default to `query`
TypeStat = "statistics"
TypeHist = "histogram"
TypeExport = "export"
Ascending = "asc"
Descending = "desc"
)
type handler struct {
k k8s.Client
lo logging.LoggingOperator
}
func newHandler(k k8s.Client, l loggingclient.Interface) *handler {
return &handler{k, logging.NewLoggingOperator(l)}
}
func (h handler) handleClusterQuery(req *restful.Request, resp *restful.Response) {
h.get(req, LevelCluster, resp)
}
func (h handler) handleContainerQuery(req *restful.Request, resp *restful.Response) {
h.get(req, LevelContainer, resp)
}
func (h handler) get(req *restful.Request, lvl int, resp *restful.Response) {
typ := req.QueryParameter("type")
noHit, sf, err := h.newSearchFilter(req, lvl)
if err != nil {
api.HandleBadRequest(resp, nil, err)
}
if noHit {
handleNoHit(typ, resp)
return
}
switch typ {
case TypeStat:
res, err := h.lo.GetCurrentStats(sf)
if err != nil {
api.HandleInternalError(resp, nil, err)
}
resp.WriteAsJson(res)
case TypeHist:
interval := req.QueryParameter("interval")
res, err := h.lo.CountLogsByInterval(sf, interval)
if err != nil {
api.HandleInternalError(resp, nil, err)
}
resp.WriteAsJson(res)
case TypeExport:
resp.Header().Set(restful.HEADER_ContentType, "text/plain")
resp.Header().Set("Content-Disposition", "attachment")
err := h.lo.ExportLogs(sf, resp.ResponseWriter)
if err != nil {
api.HandleInternalError(resp, nil, err)
}
default:
from, _ := strconv.ParseInt(req.QueryParameter("from"), 10, 64)
size, err := strconv.ParseInt(req.QueryParameter("size"), 10, 64)
if err != nil {
size = 10
}
order := req.QueryParameter("sort")
if order != Ascending {
order = Descending
}
res, err := h.lo.SearchLogs(sf, from, size, order)
if err != nil {
api.HandleInternalError(resp, nil, err)
}
resp.WriteAsJson(res)
}
}
func (h handler) newSearchFilter(req *restful.Request, level int) (bool, loggingclient.SearchFilter, error) {
var sf loggingclient.SearchFilter
switch level {
case LevelCluster:
sf.NamespaceFilter = h.intersect(
util.Split(req.QueryParameter("namespaces"), ","),
util.Split(strings.ToLower(req.QueryParameter("namespace_query")), ","),
util.Split(req.QueryParameter("workspaces"), ","),
util.Split(strings.ToLower(req.QueryParameter("workspace_query")), ","))
sf.WorkloadFilter = util.Split(req.QueryParameter("workloads"), ",")
sf.WorkloadSearch = util.Split(req.QueryParameter("workload_query"), ",")
sf.PodFilter = util.Split(req.QueryParameter("pods"), ",")
sf.PodSearch = util.Split(req.QueryParameter("pod_query"), ",")
sf.ContainerFilter = util.Split(req.QueryParameter("containers"), ",")
sf.ContainerSearch = util.Split(req.QueryParameter("container_query"), ",")
case LevelContainer:
sf.NamespaceFilter = h.withCreationTime(req.PathParameter("namespace"))
sf.PodFilter = []string{req.PathParameter("pod")}
sf.ContainerFilter = []string{req.PathParameter("container")}
}
sf.LogSearch = util.Split(req.QueryParameter("log_query"), ",")
var err error
now := time.Now()
// If time is not given, set it to now.
if req.QueryParameter("start_time") == "" {
sf.Starttime = now
} else {
sf.Starttime, err = time.Parse(time.RFC3339, req.QueryParameter("start_time"))
if err != nil {
return false, sf, err
}
}
if req.QueryParameter("end_time") == "" {
sf.Endtime = now
} else {
sf.Endtime, err = time.Parse(time.RFC3339, req.QueryParameter("end_time"))
if err != nil {
return false, sf, err
}
}
return len(sf.NamespaceFilter) == 0, sf, nil
}
func handleNoHit(typ string, resp *restful.Response) {
switch typ {
case TypeStat:
resp.WriteAsJson(new(loggingclient.Statistics))
case TypeHist:
resp.WriteAsJson(new(loggingclient.Histogram))
case TypeExport:
resp.Header().Set(restful.HEADER_ContentType, "text/plain")
resp.Header().Set("Content-Disposition", "attachment")
resp.Write(nil)
default:
resp.WriteAsJson(new(loggingclient.Logs))
}
}

56
pkg/kapis/logging/v1alpha2/helper.go
View File

@@ -1,56 +0,0 @@
package v1alpha2
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/utils/stringutils"
"strings"
"time"
)
func (h handler) intersect(nsFilter []string, nsSearch []string, wsFilter []string, wsSearch []string) map[string]time.Time {
nsList, err := h.k.Kubernetes().CoreV1().Namespaces().List(v1.ListOptions{})
if err != nil {
klog.Errorf("failed to list namespace, error: %s", err)
return nil
}
inner := make(map[string]time.Time)
// if no search condition is set on both namespace and workspace,
// then return all namespaces
if nsSearch == nil && nsFilter == nil && wsSearch == nil && wsFilter == nil {
for _, ns := range nsList.Items {
inner[ns.Name] = ns.CreationTimestamp.Time
}
} else {
for _, ns := range nsList.Items {
if stringutils.StringIn(ns.Name, nsFilter) ||
stringutils.StringIn(ns.Annotations[constants.WorkspaceLabelKey], wsFilter) ||
containsIn(ns.Name, nsSearch) ||
containsIn(ns.Annotations[constants.WorkspaceLabelKey], wsSearch) {
inner[ns.Name] = ns.CreationTimestamp.Time
}
}
}
return inner
}
func containsIn(str string, subStrs []string) bool {
for _, sub := range subStrs {
if strings.Contains(str, sub) {
return true
}
}
return false
}
func (h handler) withCreationTime(name string) map[string]time.Time {
ns, err := h.k.Kubernetes().CoreV1().Namespaces().Get(name, v1.GetOptions{})
if err == nil {
return map[string]time.Time{name: ns.CreationTimestamp.Time}
}
return nil
}

93
pkg/kapis/logging/v1alpha2/register.go
View File

@@ -1,93 +0,0 @@
/*
Copyright 2019 The KubeSphere Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha2
import (
"github.com/emicklei/go-restful"
"github.com/emicklei/go-restful-openapi"
"k8s.io/apimachinery/pkg/runtime/schema"
"kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/simple/client/k8s"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"net/http"
)
const (
GroupName = "logging.kubesphere.io"
RespOK = "ok"
)
var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
func AddToContainer(c *restful.Container, k8sClient k8s.Client, loggingClient logging.Interface) error {
ws := runtime.NewWebService(GroupVersion)
h := newHandler(k8sClient, loggingClient)
ws.Route(ws.GET("/cluster").
To(h.handleClusterQuery).
Doc("Query logs against the cluster.").
Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)).
Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)).
Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)).
Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)).
Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)).
Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
Writes(v1alpha2.APIResponse{}).
Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON, "text/plain")
ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}").
To(h.handleContainerQuery).
Doc("Query logs against the specific container.").
Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)).
Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)).
Param(ws.PathParameter("container", "Container name.").DataType("string").Required(true)).
Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
Writes(v1alpha2.APIResponse{}).
Returns(http.StatusOK, RespOK, v1alpha2.APIResponse{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON, restful.MIME_OCTET)
c.Add(ws)
return nil
}

41
pkg/kapis/tenant/v1alpha2/handler.go
View File

@@ -9,6 +9,7 @@ import (
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/api"
eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/apiserver/request"
@@ -17,16 +18,17 @@ import (
"kubesphere.io/kubesphere/pkg/models/tenant"
servererr "kubesphere.io/kubesphere/pkg/server/errors"
"kubesphere.io/kubesphere/pkg/simple/client/events"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
)
type tenantHandler struct {
tenant tenant.Interface
}
func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) *tenantHandler {
func newTenantHandler(factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) *tenantHandler {
return &tenantHandler{
tenant: tenant.New(factory, k8sclient, ksclient, evtsClient),
tenant: tenant.New(factory, k8sclient, ksclient, evtsClient, loggingClient),
}
}
@@ -245,3 +247,38 @@ func (h *tenantHandler) Events(req *restful.Request, resp *restful.Response) {
resp.WriteEntity(result)
}
func (h *tenantHandler) QueryLogs(req *restful.Request, resp *restful.Response) {
user, ok := request.UserFrom(req.Request.Context())
if !ok {
err := fmt.Errorf("cannot obtain user info")
klog.Errorln(err)
api.HandleForbidden(resp, req, err)
return
}
queryParam, err := loggingv1alpha2.ParseQueryParameter(req)
if err != nil {
klog.Errorln(err)
api.HandleInternalError(resp, req, err)
return
}
if queryParam.Operation == loggingv1alpha2.OperationExport {
resp.Header().Set(restful.HEADER_ContentType, "text/plain")
resp.Header().Set("Content-Disposition", "attachment")
err := h.tenant.ExportLogs(user, queryParam, resp)
if err != nil {
klog.Errorln(err)
api.HandleInternalError(resp, req, err)
return
}
} else {
result, err := h.tenant.QueryLogs(user, queryParam)
if err != nil {
klog.Errorln(err)
api.HandleInternalError(resp, req, err)
return
}
resp.WriteAsJson(result)
}
}

35
pkg/kapis/tenant/v1alpha2/register.go
View File

@@ -24,6 +24,7 @@ import (
"k8s.io/client-go/kubernetes"
"kubesphere.io/kubesphere/pkg/api"
eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
@@ -32,6 +33,7 @@ import (
"kubesphere.io/kubesphere/pkg/models"
"kubesphere.io/kubesphere/pkg/server/errors"
"kubesphere.io/kubesphere/pkg/simple/client/events"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"net/http"
)
@@ -41,9 +43,9 @@ const (
var GroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1alpha2"}
func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client) error {
func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient events.Client, loggingClient logging.Interface) error {
ws := runtime.NewWebService(GroupVersion)
handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient)
handler := newTenantHandler(factory, k8sclient, ksclient, evtsClient, loggingClient)
ws.Route(ws.POST("/workspaces").
To(handler.CreateWorkspace).
@@ -95,7 +97,7 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s
ws.Route(ws.GET("/events").
To(handler.Events).
Doc("Query events against the cluster").
Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")).
Param(ws.QueryParameter("operation", "Operation type. This can be one of three types: `query` (for querying events), `statistics` (for retrieving statistical data), `histogram` (for displaying events count by time interval). Defaults to query.").DefaultValue("query")).
Param(ws.QueryParameter("workspace_filter", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`.")).
Param(ws.QueryParameter("workspace_search", "A comma-separated list of keywords. Differing from **workspace_filter**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.")).
Param(ws.QueryParameter("involved_object_namespace_filter", "A comma-separated list of namespaces. This field restricts the query to specified `involvedObject.namespace`.")).
@@ -117,6 +119,33 @@ func AddToContainer(c *restful.Container, factory informers.InformerFactory, k8s
Writes(eventsv1alpha1.APIResponse{}).
Returns(http.StatusOK, api.StatusOK, eventsv1alpha1.APIResponse{}))
ws.Route(ws.GET("/logs").
To(handler.QueryLogs).
Doc("Query logs against the cluster.").
Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)).
Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)).
Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)).
Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)).
Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)).
Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)).
Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)).
Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)).
Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)).
Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)).
Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing seconds since the epoch, eg. 1559664000.").DataType("string").Required(false)).
Param(ws.QueryParameter("sort", "Sort order. One of asc, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)).
Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)).
Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.LogQueryTag}).
Writes(loggingv1alpha2.APIResponse{}).
Returns(http.StatusOK, api.StatusOK, loggingv1alpha2.APIResponse{})).
Consumes(restful.MIME_JSON, restful.MIME_XML).
Produces(restful.MIME_JSON, "text/plain")
c.Add(ws)
return nil
}

134
pkg/models/tenant/tenant.go
View File

@@ -18,6 +18,7 @@ package tenant
import (
"fmt"
"io"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,9 +28,9 @@ import (
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/api"
eventsv1alpha1 "kubesphere.io/kubesphere/pkg/api/events/v1alpha1"
loggingv1alpha2 "kubesphere.io/kubesphere/pkg/api/logging/v1alpha2"
clusterv1alpha1 "kubesphere.io/kubesphere/pkg/apis/cluster/v1alpha1"
tenantv1alpha1 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1"
tenantv1alpha2 "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha2"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer"
"kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory"
@@ -38,9 +39,11 @@ import (
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/models/events"
"kubesphere.io/kubesphere/pkg/models/iam/am"
"kubesphere.io/kubesphere/pkg/models/logging"
resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
resourcesv1alpha3 "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/resource"
eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events"
loggingclient "kubesphere.io/kubesphere/pkg/simple/client/logging"
"kubesphere.io/kubesphere/pkg/utils/stringutils"
"strings"
"time"
@@ -57,6 +60,8 @@ type Interface interface {
ListWorkspaceClusters(workspace string) (*api.ListResult, error)
Events(user user.Info, queryParam *eventsv1alpha1.Query) (*eventsv1alpha1.APIResponse, error)
QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error)
ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error
}
type tenantOperator struct {
@@ -66,9 +71,10 @@ type tenantOperator struct {
ksclient kubesphere.Interface
resourceGetter *resourcesv1alpha3.ResourceGetter
events events.Interface
lo logging.LoggingOperator
}
func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client) Interface {
func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ksclient kubesphere.Interface, evtsClient eventsclient.Client, loggingClient loggingclient.Interface) Interface {
amOperator := am.NewReadOnlyOperator(informers)
authorizer := authorizerfactory.NewRBACAuthorizer(amOperator)
return &tenantOperator{
@@ -78,6 +84,7 @@ func New(informers informers.InformerFactory, k8sclient kubernetes.Interface, ks
k8sclient: k8sclient,
ksclient: ksclient,
events: events.NewEventsOperator(evtsClient),
lo: logging.NewLoggingOperator(loggingClient),
}
}
@@ -406,6 +413,129 @@ func (t *tenantOperator) Events(user user.Info, queryParam *eventsv1alpha1.Query
})
}
func (t *tenantOperator) QueryLogs(user user.Info, query *loggingv1alpha2.Query) (*loggingv1alpha2.APIResponse, error) {
iNamespaces, err := t.listIntersectedNamespaces(user,
stringutils.Split(query.WorkspaceFilter, ","),
stringutils.Split(query.WorkspaceSearch, ","),
stringutils.Split(query.NamespaceFilter, ","),
stringutils.Split(query.NamespaceSearch, ","))
if err != nil {
klog.Error(err)
return nil, err
}
namespaceCreateTimeMap := make(map[string]time.Time)
for _, ns := range iNamespaces {
podLogs := authorizer.AttributesRecord{
User: user,
Verb: "get",
APIGroup: "",
APIVersion: "v1",
Namespace: ns.Name,
Resource: "pods",
Subresource: "log",
ResourceRequest: true,
}
decision, _, err := t.authorizer.Authorize(podLogs)
if err != nil {
klog.Error(err)
return nil, err
}
if decision == authorizer.DecisionAllow {
namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time
}
}
sf := loggingclient.SearchFilter{
NamespaceFilter: namespaceCreateTimeMap,
WorkloadSearch: stringutils.Split(query.WorkloadSearch, ","),
WorkloadFilter: stringutils.Split(query.WorkloadFilter, ","),
PodSearch: stringutils.Split(query.PodSearch, ","),
PodFilter: stringutils.Split(query.PodFilter, ","),
ContainerSearch: stringutils.Split(query.ContainerSearch, ","),
ContainerFilter: stringutils.Split(query.ContainerFilter, ","),
LogSearch: stringutils.Split(query.LogSearch, ","),
Starttime: query.StartTime,
Endtime: query.EndTime,
}
var ar loggingv1alpha2.APIResponse
switch query.Operation {
case loggingv1alpha2.OperationStatistics:
if len(namespaceCreateTimeMap) == 0 {
ar.Statistics = &loggingclient.Statistics{}
} else {
ar, err = t.lo.GetCurrentStats(sf)
}
case loggingv1alpha2.OperationHistogram:
if len(namespaceCreateTimeMap) == 0 {
ar.Histogram = &loggingclient.Histogram{}
} else {
ar, err = t.lo.CountLogsByInterval(sf, query.Interval)
}
default:
if len(namespaceCreateTimeMap) == 0 {
ar.Logs = &loggingclient.Logs{}
} else {
ar, err = t.lo.SearchLogs(sf, query.From, query.Size, query.Sort)
}
}
return &ar, err
}
func (t *tenantOperator) ExportLogs(user user.Info, query *loggingv1alpha2.Query, writer io.Writer) error {
iNamespaces, err := t.listIntersectedNamespaces(user,
stringutils.Split(query.WorkspaceFilter, ","),
stringutils.Split(query.WorkspaceSearch, ","),
stringutils.Split(query.NamespaceFilter, ","),
stringutils.Split(query.NamespaceSearch, ","))
if err != nil {
klog.Error(err)
return err
}
namespaceCreateTimeMap := make(map[string]time.Time)
for _, ns := range iNamespaces {
podLogs := authorizer.AttributesRecord{
User: user,
Verb: "get",
APIGroup: "",
APIVersion: "v1",
Namespace: ns.Name,
Resource: "pods",
Subresource: "log",
ResourceRequest: true,
}
decision, _, err := t.authorizer.Authorize(podLogs)
if err != nil {
klog.Error(err)
return err
}
if decision == authorizer.DecisionAllow {
namespaceCreateTimeMap[ns.Name] = ns.CreationTimestamp.Time
}
}
sf := loggingclient.SearchFilter{
NamespaceFilter: namespaceCreateTimeMap,
WorkloadSearch: stringutils.Split(query.WorkloadSearch, ","),
WorkloadFilter: stringutils.Split(query.WorkloadFilter, ","),
PodSearch: stringutils.Split(query.PodSearch, ","),
PodFilter: stringutils.Split(query.PodFilter, ","),
ContainerSearch: stringutils.Split(query.ContainerSearch, ","),
ContainerFilter: stringutils.Split(query.ContainerFilter, ","),
LogSearch: stringutils.Split(query.LogSearch, ","),
Starttime: query.StartTime,
Endtime: query.EndTime,
}
if len(namespaceCreateTimeMap) == 0 {
return nil
} else {
return t.lo.ExportLogs(sf, writer)
}
}
func contains(objects []runtime.Object, object runtime.Object) bool {
for _, item := range objects {
if item == object {

2
pkg/models/tenant/tenent_test.go
View File

@@ -328,5 +328,5 @@ func prepare() Interface {
RoleBindings().Informer().GetIndexer().Add(roleBinding)
}
return New(fakeInformerFactory, nil, nil, nil)
return New(fakeInformerFactory, nil, nil, nil, nil)
}

86
pkg/simple/client/logging/elasticsearch/api_body.go
View File

@@ -2,6 +2,7 @@ package elasticsearch
import (
"fmt"
"github.com/json-iterator/go"
"k8s.io/klog"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"time"
@@ -13,6 +14,9 @@ const (
replicaSetSuffixMaxLength = 11 // max 10 characters + 1 hyphen
)
// TODO: elastic/go-elasticsearch is working on Query DSL support.
// See https://github.com/elastic/go-elasticsearch/issues/42.
// We need refactor our query body builder when that is ready.
type bodyBuilder struct {
Body
}
@@ -22,75 +26,9 @@ func newBodyBuilder() *bodyBuilder {
}
func (bb *bodyBuilder) bytes() ([]byte, error) {
return json.Marshal(bb.Body)
return jsoniter.Marshal(bb.Body)
}
// The mainBody func builds api body for query.
// TODO: Should use an elegant pakcage for building query body, but `elastic/go-elasticsearch` doesn't provide it currently.
//
// Example:
// GET kapis/logging.kubesphere.io/v1alpha2/cluster?start_time=0&end_time=156576063993&namespaces=kubesphere-system&pod_query=ks-apiserver
// -----
//{
// "from":0,
// "size":10,
// "sort":[
// {
// "time": "desc"
// }
// ],
// "query":{
// "bool":{
// "filter":[
// {
// "bool":{
// "should":[
// {
// "bool":{
// "filter":[
// {
// "match_phrase":{
// "kubernetes.namespace_name.keyword":"kubesphere-system"
// }
// },
// {
// "range":{
// "time":{
// "gte":"1572315987000"
// }
// }
// }
// ]
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "bool":{
// "should":[
// {
// "match_phrase_prefix":{
// "kubernetes.pod_name":"ks-apiserver"
// }
// }
// ],
// "minimum_should_match":1
// }
// },
// {
// "range":{
// "time":{
// "gte":"0",
// "lte":"156576063993"
// }
// }
// }
// ]
// }
// }
//}
func (bb *bodyBuilder) mainBool(sf logging.SearchFilter) *bodyBuilder {
var ms []Match
@@ -207,10 +145,6 @@ func (bb *bodyBuilder) cardinalityAggregation() *bodyBuilder {
}
func (bb *bodyBuilder) dateHistogramAggregation(interval string) *bodyBuilder {
if interval == "" {
interval = "15m"
}
bb.Body.Aggs = &Aggs{
DateHistogramAggregation: &DateHistogramAggregation{
&DateHistogram{
@@ -232,12 +166,8 @@ func (bb *bodyBuilder) size(n int64) *bodyBuilder {
return bb
}
func (bb *bodyBuilder) sort(o string) *bodyBuilder {
if o != "asc" {
o = "desc"
}
bb.Sorts = []map[string]string{{"time": o}}
func (bb *bodyBuilder) sort(order string) *bodyBuilder {
bb.Sorts = []map[string]string{{"time": order}}
return bb
}
@@ -268,7 +198,7 @@ func podNameRegexp(workloadName string) string {
func parseResponse(body []byte) (Response, error) {
var res Response
err := json.Unmarshal(body, &res)
err := jsoniter.Unmarshal(body, &res)
if err != nil {
klog.Error(err)
return Response{}, err

137
pkg/simple/client/logging/elasticsearch/api_body_test.go
View File

@@ -1,50 +1,117 @@
package elasticsearch
import (
"fmt"
"github.com/google/go-cmp/cmp"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
"reflect"
"testing"
"time"
)
func TestCardinalityAggregation(t *testing.T) {
var test = struct {
description string
searchFilter logging.SearchFilter
expected *bodyBuilder
func TestMainBool(t *testing.T) {
var tests = []struct {
filter logging.SearchFilter
expected string
}{
description: "add cardinality aggregation",
searchFilter: logging.SearchFilter{
LogSearch: []string{"info"},
{
filter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"default": time.Unix(1589981934, 0),
},
},
expected: "api_body_1.json",
},
expected: &bodyBuilder{Body{
Query: &Query{
Bool: Bool{
Filter: []Match{
{
Bool: &Bool{
Should: []Match{
{
MatchPhrasePrefix: map[string]string{"log": "info"},
},
},
MinimumShouldMatch: 1,
},
},
},
},
{
filter: logging.SearchFilter{
WorkloadFilter: []string{"mysql"},
Starttime: time.Unix(1589980934, 0),
Endtime: time.Unix(1589981934, 0),
},
Aggs: &Aggs{
CardinalityAggregation: &CardinalityAggregation{
Cardinality: &Cardinality{Field: "kubernetes.docker_id.keyword"},
},
expected: "api_body_2.json",
},
{
filter: logging.SearchFilter{
PodFilter: []string{"mysql"},
PodSearch: []string{"mysql-a8w3s-10945j"},
LogSearch: []string{"info"},
},
}},
expected: "api_body_3.json",
},
{
filter: logging.SearchFilter{
ContainerFilter: []string{"mysql-1"},
ContainerSearch: []string{"mysql-3"},
},
expected: "api_body_4.json",
},
}
t.Run(test.description, func(t *testing.T) {
body := newBodyBuilder().mainBool(test.searchFilter).cardinalityAggregation()
if diff := cmp.Diff(body, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().mainBool(test.filter).Body
if diff := cmp.Diff(result, expected); diff != "" {
fmt.Printf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func TestCardinalityAggregation(t *testing.T) {
var tests = []struct {
expected string
}{
{
expected: "api_body_5.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().cardinalityAggregation().Body
if !reflect.DeepEqual(result, expected) {
t.Fatalf("expected: %v, but got %v", expected, result)
}
})
}
}
func TestDateHistogramAggregation(t *testing.T) {
var tests = []struct {
expected string
}{
{
expected: "api_body_6.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected Body
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
result := newBodyBuilder().dateHistogramAggregation("15m").Body
if !reflect.DeepEqual(result, expected) {
t.Fatalf("expected: %v, but got %v", expected, result)
}
})
}
}

73
pkg/simple/client/logging/elasticsearch/elasticsearch.go
View File

@@ -4,12 +4,12 @@ import (
"bytes"
"context"
"fmt"
jsoniter "github.com/json-iterator/go"
"github.com/json-iterator/go"
"io"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/utils/stringutils"
"strings"
)
@@ -20,8 +20,6 @@ const (
ElasticV7 = "7"
)
var json = jsoniter.ConfigCompatibleWithStandardLibrary
// Elasticsearch implement logging interface
type Elasticsearch struct {
c client
@@ -29,8 +27,7 @@ type Elasticsearch struct {
// versioned es client interface
type client interface {
// Perform Search API
Search(body []byte) ([]byte, error)
Search(body []byte, scroll bool) ([]byte, error)
Scroll(id string) ([]byte, error)
ClearScroll(id string)
GetTotalHitCount(v interface{}) int64
@@ -83,11 +80,10 @@ func detectVersionMajor(host string) (string, error) {
if err != nil {
return "", err
}
defer res.Body.Close()
var b map[string]interface{}
if err = json.NewDecoder(res.Body).Decode(&b); err != nil {
if err = jsoniter.NewDecoder(res.Body).Decode(&b); err != nil {
return "", err
}
if res.IsError() {
@@ -116,7 +112,7 @@ func (es Elasticsearch) GetCurrentStats(sf logging.SearchFilter) (logging.Statis
return logging.Statistics{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, true)
if err != nil {
return logging.Statistics{}, err
}
@@ -142,7 +138,7 @@ func (es Elasticsearch) CountLogsByInterval(sf logging.SearchFilter, interval st
return logging.Histogram{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, false)
if err != nil {
return logging.Histogram{}, err
}
@@ -174,7 +170,7 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
return logging.Logs{}, err
}
b, err := es.c.Search(body)
b, err := es.c.Search(body, false)
if err != nil {
return logging.Logs{}, err
}
@@ -200,33 +196,50 @@ func (es Elasticsearch) SearchLogs(sf logging.SearchFilter, f, s int64, o string
func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
var id string
var from int64 = 0
var size int64 = 1000
var data []string
res, err := es.SearchLogs(sf, from, size, "desc")
defer es.ClearScroll(id)
// Initial Search
body, err := newBodyBuilder().
mainBool(sf).
from(0).
size(1000).
sort("desc").
bytes()
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
b, err := es.c.Search(body, true)
defer es.ClearScroll(id)
if err != nil {
return err
}
res, err := parseResponse(b)
if err != nil {
return err
}
id = res.ScrollId
for _, hit := range res.AllHits {
data = append(data, hit.Log)
}
if len(data) == 0 {
return nil
}
// limit to retrieve max 100k records
for i := 0; i < 100; i++ {
res, id, err = es.scroll(id)
data, id, err = es.scroll(id)
if err != nil {
return err
}
if res.Records == nil || len(res.Records) == 0 {
if len(data) == 0 {
return nil
}
output := new(bytes.Buffer)
for _, r := range res.Records {
output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(r.Log)))
for _, l := range data {
output.WriteString(fmt.Sprintf(`%s`, stringutils.StripAnsi(l)))
}
_, err = io.Copy(w, output)
if err != nil {
@@ -236,24 +249,22 @@ func (es Elasticsearch) ExportLogs(sf logging.SearchFilter, w io.Writer) error {
return nil
}
func (es *Elasticsearch) scroll(id string) (logging.Logs, string, error) {
func (es *Elasticsearch) scroll(id string) ([]string, string, error) {
b, err := es.c.Scroll(id)
if err != nil {
return logging.Logs{}, id, err
return nil, id, err
}
res, err := parseResponse(b)
if err != nil {
return logging.Logs{}, id, err
return nil, id, err
}
var l logging.Logs
var data []string
for _, hit := range res.AllHits {
l.Records = append(l.Records, logging.Record{
Log: hit.Log,
})
data = append(data, hit.Log)
}
return l, res.ScrollId, nil
return data, res.ScrollId, nil
}
func (es *Elasticsearch) ClearScroll(id string) {

515
pkg/simple/client/logging/elasticsearch/elasticsearch_test.go
View File

@@ -1,70 +1,46 @@
package elasticsearch
import (
"fmt"
"github.com/google/go-cmp/cmp"
"github.com/json-iterator/go"
"io/ioutil"
"kubesphere.io/kubesphere/pkg/simple/client/logging"
v5 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
v6 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
v7 "kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v5"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v6"
"kubesphere.io/kubesphere/pkg/simple/client/logging/elasticsearch/versions/v7"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func MockElasticsearchService(pattern string, fakeResp string) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
res.Write([]byte(fakeResp))
})
return httptest.NewServer(mux)
}
func TestDetectVersionMajor(t *testing.T) {
var tests = []struct {
description string
fakeResp string
expected string
expectedError bool
fakeResp string
expected string
}{
{
description: "detect es 6.x version number",
fakeResp: `{
"name" : "elasticsearch-logging-data-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
"version" : {
"number" : "6.7.0",
"build_flavor" : "oss",
"build_type" : "docker",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}`,
expected: ElasticV6,
expectedError: false,
fakeResp: "es6_detect_version_major_200.json",
expected: ElasticV6,
},
{
fakeResp: "es7_detect_version_major_200.json",
expected: ElasticV7,
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
es := mockElasticsearchService("/", test.fakeResp, http.StatusOK)
defer es.Close()
v, err := detectVersionMajor(es.URL)
if err == nil && test.expectedError {
t.Fatalf("expected error while got nothing")
} else if err != nil && !test.expectedError {
result, err := detectVersionMajor(es.URL)
if err != nil {
t.Fatal(err)
}
if v != test.expected {
t.Fatalf("expected get version %s, but got %s", test.expected, v)
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
@@ -72,297 +48,202 @@ func TestDetectVersionMajor(t *testing.T) {
func TestGetCurrentStats(t *testing.T) {
var tests = []struct {
description string
searchFilter logging.SearchFilter
fakeVersion string
fakeResp string
expected logging.Statistics
expectedError bool
fakeVersion string
fakeResp string
fakeCode int
expected logging.Statistics
expectedErr string
}{
{
description: "[es 6.x] run as admin",
searchFilter: logging.SearchFilter{},
fakeVersion: ElasticV6,
fakeResp: `{
"took": 171,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 241222,
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Hn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:29.015Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:29.015492329Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "I31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.103Z",
"log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
"time": "2020-02-28T19:25:33.103075891Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "JX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.113Z",
"log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
"time": "2020-02-28T19:25:33.113110332Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Kn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:34.168983384Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "LH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:25:34.168997393Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "NX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:42.868Z",
"log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n",
"time": "2020-02-28T19:25:42.868578188Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Q31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:13.881Z",
"log": "- name: BasePath\n",
"time": "2020-02-28T19:26:13.881180681Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "S31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:26:14.597702238Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "TH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
"time": "2020-02-28T19:26:14.597708432Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "UX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:15.920Z",
"log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
"time": "2020-02-28T19:26:15.920650572Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 93
}
}
}`,
fakeVersion: ElasticV6,
fakeResp: "es6_get_current_stats_200.json",
fakeCode: http.StatusOK,
expected: logging.Statistics{
Containers: 93,
Logs: 241222,
},
expectedError: false,
},
{
description: "[es 6.x] index not found",
searchFilter: logging.SearchFilter{
NamespaceFilter: map[string]time.Time{
"workspace-1-project-a": time.Unix(1582000000, 0),
"workspace-1-project-b": time.Unix(1582333333, 0),
},
},
fakeVersion: ElasticV6,
fakeResp: `{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
}
],
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
},
"status": 404
}`,
fakeResp: "es6_get_current_stats_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index",
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_get_current_stats_200.json",
fakeCode: http.StatusOK,
expected: logging.Statistics{
Containers: 0,
Logs: 0,
Containers: 48,
Logs: 9726,
},
expectedError: true,
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_get_current_stats_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-2020.05.2]",
},
}
for _, test := range tests {
t.Run(test.description, func(t *testing.T) {
es := MockElasticsearchService("/", test.fakeResp)
defer es.Close()
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
clientv5 := Elasticsearch{c: v5.New(es.URL, "ks-logstash-log")}
clientv6 := Elasticsearch{c: v6.New(es.URL, "ks-logstash-log")}
clientv7 := Elasticsearch{c: v7.New(es.URL, "ks-logstash-log")}
es := newElasticsearchClient(srv, test.fakeVersion)
var stats logging.Statistics
var err error
switch test.fakeVersion {
case ElasticV5:
stats, err = clientv5.GetCurrentStats(test.searchFilter)
case ElasticV6:
stats, err = clientv6.GetCurrentStats(test.searchFilter)
case ElasticV7:
stats, err = clientv7.GetCurrentStats(test.searchFilter)
result, err := es.GetCurrentStats(logging.SearchFilter{})
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if err != nil && !test.expectedError {
t.Fatal(err)
} else if diff := cmp.Diff(stats, test.expected); diff != "" {
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}
func TestCountLogsByInterval(t *testing.T) {
var tests = []struct {
fakeVersion string
fakeResp string
fakeCode int
expected logging.Histogram
expectedErr string
}{
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_200.json",
fakeCode: http.StatusOK,
expected: logging.Histogram{
Total: 10000,
Buckets: []logging.Bucket{
{
Time: 1589644800000,
Count: 410,
},
{
Time: 1589646600000,
Count: 7465,
},
{
Time: 1589648400000,
Count: 12790,
},
},
},
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_400.json",
fakeCode: http.StatusBadRequest,
expectedErr: "type: search_phase_execution_exception, reason: all shards failed",
},
{
fakeVersion: ElasticV7,
fakeResp: "es7_count_logs_by_interval_404.json",
fakeCode: http.StatusNotFound,
expectedErr: "type: index_not_found_exception, reason: no such index [ks-logstash-log-20]",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
es := newElasticsearchClient(srv, test.fakeVersion)
result, err := es.CountLogsByInterval(logging.SearchFilter{}, "15m")
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if diff := cmp.Diff(result, test.expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
}
})
}
}
func TestSearchLogs(t *testing.T) {
var tests = []struct {
fakeVersion string
fakeResp string
fakeCode int
expected string
expectedErr string
}{
{
fakeVersion: ElasticV7,
fakeResp: "es7_search_logs_200.json",
fakeCode: http.StatusOK,
expected: "es7_search_logs_200_result.json",
},
}
for i, test := range tests {
t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
var expected logging.Logs
err := JsonFromFile(test.expected, &expected)
if err != nil {
t.Fatal(err)
}
srv := mockElasticsearchService("/ks-logstash-log*/_search", test.fakeResp, test.fakeCode)
defer srv.Close()
es := newElasticsearchClient(srv, test.fakeVersion)
result, err := es.SearchLogs(logging.SearchFilter{}, 0, 10, "asc")
if test.expectedErr != "" {
if diff := cmp.Diff(fmt.Sprint(err), test.expectedErr); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", test.expectedErr, diff)
}
}
if diff := cmp.Diff(result, expected); diff != "" {
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
}
})
}
}
func mockElasticsearchService(pattern, fakeResp string, fakeCode int) *httptest.Server {
mux := http.NewServeMux()
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
b, _ := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", fakeResp))
res.WriteHeader(fakeCode)
res.Write(b)
})
return httptest.NewServer(mux)
}
func newElasticsearchClient(srv *httptest.Server, version string) Elasticsearch {
var es Elasticsearch
switch version {
case ElasticV5:
es = Elasticsearch{c: v5.New(srv.URL, "ks-logstash-log")}
case ElasticV6:
es = Elasticsearch{c: v6.New(srv.URL, "ks-logstash-log")}
case ElasticV7:
es = Elasticsearch{c: v7.New(srv.URL, "ks-logstash-log")}
}
return es
}
func JsonFromFile(expectedFile string, expectedJsonPtr interface{}) error {
json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile))
if err != nil {
return err
}
err = jsoniter.Unmarshal(json, expectedJsonPtr)
if err != nil {
return err
}
return nil
}

33
pkg/simple/client/logging/elasticsearch/testdata/api_body_1.json vendored Normal file
View File

@@ -0,0 +1,33 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"bool":{
"filter":[
{
"match_phrase":{
"kubernetes.namespace_name.keyword":"default"
}
},
{
"range":{
"time":{
"gte":"2020-05-20T21:38:54+08:00"
}
}
}
]
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

28
pkg/simple/client/logging/elasticsearch/testdata/api_body_2.json vendored Normal file
View File

@@ -0,0 +1,28 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"regexp":{
"kubernetes.pod_name.keyword":"mysql-[bcdfghjklmnpqrstvwxz2456789]{1,10}-[a-z0-9]{5}|mysql-[0-9]+|mysql-[a-z0-9]{5}"
}
}
],
"minimum_should_match":1
}
},
{
"range":{
"time":{
"gte":"2020-05-20T21:22:14+08:00",
"lte":"2020-05-20T21:38:54+08:00"
}
}
}
]
}
}
}

44
pkg/simple/client/logging/elasticsearch/testdata/api_body_3.json vendored Normal file
View File

@@ -0,0 +1,44 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"match_phrase":{
"kubernetes.pod_name.keyword":"mysql"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"kubernetes.pod_name":"mysql-a8w3s-10945j"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"log":"info"
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

32
pkg/simple/client/logging/elasticsearch/testdata/api_body_4.json vendored Normal file
View File

@@ -0,0 +1,32 @@
{
"query":{
"bool":{
"filter":[
{
"bool":{
"should":[
{
"match_phrase":{
"kubernetes.container_name.keyword":"mysql-1"
}
}
],
"minimum_should_match":1
}
},
{
"bool":{
"should":[
{
"match_phrase_prefix":{
"kubernetes.container_name":"mysql-3"
}
}
],
"minimum_should_match":1
}
}
]
}
}
}

9
pkg/simple/client/logging/elasticsearch/testdata/api_body_5.json vendored Normal file
View File

@@ -0,0 +1,9 @@
{
"aggs":{
"container_count":{
"cardinality":{
"field":"kubernetes.docker_id.keyword"
}
}
}
}

10
pkg/simple/client/logging/elasticsearch/testdata/api_body_6.json vendored Normal file
View File

@@ -0,0 +1,10 @@
{
"aggs":{
"log_count_over_time":{
"date_histogram":{
"field":"time",
"interval":"15m"
}
}
}
}

17
pkg/simple/client/logging/elasticsearch/testdata/es6_detect_version_major_200.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"name" : "elasticsearch-logging-data-0",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "uLm0838MSd60T1XEh5P2Qg",
"version" : {
"number" : "6.7.0",
"build_flavor" : "oss",
"build_type" : "docker",
"build_hash" : "8453f77",
"build_date" : "2019-03-21T15:32:29.844721Z",
"build_snapshot" : false,
"lucene_version" : "7.7.0",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}

211
pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_200.json vendored Normal file
View File

@@ -0,0 +1,211 @@
{
"took": 171,
"timed_out": false,
"_shards": {
"total": 10,
"successful": 10,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 241222,
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Hn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:29.015Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:29.015492329Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "I31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.103Z",
"log": "I0228 19:25:33.102631 1 controller.go:1040] provision \"kubesphere-system/redis-pvc\" class \"local\": trying to save persistentvolume \"pvc-be6d127d-9366-4ea8-b1ce-f30c1b3a447b\"\n",
"time": "2020-02-28T19:25:33.103075891Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "JX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:33.113Z",
"log": "I0228 19:25:33.112200 1 controller.go:1088] provision \"kubesphere-system/redis-pvc\" class \"local\": succeeded\n",
"time": "2020-02-28T19:25:33.113110332Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Kn1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"hostpath\"\n",
"time": "2020-02-28T19:25:34.168983384Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "LH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:34.168Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:25:34.168997393Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "NX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:25:42.868Z",
"log": "I0228 19:25:42.868413 1 config.go:83] SC local has config:- name: StorageType\n",
"time": "2020-02-28T19:25:42.868578188Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "Q31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:13.881Z",
"log": "- name: BasePath\n",
"time": "2020-02-28T19:26:13.881180681Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "S31GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": " value: \"/var/openebs/local/\"\n",
"time": "2020-02-28T19:26:14.597702238Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "TH1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:14.597Z",
"log": "I0228 19:26:14.597007 1 provisioner_hostpath.go:42] Creating volume pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a at ks-allinone:/var/openebs/local/pvc-c3b1e67f-00d2-407d-8c45-690bb273c16a\n",
"time": "2020-02-28T19:26:14.597708432Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
},
{
"_index": "ks-logstash-log-2020.02.28",
"_type": "flb_type",
"_id": "UX1GjXABMO5aQxyNsyxy",
"_score": 1.0,
"_source": {
"@timestamp": "2020-02-28T19:26:15.920Z",
"log": "I0228 19:26:15.915071 1 event.go:221] Event(v1.ObjectReference{Kind:\"PersistentVolumeClaim\", Namespace:\"kubesphere-system\", Name:\"mysql-pvc\", UID:\"1e87deb5-eaec-475f-8eb6-8613b3be80a4\", APIVersion:\"v1\", ResourceVersion:\"2397\", FieldPath:\"\"}): type: 'Normal' reason: 'ProvisioningSucceeded' Successfully provisioned volume pvc-1e87deb5-eaec-475f-8eb6-8613b3be80a4\n",
"time": "2020-02-28T19:26:15.920650572Z",
"kubernetes": {
"pod_name": "openebs-localpv-provisioner-55c66b57b4-jgtjc",
"namespace_name": "kube-system",
"host": "ks-allinone",
"container_name": "openebs-localpv-provisioner",
"docker_id": "cac01cd01cc79d8a8903ddbe6fbde9ac7497919a3f33c61861443703a9e08b39",
"container_hash": "25d789bcd3d12a4ba50bbb56eed1de33279d04352adbba8fd7e3b7b938aec806"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 93
}
}
}

21
pkg/simple/client/logging/elasticsearch/testdata/es6_get_current_stats_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
}
],
"type": "index_not_found_exception",
"reason": "no such index",
"resource.type": "index_or_alias",
"resource.id": "ks-lsdfsdfsdfs",
"index_uuid": "_na_",
"index": "ks-lsdfsdfsdfs"
},
"status": 404
}

230
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_200.json vendored Normal file
View File

@@ -0,0 +1,230 @@
{
"took": 23,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "jxt2MXIBlcWZ594bpIVN",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:07.648Z",
"log": "ls: cannot access '/calico-secrets': No such file or directory\n",
"time": "2020-05-16T16:00:07.64848716Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.164Z",
"log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
"time": "2020-05-16T16:00:08.164003996Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "CNI plugin version: v3.7.3\n",
"time": "2020-05-16T16:00:08.350585959Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kht2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "/host/secondary-bin-dir is non-writeable, skipping\n",
"time": "2020-05-16T16:00:08.350625112Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kxt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
"time": "2020-05-16T16:00:08.350692011Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": "CNI config: {\n",
"time": "2020-05-16T16:00:08.454417144Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": " \"name\": \"cni0\",\n",
"time": "2020-05-16T16:00:08.454452649Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
}
]
},
"aggregations": {
"log_count_over_time": {
"buckets": [
{
"key_as_string": "2020-05-16T16:00:00.000Z",
"key": 1589644800000,
"doc_count": 410
},
{
"key_as_string": "2020-05-16T16:30:00.000Z",
"key": 1589646600000,
"doc_count": 7465
},
{
"key_as_string": "2020-05-16T17:00:00.000Z",
"key": 1589648400000,
"doc_count": 12790
}
]
}
}
}

47
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_400.json vendored Normal file
View File

@@ -0,0 +1,47 @@
{
"error": {
"root_cause": [
{
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
},
{
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
],
"type": "search_phase_execution_exception",
"reason": "all shards failed",
"phase": "query",
"grouped": true,
"failed_shards": [
{
"shard": 0,
"index": "ks-logstash-log-2020.05.16",
"node": "Zr2OFlfeSJmK_W3Re4UBlg",
"reason": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
},
{
"shard": 0,
"index": "ks-logstash-log-2020.05.20",
"node": "pbGNYbV3QUuV5yJAgxbp3g",
"reason": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
}
],
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]",
"caused_by": {
"type": "illegal_argument_exception",
"reason": "Unable to parse interval [30m0s]"
}
}
},
"status": 400
}

21
pkg/simple/client/logging/elasticsearch/testdata/es7_count_logs_by_interval_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-20]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-20",
"index_uuid": "_na_",
"index": "ks-logstash-log-20"
}
],
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-20]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-20",
"index_uuid": "_na_",
"index": "ks-logstash-log-20"
},
"status": 404
}

17
pkg/simple/client/logging/elasticsearch/testdata/es7_detect_version_major_200.json vendored Normal file
View File

@@ -0,0 +1,17 @@
{
"name" : "elasticsearch-master-2",
"cluster_name" : "elasticsearch",
"cluster_uuid" : "_A-3shR0R0i-2M9CzOWP8g",
"version" : {
"number" : "7.7.0",
"build_flavor" : "default",
"build_type" : "docker",
"build_hash" : "81a1e9eda8e6183f5237786246f6dced26a10eaf",
"build_date" : "2020-05-12T02:01:37.602180Z",
"build_snapshot" : false,
"lucene_version" : "8.5.1",
"minimum_wire_compatibility_version" : "6.8.0",
"minimum_index_compatibility_version" : "6.0.0-beta1"
},
"tagline" : "You Know, for Search"
}

214
pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_200.json vendored Normal file
View File

@@ -0,0 +1,214 @@
{
"took": 1455,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 9726,
"relation": "eq"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "10.233.56.100 redis-ha-announce-2.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "jxt2MXIBlcWZ594bpIVN",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:07.648Z",
"log": "ls: cannot access '/calico-secrets': No such file or directory\n",
"time": "2020-05-16T16:00:07.64848716Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.164Z",
"log": "Wrote Calico CNI binaries to /host/opt/cni/bin\n",
"time": "2020-05-16T16:00:08.164003996Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "CNI plugin version: v3.7.3\n",
"time": "2020-05-16T16:00:08.350585959Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kht2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "/host/secondary-bin-dir is non-writeable, skipping\n",
"time": "2020-05-16T16:00:08.350625112Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "kxt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.350Z",
"log": "Using CNI config template from /host/etc/cni/net.d/calico.conflist.template.\n",
"time": "2020-05-16T16:00:08.350692011Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lBt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": "CNI config: {\n",
"time": "2020-05-16T16:00:08.454417144Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "lRt2MXIBlcWZ594bpIV0",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:08.454Z",
"log": " \"name\": \"cni0\",\n",
"time": "2020-05-16T16:00:08.454452649Z",
"kubernetes": {
"pod_name": "calico-node-gc7pp",
"namespace_name": "kube-system",
"host": "master0",
"container_name": "install-cni",
"docker_id": "c00abd0a7fe3d37c1328be560480239bb314fe78d31f7785e260ccdc0260cd7a",
"container_hash": "258a0cb3c25022e44ebda3606112c40865adb67b8fb7be3d119f960957301ad6"
}
}
}
]
},
"aggregations": {
"container_count": {
"value": 48
}
}
}

21
pkg/simple/client/logging/elasticsearch/testdata/es7_get_current_stats_404.json vendored Normal file
View File

@@ -0,0 +1,21 @@
{
"error": {
"root_cause": [
{
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-2020.05.2]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-2020.05.2",
"index_uuid": "_na_",
"index": "ks-logstash-log-2020.05.2"
}
],
"type": "index_not_found_exception",
"reason": "no such index [ks-logstash-log-2020.05.2]",
"resource.type": "index_or_alias",
"resource.id": "ks-logstash-log-2020.05.2",
"index_uuid": "_na_",
"index": "ks-logstash-log-2020.05.2"
},
"status": 404
}

76
pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200.json vendored Normal file
View File

@@ -0,0 +1,76 @@
{
"took": 772,
"timed_out": false,
"_shards": {
"total": 2,
"successful": 2,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10000,
"relation": "gte"
},
"max_score": 1.0,
"hits": [
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tRt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.608Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.608962452Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "tht2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.670Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"time": "2020-05-16T16:00:42.670430525Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "kubesphere-system",
"host": "master0",
"container_name": "config-init",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
},
{
"_index": "ks-logstash-log-2020.05.16",
"_type": "flb_type",
"_id": "txt2MXIBlcWZ594bqIUO",
"_score": 1.0,
"_source": {
"@timestamp": "2020-05-16T16:00:42.731Z",
"log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
"time": "2020-05-16T16:00:42.731865428Z",
"kubernetes": {
"pod_name": "redis-ha-haproxy-ffb8d889d-8x9kj",
"namespace_name": "istio-system",
"host": "node0",
"container_name": "mixer",
"docker_id": "a673327e5e3dfefca3e773273e69eca64baaa4499fdc04e6eb9d621ad8688ad0",
"container_hash": "cd4b3d4d27ae5931dc96b9632188590b7a6880469bcf07f478a3280dd0955336"
}
}
}
]
}
}

29
pkg/simple/client/logging/elasticsearch/testdata/es7_search_logs_200_result.json vendored Normal file
View File

@@ -0,0 +1,29 @@
{
"total": 10000,
"records": [
{
"time": "2020-05-16T16:00:42.608962452Z",
"log": "10.233.30.76 redis-ha-announce-0.kubesphere-system.svc.cluster.local\n",
"namespace": "kubesphere-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "config-init",
"host": "master0"
},
{
"time": "2020-05-16T16:00:42.670430525Z",
"log": "10.233.30.204 redis-ha-announce-1.kubesphere-system.svc.cluster.local\n",
"namespace": "kubesphere-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "config-init",
"host": "master0"
},
{
"time": "2020-05-16T16:00:42.731865428Z",
"log": "scvg14005: inuse: 16, idle: 42, sys: 58, released: 40, consumed: 17 (MB)\n",
"namespace": "istio-system",
"pod": "redis-ha-haproxy-ffb8d889d-8x9kj",
"container": "mixer",
"host": "node0"
}
]
}

11
pkg/simple/client/logging/elasticsearch/versions/v5/v5.go
View File

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.client.Search.WithScroll(time.Minute))
}
response, err := e.client.Search(opts...)
if err != nil {
return nil, err
}

11
pkg/simple/client/logging/elasticsearch/versions/v6/v6.go
View File

@@ -29,12 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{Client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.Client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.Client.Search.WithContext(context.Background()),
e.Client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.Client.Search.WithBody(bytes.NewBuffer(body)),
e.Client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.Client.Search.WithScroll(time.Minute))
}
response, err := e.Client.Search(opts...)
if err != nil {
return nil, err
}

12
pkg/simple/client/logging/elasticsearch/versions/v7/v7.go
View File

@@ -29,13 +29,17 @@ func New(address string, index string) *Elastic {
return &Elastic{client: client, index: index}
}
func (e *Elastic) Search(body []byte) ([]byte, error) {
response, err := e.client.Search(
func (e *Elastic) Search(body []byte, scroll bool) ([]byte, error) {
opts := []func(*esapi.SearchRequest){
e.client.Search.WithContext(context.Background()),
e.client.Search.WithIndex(fmt.Sprintf("%s*", e.index)),
e.client.Search.WithTrackTotalHits(true),
e.client.Search.WithBody(bytes.NewBuffer(body)),
e.client.Search.WithScroll(time.Minute))
}
if scroll {
opts = append(opts, e.client.Search.WithScroll(time.Minute))
}
response, err := e.client.Search(opts...)
if err != nil {
return nil, err
}

4
pkg/simple/client/logging/interface.go
View File

@@ -6,13 +6,9 @@ import (
)
type Interface interface {
// Current stats about log store, eg. total number of logs and containers
GetCurrentStats(sf SearchFilter) (Statistics, error)
CountLogsByInterval(sf SearchFilter, interval string) (Histogram, error)
SearchLogs(sf SearchFilter, from, size int64, order string) (Logs, error)
ExportLogs(sf SearchFilter, w io.Writer) error
}

4
tools/cmd/doc-gen/main.go
View File

@@ -37,7 +37,6 @@ import (
"kubesphere.io/kubesphere/pkg/informers"
devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2"
monitoringv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha3"
networkv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/network/v1alpha2"
openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1"
@@ -113,13 +112,12 @@ func generateSwaggerJson() []byte {
urlruntime.Must(devopsv1alpha2.AddToContainer(container, informerFactory.KubeSphereSharedInformerFactory(), &fake.Devops{}, nil, clientsets.KubeSphere(), fakes3.NewFakeS3()))
urlruntime.Must(iamv1alpha2.AddToContainer(container, im.NewOperator(clientsets.KubeSphere(), informerFactory), am.NewReadOnlyOperator(informerFactory), authoptions.NewAuthenticateOptions()))
urlruntime.Must(loggingv1alpha2.AddToContainer(container, clientsets, nil))
urlruntime.Must(monitoringv1alpha3.AddToContainer(container, clientsets.Kubernetes(), nil, informerFactory, nil))
urlruntime.Must(openpitrixv1.AddToContainer(container, informerFactory, nil))
urlruntime.Must(operationsv1alpha2.AddToContainer(container, clientsets.Kubernetes()))
urlruntime.Must(resourcesv1alpha2.AddToContainer(container, clientsets.Kubernetes(), informerFactory))
urlruntime.Must(resourcesv1alpha3.AddToContainer(container, informerFactory))
urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil))
urlruntime.Must(tenantv1alpha2.AddToContainer(container, informerFactory, nil, nil, nil, nil))
urlruntime.Must(terminalv1alpha2.AddToContainer(container, clientsets.Kubernetes(), nil))
urlruntime.Must(metricsv1alpha2.AddToContainer(container))
urlruntime.Must(networkv1alpha2.AddToContainer(container, ""))