diff --git a/pkg/apis/logging/v1alpha2/register.go b/pkg/apis/logging/v1alpha2/register.go index 5205dd9a2..b98dce924 100644 --- a/pkg/apis/logging/v1alpha2/register.go +++ b/pkg/apis/logging/v1alpha2/register.go @@ -24,7 +24,6 @@ import ( "kubesphere.io/kubesphere/pkg/apiserver/logging" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/filter" "kubesphere.io/kubesphere/pkg/models/log" esclient "kubesphere.io/kubesphere/pkg/simple/client/elasticsearch" fluentbitclient "kubesphere.io/kubesphere/pkg/simple/client/fluentbit" @@ -47,7 +46,6 @@ func addWebService(c *restful.Container) error { ws := runtime.NewWebService(GroupVersion) ws.Route(ws.GET("/cluster").To(logging.LoggingQueryCluster). - Filter(filter.Logging). Doc("Query logs against the cluster."). Param(ws.QueryParameter("operation", "Query type. This can be one of three types: query (for querying logs), statistics (for retrieving statistical data), and histogram (for displaying log count by time interval). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)). @@ -74,7 +72,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/workspaces/{workspace}").To(logging.LoggingQueryWorkspace). - Filter(filter.Logging). Doc("Query logs against the specific workspace."). Param(ws.PathParameter("workspace", "The name of the workspace.").DataType("string").Required(true)). Param(ws.QueryParameter("operation", "Query type. This can be one of three types: query (for querying logs), statistics (for retrieving statistical data), and histogram (for displaying log count by time interval). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). @@ -100,7 +97,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/namespaces/{namespace}").To(logging.LoggingQueryNamespace). - Filter(filter.Logging). Doc("Query logs against the specific namespace."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.QueryParameter("operation", "Query type. This can be one of three types: query (for querying logs), statistics (for retrieving statistical data), and histogram (for displaying log count by time interval). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). @@ -124,7 +120,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/namespaces/{namespace}/workloads/{workload}").To(logging.LoggingQueryWorkload). - Filter(filter.Logging). Doc("Query logs against the specific workload."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("workload", "The name of the workload.").DataType("string").Required(true)). @@ -147,7 +142,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}").To(logging.LoggingQueryPod). - Filter(filter.Logging). Doc("Query logs against the specific pod."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -168,7 +162,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/namespaces/{namespace}/pods/{pod}/containers/{container}").To(logging.LoggingQueryContainer). - Filter(filter.Logging). Doc("Query logs against the specific container."). Param(ws.PathParameter("namespace", "The name of the namespace.").DataType("string").Required(true)). Param(ws.PathParameter("pod", "Pod name.").DataType("string").Required(true)). @@ -188,7 +181,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.GET("/fluentbit/outputs").To(logging.LoggingQueryFluentbitOutputs). - Filter(filter.Logging). Doc("List all Fluent bit output plugins."). Metadata(restfulspec.KeyOpenAPITags, []string{constants.FluentBitSetting}). Writes(log.FluentbitOutputsResult{}). @@ -197,7 +189,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.POST("/fluentbit/outputs").To(logging.LoggingInsertFluentbitOutput). - Filter(filter.Logging). Doc("Add a new Fluent bit output plugin."). Metadata(restfulspec.KeyOpenAPITags, []string{constants.FluentBitSetting}). Reads(fluentbitclient.OutputPlugin{}). @@ -207,7 +198,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.PUT("/fluentbit/outputs/{output}").To(logging.LoggingUpdateFluentbitOutput). - Filter(filter.Logging). Doc("Update the specific Fluent bit output plugin."). Param(ws.PathParameter("output", "ID of the output.").DataType("string").Required(true)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.FluentBitSetting}). @@ -218,7 +208,6 @@ func addWebService(c *restful.Container) error { Produces(restful.MIME_JSON) ws.Route(ws.DELETE("/fluentbit/outputs/{output}").To(logging.LoggingDeleteFluentbitOutput). - Filter(filter.Logging). Doc("Delete the specific Fluent bit output plugin."). Param(ws.PathParameter("output", "ID of the output.").DataType("string").Required(true)). Metadata(restfulspec.KeyOpenAPITags, []string{constants.FluentBitSetting}). diff --git a/pkg/apiserver/logging/logging.go b/pkg/apiserver/logging/logging.go index 9df5a91d3..fd99e5ad6 100644 --- a/pkg/apiserver/logging/logging.go +++ b/pkg/apiserver/logging/logging.go @@ -91,27 +91,6 @@ func LoggingQueryContainer(request *restful.Request, response *restful.Response) response.WriteAsJson(res) } -func LoggingQueryFluentbitFilters(request *restful.Request, response *restful.Response) { - res := log.FluentbitFiltersQuery() - response.WriteAsJson(res) -} - -func LoggingUpdateFluentbitFilters(request *restful.Request, response *restful.Response) { - - var res *log.FluentbitFiltersResult - - filters := new([]log.FluentbitFilter) - - err := request.ReadEntity(&filters) - if err != nil { - res = &log.FluentbitFiltersResult{Status: http.StatusBadRequest} - } else { - res = log.FluentbitFiltersUpdate(filters) - } - - response.WriteAsJson(res) -} - func LoggingQueryFluentbitOutputs(request *restful.Request, response *restful.Response) { res := log.FluentbitOutputsQuery() if res.Status != http.StatusOK { diff --git a/pkg/filter/logging.go b/pkg/filter/logging.go index cc560e0ee..0fdeb3201 100644 --- a/pkg/filter/logging.go +++ b/pkg/filter/logging.go @@ -19,17 +19,17 @@ package filter import ( + "k8s.io/klog" "strings" "time" "github.com/emicklei/go-restful" - "github.com/golang/glog" ) func Logging(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) { start := time.Now() chain.ProcessFilter(req, resp) - glog.Infof("%s - \"%s %s %s\" %d %d %dms", + klog.V(4).Infof("%s - \"%s %s %s\" %d %d %dms", strings.Split(req.Request.RemoteAddr, ":")[0], req.Request.Method, req.Request.RequestURI, diff --git a/pkg/models/log/logcrd.go b/pkg/models/log/logcrd.go index 7862313ae..a7b1098c6 100644 --- a/pkg/models/log/logcrd.go +++ b/pkg/models/log/logcrd.go @@ -53,180 +53,6 @@ func createCRDClientSet() (*rest.RESTClient, *runtime.Scheme, error) { return fb.NewFluentbitCRDClient(config) } -func getParameterValue(parameters []fb.Parameter, name string) string { - var value string - - value = "" - for _, parameter := range parameters { - if parameter.Name == name { - value = parameter.Value - } - } - - return value -} - -func getFilters(result *FluentbitFiltersResult, Filters []fb.Plugin) { - for _, filter := range Filters { - if strings.Compare(filter.Name, "fluentbit-filter-input-regex") == 0 { - parameters := strings.Split(getParameterValue(filter.Parameters, "Regex"), " ") - field := strings.TrimSuffix(strings.TrimPrefix(parameters[0], "kubernetes_"), "_name") - expression := parameters[1] - result.Filters = append(result.Filters, FluentbitFilter{"Regex", field, expression}) - } - if strings.Compare(filter.Name, "fluentbit-filter-input-exclude") == 0 { - parameters := strings.Split(getParameterValue(filter.Parameters, "Exclude"), " ") - field := strings.TrimSuffix(strings.TrimPrefix(parameters[0], "kubernetes_"), "_name") - expression := parameters[1] - result.Filters = append(result.Filters, FluentbitFilter{"Exclude", field, expression}) - } - } -} - -func FluentbitFiltersQuery() *FluentbitFiltersResult { - var result FluentbitFiltersResult - - crdcs, scheme, err := createCRDClientSet() - if err != nil { - result.Status = http.StatusInternalServerError - return &result - } - - // Create a CRD client interface - crdclient := fb.CrdClient(crdcs, scheme, LoggingNamespace) - - item, err := crdclient.Get("fluent-bit") - if err != nil { - result.Status = http.StatusInternalServerError - return &result - } - - getFilters(&result, item.Spec.Filter) - - result.Status = http.StatusOK - - return &result -} - -func FluentbitFiltersUpdate(filters *[]FluentbitFilter) *FluentbitFiltersResult { - var result FluentbitFiltersResult - - //Generate filter plugin config - var filter []fb.Plugin - - var para_kubernetes []fb.Parameter - para_kubernetes = append(para_kubernetes, fb.Parameter{Name: "Name", Value: "kubernetes"}) - para_kubernetes = append(para_kubernetes, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_kubernetes = append(para_kubernetes, fb.Parameter{Name: "Kube_URL", Value: "https://kubernetes.default.svc:443"}) - para_kubernetes = append(para_kubernetes, fb.Parameter{Name: "Kube_CA_File", Value: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"}) - para_kubernetes = append(para_kubernetes, fb.Parameter{Name: "Kube_Token_File", Value: "/var/run/secrets/kubernetes.io/serviceaccount/token"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-kubernetes", Parameters: para_kubernetes}) - - var para_lift []fb.Parameter - para_lift = append(para_lift, fb.Parameter{Name: "Name", Value: "nest"}) - para_lift = append(para_lift, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_lift = append(para_lift, fb.Parameter{Name: "Operation", Value: "lift"}) - para_lift = append(para_lift, fb.Parameter{Name: "Nested_under", Value: "kubernetes"}) - para_lift = append(para_lift, fb.Parameter{Name: "Prefix_with", Value: "kubernetes_"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-lift", Parameters: para_lift}) - - var para_remove_stream []fb.Parameter - para_remove_stream = append(para_remove_stream, fb.Parameter{Name: "Name", Value: "modify"}) - para_remove_stream = append(para_remove_stream, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_remove_stream = append(para_remove_stream, fb.Parameter{Name: "Remove", Value: "stream"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-remove-stream", Parameters: para_remove_stream}) - - var para_remove_labels []fb.Parameter - para_remove_labels = append(para_remove_labels, fb.Parameter{Name: "Name", Value: "modify"}) - para_remove_labels = append(para_remove_labels, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_remove_labels = append(para_remove_labels, fb.Parameter{Name: "Remove", Value: "kubernetes_labels"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-remove-labels", Parameters: para_remove_labels}) - - var para_remove_annotations []fb.Parameter - para_remove_annotations = append(para_remove_annotations, fb.Parameter{Name: "Name", Value: "modify"}) - para_remove_annotations = append(para_remove_annotations, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_remove_annotations = append(para_remove_annotations, fb.Parameter{Name: "Remove", Value: "kubernetes_annotations"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-remove-annotations", Parameters: para_remove_annotations}) - - var para_remove_pod_id []fb.Parameter - para_remove_pod_id = append(para_remove_pod_id, fb.Parameter{Name: "Name", Value: "modify"}) - para_remove_pod_id = append(para_remove_pod_id, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_remove_pod_id = append(para_remove_pod_id, fb.Parameter{Name: "Remove", Value: "kubernetes_pod_id"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-remove-podid", Parameters: para_remove_pod_id}) - - var para_remove_docker_id []fb.Parameter - para_remove_docker_id = append(para_remove_docker_id, fb.Parameter{Name: "Name", Value: "modify"}) - para_remove_docker_id = append(para_remove_docker_id, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_remove_docker_id = append(para_remove_docker_id, fb.Parameter{Name: "Remove", Value: "kubernetes_docker_id"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-remove-dockerid", Parameters: para_remove_docker_id}) - - if len(*filters) > 0 { - for _, item := range *filters { - if strings.Compare(item.Type, "Regex") == 0 { - field := "kubernetes_" + strings.TrimSpace(item.Field) + "_name" - expression := strings.TrimSpace(item.Expression) - - var para_regex []fb.Parameter - para_regex = append(para_regex, fb.Parameter{Name: "Name", Value: "grep"}) - para_regex = append(para_regex, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_regex = append(para_regex, fb.Parameter{Name: "Regex", Value: field + " " + expression}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-regex", Parameters: para_regex}) - } - - if strings.Compare(item.Type, "Exclude") == 0 { - field := "kubernetes_" + strings.TrimSpace(item.Field) + "_name" - expression := strings.TrimSpace(item.Expression) - - var para_exclude []fb.Parameter - para_exclude = append(para_exclude, fb.Parameter{Name: "Name", Value: "grep"}) - para_exclude = append(para_exclude, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_exclude = append(para_exclude, fb.Parameter{Name: "Exclude", Value: field + " " + expression}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-exclude", Parameters: para_exclude}) - } - } - } - - var para_nest []fb.Parameter - para_nest = append(para_nest, fb.Parameter{Name: "Name", Value: "nest"}) - para_nest = append(para_nest, fb.Parameter{Name: "Match", Value: "kube.*"}) - para_nest = append(para_nest, fb.Parameter{Name: "Operation", Value: "nest"}) - para_nest = append(para_nest, fb.Parameter{Name: "Wildcard", Value: "kubernetes_*"}) - para_nest = append(para_nest, fb.Parameter{Name: "Nested_under", Value: "kubernetes"}) - para_nest = append(para_nest, fb.Parameter{Name: "Remove_prefix", Value: "kubernetes_"}) - filter = append(filter, fb.Plugin{Type: "fluentbit_filter", Name: "fluentbit-filter-input-nest", Parameters: para_nest}) - - crdcs, scheme, err := createCRDClientSet() - if err != nil { - result.Status = http.StatusInternalServerError - return &result - } - - // Create a CRD client interface - crdclient := fb.CrdClient(crdcs, scheme, LoggingNamespace) - - var item *fb.FluentBit - var err_read error - - item, err_read = crdclient.Get("fluent-bit") - if err_read != nil { - result.Status = http.StatusInternalServerError - return &result - } - - item.Spec.Filter = filter - - itemnew, err := crdclient.Update("fluent-bit", item) - if err != nil { - result.Status = http.StatusInternalServerError - return &result - } - - getFilters(&result, itemnew.Spec.Filter) - result.Status = http.StatusOK - - return &result -} - func FluentbitOutputsQuery() *FluentbitOutputsResult { var result FluentbitOutputsResult diff --git a/pkg/models/log/types.go b/pkg/models/log/types.go index e7d75c0c1..b4b402fa6 100644 --- a/pkg/models/log/types.go +++ b/pkg/models/log/types.go @@ -22,31 +22,6 @@ import ( fb "kubesphere.io/kubesphere/pkg/simple/client/fluentbit" ) -type FluentbitCRDResult struct { - Status int `json:"status"` - CRD fb.FluentBitSpec `json:"CRD,omitempty"` -} - -type FluentbitCRDDeleteResult struct { - Status int `json:"status"` -} - -type FluentbitSettingsResult struct { - Status int `json:"status"` - Enable string `json:"Enable,omitempty"` -} - -type FluentbitFilter struct { - Type string `json:"type"` - Field string `json:"field"` - Expression string `json:"expression"` -} - -type FluentbitFiltersResult struct { - Status int `json:"status"` - Filters []FluentbitFilter `json:"filters,omitempty"` -} - type FluentbitOutputsResult struct { Status int `json:"status" description:"response status"` Error string `json:"error,omitempty" description:"debug information"`