zryfish
@@ -1,178 +1,153 @@
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/json-iterator/go"
|
||||
"io/ioutil"
|
||||
"context"
|
||||
"github.com/prometheus/client_golang/api"
|
||||
apiv1 "github.com/prometheus/client_golang/api/prometheus/v1"
|
||||
"github.com/prometheus/common/model"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
|
||||
// prometheus implements monitoring interface backed by Prometheus
|
||||
type prometheus struct {
|
||||
options *Options
|
||||
client *http.Client
|
||||
client apiv1.API
|
||||
}
|
||||
|
||||
func NewPrometheus(options *Options) monitoring.Interface {
|
||||
return &prometheus{
|
||||
options: options,
|
||||
client: &http.Client{Timeout: 10 * time.Second},
|
||||
func NewPrometheus(options *Options) (monitoring.Interface, error) {
|
||||
cfg := api.Config{
|
||||
Address: options.Endpoint,
|
||||
}
|
||||
|
||||
client, err := api.NewClient(cfg)
|
||||
return prometheus{client: apiv1.NewAPI(client)}, err
|
||||
}
|
||||
|
||||
// TODO(huanggze): reserve for custom monitoring
|
||||
func (p *prometheus) GetMetrics(stmts []string, time time.Time) ([]monitoring.Metric, error) {
|
||||
func (p prometheus) GetMetrics(stmts []string, time time.Time) []monitoring.Metric {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
// TODO(huanggze): reserve for custom monitoring
|
||||
func (p *prometheus) GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) ([]monitoring.Metric, error) {
|
||||
func (p prometheus) GetMetricsOverTime(stmts []string, start, end time.Time, step time.Duration) []monitoring.Metric {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (p *prometheus) GetNamedMetrics(ts time.Time, o monitoring.QueryOption) ([]monitoring.Metric, error) {
|
||||
metrics := make([]monitoring.Metric, 0)
|
||||
var mtx sync.Mutex // guard metrics
|
||||
func (p prometheus) GetNamedMetrics(metrics []string, ts time.Time, o monitoring.QueryOption) []monitoring.Metric {
|
||||
var res []monitoring.Metric
|
||||
var mtx sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
|
||||
opts := monitoring.NewQueryOptions()
|
||||
o.Apply(opts)
|
||||
|
||||
errCh := make(chan error)
|
||||
for _, metric := range opts.NamedMetrics {
|
||||
matched, _ := regexp.MatchString(opts.MetricFilter, metric)
|
||||
if matched {
|
||||
exp := makeExpression(metric, *opts)
|
||||
wg.Add(1)
|
||||
go func(metric, exp string) {
|
||||
res, err := p.query(exp, ts)
|
||||
if err != nil {
|
||||
select {
|
||||
case errCh <- err: // Record error once
|
||||
default:
|
||||
}
|
||||
} else {
|
||||
res.MetricName = metric // Add metric name
|
||||
mtx.Lock()
|
||||
metrics = append(metrics, res)
|
||||
mtx.Unlock()
|
||||
}
|
||||
wg.Done()
|
||||
}(metric, exp)
|
||||
}
|
||||
for _, metric := range metrics {
|
||||
wg.Add(1)
|
||||
go func(metric string) {
|
||||
parsedResp := monitoring.Metric{MetricName: metric}
|
||||
|
||||
value, err := p.client.Query(context.Background(), makeExpr(metric, *opts), ts)
|
||||
if err != nil {
|
||||
parsedResp.Error = err.(*apiv1.Error).Msg
|
||||
} else {
|
||||
parsedResp.MetricData = parseQueryResp(value)
|
||||
}
|
||||
|
||||
mtx.Lock()
|
||||
res = append(res, parsedResp)
|
||||
mtx.Unlock()
|
||||
|
||||
wg.Done()
|
||||
}(metric)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return nil, err
|
||||
default:
|
||||
return metrics, nil
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (p *prometheus) GetNamedMetricsOverTime(start, end time.Time, step time.Duration, o monitoring.QueryOption) ([]monitoring.Metric, error) {
|
||||
metrics := make([]monitoring.Metric, 0)
|
||||
var mtx sync.Mutex // guard metrics
|
||||
func (p prometheus) GetNamedMetricsOverTime(metrics []string, start, end time.Time, step time.Duration, o monitoring.QueryOption) []monitoring.Metric {
|
||||
var res []monitoring.Metric
|
||||
var mtx sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
|
||||
opts := monitoring.NewQueryOptions()
|
||||
o.Apply(opts)
|
||||
|
||||
errCh := make(chan error)
|
||||
for _, metric := range opts.NamedMetrics {
|
||||
matched, _ := regexp.MatchString(opts.MetricFilter, metric)
|
||||
if matched {
|
||||
exp := makeExpression(metric, *opts)
|
||||
wg.Add(1)
|
||||
go func(metric, exp string) {
|
||||
res, err := p.rangeQuery(exp, start, end, step)
|
||||
if err != nil {
|
||||
select {
|
||||
case errCh <- err: // Record error once
|
||||
default:
|
||||
}
|
||||
} else {
|
||||
res.MetricName = metric // Add metric name
|
||||
mtx.Lock()
|
||||
metrics = append(metrics, res)
|
||||
mtx.Unlock()
|
||||
}
|
||||
wg.Done()
|
||||
}(metric, exp)
|
||||
}
|
||||
timeRange := apiv1.Range{
|
||||
Start: start,
|
||||
End: end,
|
||||
Step: step,
|
||||
}
|
||||
|
||||
for _, metric := range metrics {
|
||||
wg.Add(1)
|
||||
go func(metric string) {
|
||||
parsedResp := monitoring.Metric{MetricName: metric}
|
||||
|
||||
value, err := p.client.QueryRange(context.Background(), makeExpr(metric, *opts), timeRange)
|
||||
if err != nil {
|
||||
parsedResp.Error = err.(*apiv1.Error).Msg
|
||||
} else {
|
||||
parsedResp.MetricData = parseQueryRangeResp(value)
|
||||
}
|
||||
|
||||
mtx.Lock()
|
||||
res = append(res, parsedResp)
|
||||
mtx.Unlock()
|
||||
|
||||
wg.Done()
|
||||
}(metric)
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
return nil, err
|
||||
default:
|
||||
return metrics, nil
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (p prometheus) query(exp string, ts time.Time) (monitoring.Metric, error) {
|
||||
params := &url.Values{}
|
||||
params.Set("time", ts.Format(time.RFC3339))
|
||||
params.Set("query", exp)
|
||||
func parseQueryRangeResp(value model.Value) monitoring.MetricData {
|
||||
res := monitoring.MetricData{MetricType: monitoring.MetricTypeMatrix}
|
||||
|
||||
u := fmt.Sprintf("%s/api/v1/query?%s", p.options.Endpoint, params.Encode())
|
||||
data, _ := value.(model.Matrix)
|
||||
|
||||
var m monitoring.Metric
|
||||
response, err := p.client.Get(u)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
for _, v := range data {
|
||||
mv := monitoring.MetricValue{
|
||||
Metadata: make(map[string]string),
|
||||
}
|
||||
|
||||
for k, v := range v.Metric {
|
||||
mv.Metadata[string(k)] = string(v)
|
||||
}
|
||||
|
||||
for _, k := range v.Values {
|
||||
mv.Series = append(mv.Series, monitoring.Point{float64(k.Timestamp) / 1000, float64(k.Value)})
|
||||
}
|
||||
|
||||
res.MetricValues = append(res.MetricValues, mv)
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
err = json.Unmarshal(body, m)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
return res
|
||||
}
|
||||
|
||||
func (p prometheus) rangeQuery(exp string, start, end time.Time, step time.Duration) (monitoring.Metric, error) {
|
||||
params := &url.Values{}
|
||||
params.Set("start", start.Format(time.RFC3339))
|
||||
params.Set("end", end.Format(time.RFC3339))
|
||||
params.Set("step", step.String())
|
||||
params.Set("query", exp)
|
||||
func parseQueryResp(value model.Value) monitoring.MetricData {
|
||||
res := monitoring.MetricData{MetricType: monitoring.MetricTypeVector}
|
||||
|
||||
u := fmt.Sprintf("%s/api/v1/query?%s", p.options.Endpoint, params.Encode())
|
||||
data, _ := value.(model.Vector)
|
||||
|
||||
var m monitoring.Metric
|
||||
response, err := p.client.Get(u)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
for _, v := range data {
|
||||
mv := monitoring.MetricValue{
|
||||
Metadata: make(map[string]string),
|
||||
}
|
||||
|
||||
for k, v := range v.Metric {
|
||||
mv.Metadata[string(k)] = string(v)
|
||||
}
|
||||
|
||||
mv.Sample = monitoring.Point{float64(v.Timestamp) / 1000, float64(v.Value)}
|
||||
|
||||
res.MetricValues = append(res.MetricValues, mv)
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
|
||||
err = json.Unmarshal(body, m)
|
||||
if err != nil {
|
||||
return monitoring.Metric{}, err
|
||||
}
|
||||
|
||||
return m, nil
|
||||
return res
|
||||
}
|
||||
|
||||
95
pkg/simple/client/monitoring/prometheus/prometheus_test.go
Normal file
95
pkg/simple/client/monitoring/prometheus/prometheus_test.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/json-iterator/go"
|
||||
"io/ioutil"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGetNamedMetrics(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fakeResp string
|
||||
expected string
|
||||
}{
|
||||
{"prom returns good values", "metrics-vector-type-prom.json", "metrics-vector-type-res.json"},
|
||||
{"prom returns error", "metrics-error-prom.json", "metrics-error-res.json"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expected, err := jsonFromFile(tt.expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := mockPrometheusService("/api/v1/query", tt.fakeResp)
|
||||
defer srv.Close()
|
||||
|
||||
client, _ := NewPrometheus(&Options{Endpoint: srv.URL})
|
||||
result := client.GetNamedMetrics([]string{"cluster_cpu_utilisation"}, time.Now(), monitoring.ClusterOption{})
|
||||
if diff := cmp.Diff(result, expected); diff != "" {
|
||||
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetNamedMetricsOverTime(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
fakeResp string
|
||||
expected string
|
||||
}{
|
||||
{"prom returns good values", "metrics-matrix-type-prom.json", "metrics-matrix-type-res.json"},
|
||||
{"prom returns error", "metrics-error-prom.json", "metrics-error-res.json"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expected, err := jsonFromFile(tt.expected)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srv := mockPrometheusService("/api/v1/query_range", tt.fakeResp)
|
||||
defer srv.Close()
|
||||
|
||||
client, _ := NewPrometheus(&Options{Endpoint: srv.URL})
|
||||
result := client.GetNamedMetricsOverTime([]string{"cluster_cpu_utilisation"}, time.Now().Add(-time.Minute*3), time.Now(), time.Minute, monitoring.ClusterOption{})
|
||||
if diff := cmp.Diff(result, expected); diff != "" {
|
||||
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func mockPrometheusService(pattern, fakeResp string) *httptest.Server {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
|
||||
b, _ := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", fakeResp))
|
||||
res.Write(b)
|
||||
})
|
||||
return httptest.NewServer(mux)
|
||||
}
|
||||
|
||||
func jsonFromFile(expectedFile string) ([]monitoring.Metric, error) {
|
||||
expectedJson := []monitoring.Metric{}
|
||||
|
||||
json, err := ioutil.ReadFile(fmt.Sprintf("./testdata/%s", expectedFile))
|
||||
if err != nil {
|
||||
return expectedJson, err
|
||||
}
|
||||
err = jsoniter.Unmarshal(json, &expectedJson)
|
||||
if err != nil {
|
||||
return expectedJson, err
|
||||
}
|
||||
|
||||
return expectedJson, nil
|
||||
}
|
||||
@@ -25,7 +25,6 @@ const (
|
||||
Deployment = "Deployment"
|
||||
)
|
||||
|
||||
//TODO(huanggze): move this part to a ConfigMap
|
||||
var promQLTemplates = map[string]string{
|
||||
//cluster
|
||||
"cluster_cpu_utilisation": ":node_cpu_utilisation:avg1m",
|
||||
@@ -256,31 +255,33 @@ var promQLTemplates = map[string]string{
|
||||
"prometheus_tsdb_head_samples_appended_rate": `prometheus:prometheus_tsdb_head_samples_appended:sum_rate`,
|
||||
}
|
||||
|
||||
func makeExpression(metric string, opt monitoring.QueryOptions) string {
|
||||
func makeExpr(metric string, opt monitoring.QueryOptions) string {
|
||||
tmpl := promQLTemplates[metric]
|
||||
switch opt.Level {
|
||||
case monitoring.LevelCluster:
|
||||
return tmpl
|
||||
case monitoring.LevelNode:
|
||||
makeNodeMetricExpression(tmpl, opt)
|
||||
return makeNodeMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelWorkspace:
|
||||
makeWorkspaceMetricExpression(tmpl, opt)
|
||||
return makeWorkspaceMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelNamespace:
|
||||
makeNamespaceMetricExpression(tmpl, opt)
|
||||
return makeNamespaceMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelWorkload:
|
||||
makeWorkloadMetricExpression(tmpl, opt)
|
||||
return makeWorkloadMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelPod:
|
||||
makePodMetricExpression(tmpl, opt)
|
||||
return makePodMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelContainer:
|
||||
makeContainerMetricExpression(tmpl, opt)
|
||||
return makeContainerMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelPVC:
|
||||
makePVCMetricExpression(tmpl, opt)
|
||||
return makePVCMetricExpr(tmpl, opt)
|
||||
case monitoring.LevelComponent:
|
||||
return tmpl
|
||||
default:
|
||||
return tmpl
|
||||
}
|
||||
return tmpl
|
||||
}
|
||||
|
||||
func makeNodeMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makeNodeMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var nodeSelector string
|
||||
if o.NodeName != "" {
|
||||
nodeSelector = fmt.Sprintf(`node="%s"`, o.NodeName)
|
||||
@@ -290,7 +291,7 @@ func makeNodeMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
return strings.Replace(tmpl, "$1", nodeSelector, -1)
|
||||
}
|
||||
|
||||
func makeWorkspaceMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makeWorkspaceMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var workspaceSelector string
|
||||
if o.WorkspaceName != "" {
|
||||
workspaceSelector = fmt.Sprintf(`label_kubesphere_io_workspace="%s"`, o.WorkspaceName)
|
||||
@@ -300,7 +301,7 @@ func makeWorkspaceMetricExpression(tmpl string, o monitoring.QueryOptions) strin
|
||||
return strings.Replace(tmpl, "$1", workspaceSelector, -1)
|
||||
}
|
||||
|
||||
func makeNamespaceMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makeNamespaceMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var namespaceSelector string
|
||||
|
||||
// For monitoring namespaces in the specific workspace
|
||||
@@ -321,7 +322,7 @@ func makeNamespaceMetricExpression(tmpl string, o monitoring.QueryOptions) strin
|
||||
return strings.Replace(tmpl, "$1", namespaceSelector, -1)
|
||||
}
|
||||
|
||||
func makeWorkloadMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makeWorkloadMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var kindSelector, workloadSelector string
|
||||
switch o.WorkloadKind {
|
||||
case "deployment":
|
||||
@@ -341,7 +342,7 @@ func makeWorkloadMetricExpression(tmpl string, o monitoring.QueryOptions) string
|
||||
return strings.NewReplacer("$1", workloadSelector, "$2", kindSelector).Replace(tmpl)
|
||||
}
|
||||
|
||||
func makePodMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makePodMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var podSelector, workloadSelector string
|
||||
|
||||
// For monitoriong pods of the specific workload
|
||||
@@ -371,7 +372,7 @@ func makePodMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
|
||||
// For monitoring pods on the specific node
|
||||
// GET /nodes/{node}/pods/{pod}
|
||||
if o.PodName != "" {
|
||||
if o.NodeName != "" {
|
||||
if o.PodName != "" {
|
||||
podSelector = fmt.Sprintf(`pod="%s", node="%s"`, o.PodName, o.NodeName)
|
||||
} else {
|
||||
@@ -381,7 +382,7 @@ func makePodMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
return strings.NewReplacer("$1", workloadSelector, "$2", podSelector).Replace(tmpl)
|
||||
}
|
||||
|
||||
func makeContainerMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makeContainerMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var containerSelector string
|
||||
if o.ContainerName != "" {
|
||||
containerSelector = fmt.Sprintf(`pod_name="%s", namespace="%s", container_name="%s"`, o.PodName, o.NamespaceName, o.ContainerName)
|
||||
@@ -391,7 +392,7 @@ func makeContainerMetricExpression(tmpl string, o monitoring.QueryOptions) strin
|
||||
return strings.Replace(tmpl, "$1", containerSelector, -1)
|
||||
}
|
||||
|
||||
func makePVCMetricExpression(tmpl string, o monitoring.QueryOptions) string {
|
||||
func makePVCMetricExpr(tmpl string, o monitoring.QueryOptions) string {
|
||||
var pvcSelector string
|
||||
|
||||
// For monitoring persistentvolumeclaims in the specific namespace
|
||||
45
pkg/simple/client/monitoring/prometheus/promql_test.go
Normal file
45
pkg/simple/client/monitoring/prometheus/promql_test.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package prometheus
|
||||
|
||||
import (
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/monitoring"
|
||||
"kubesphere.io/kubesphere/pkg/simple/client/monitoring/prometheus/testdata"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestMakeExpr(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
opt monitoring.QueryOptions
|
||||
}{
|
||||
{"cluster_cpu_utilisation", monitoring.QueryOptions{Level: monitoring.LevelCluster}},
|
||||
{"node_cpu_utilisation", monitoring.QueryOptions{Level: monitoring.LevelNode, NodeName: "i-2dazc1d6"}},
|
||||
{"node_cpu_total", monitoring.QueryOptions{Level: monitoring.LevelNode, ResourceFilter: "i-2dazc1d6|i-ezjb7gsk"}},
|
||||
{"workspace_cpu_usage", monitoring.QueryOptions{Level: monitoring.LevelWorkspace, WorkspaceName: "system-workspace"}},
|
||||
{"workspace_memory_usage", monitoring.QueryOptions{Level: monitoring.LevelWorkspace, ResourceFilter: "system-workspace|demo"}},
|
||||
{"namespace_cpu_usage", monitoring.QueryOptions{Level: monitoring.LevelNamespace, NamespaceName: "kube-system"}},
|
||||
{"namespace_memory_usage", monitoring.QueryOptions{Level: monitoring.LevelNamespace, ResourceFilter: "kube-system|default"}},
|
||||
{"namespace_memory_usage_wo_cache", monitoring.QueryOptions{Level: monitoring.LevelNamespace, WorkspaceName: "system-workspace", ResourceFilter: "kube-system|default"}},
|
||||
{"workload_cpu_usage", monitoring.QueryOptions{Level: monitoring.LevelWorkload, WorkloadKind: "deployment", NamespaceName: "default", ResourceFilter: "apiserver|coredns"}},
|
||||
{"workload_deployment_replica_available", monitoring.QueryOptions{Level: monitoring.LevelWorkload, WorkloadKind: ".*", NamespaceName: "default", ResourceFilter: "apiserver|coredns"}},
|
||||
{"pod_cpu_usage", monitoring.QueryOptions{Level: monitoring.LevelPod, NamespaceName: "default", WorkloadKind: "deployment", WorkloadName: "elasticsearch", ResourceFilter: "elasticsearch-0"}},
|
||||
{"pod_memory_usage", monitoring.QueryOptions{Level: monitoring.LevelPod, NamespaceName: "default", PodName: "elasticsearch-12345"}},
|
||||
{"pod_memory_usage_wo_cache", monitoring.QueryOptions{Level: monitoring.LevelPod, NodeName: "i-2dazc1d6", PodName: "elasticsearch-12345"}},
|
||||
{"container_cpu_usage", monitoring.QueryOptions{Level: monitoring.LevelContainer, NamespaceName: "default", PodName: "elasticsearch-12345", ContainerName: "syscall"}},
|
||||
{"container_memory_usage", monitoring.QueryOptions{Level: monitoring.LevelContainer, NamespaceName: "default", PodName: "elasticsearch-12345", ResourceFilter: "syscall"}},
|
||||
{"pvc_inodes_available", monitoring.QueryOptions{Level: monitoring.LevelPVC, NamespaceName: "default", PersistentVolumeClaimName: "db-123"}},
|
||||
{"pvc_inodes_used", monitoring.QueryOptions{Level: monitoring.LevelPVC, NamespaceName: "default", ResourceFilter: "db-123"}},
|
||||
{"pvc_inodes_total", monitoring.QueryOptions{Level: monitoring.LevelPVC, StorageClassName: "default", ResourceFilter: "db-123"}},
|
||||
{"etcd_server_list", monitoring.QueryOptions{Level: monitoring.LevelComponent}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
expected := testdata.PromQLs[tt.name]
|
||||
result := makeExpr(tt.name, tt.opt)
|
||||
if diff := cmp.Diff(result, expected); diff != "" {
|
||||
t.Fatalf("%T differ (-got, +want): %s", expected, diff)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
5
pkg/simple/client/monitoring/prometheus/testdata/metrics-error-prom.json
vendored
Normal file
5
pkg/simple/client/monitoring/prometheus/testdata/metrics-error-prom.json
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"status":"error",
|
||||
"errorType":"internal",
|
||||
"error":"inconsistent body for response code"
|
||||
}
|
||||
6
pkg/simple/client/monitoring/prometheus/testdata/metrics-error-res.json
vendored
Normal file
6
pkg/simple/client/monitoring/prometheus/testdata/metrics-error-res.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
||||
[
|
||||
{
|
||||
"metric_name": "cluster_cpu_utilisation",
|
||||
"error": "inconsistent body for response code"
|
||||
}
|
||||
]
|
||||
206
pkg/simple/client/monitoring/prometheus/testdata/metrics-matrix-type-prom.json
vendored
Normal file
206
pkg/simple/client/monitoring/prometheus/testdata/metrics-matrix-type-prom.json
vendored
Normal file
@@ -0,0 +1,206 @@
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"matrix",
|
||||
"result":[
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https",
|
||||
"instance":"192.168.2.2:9100",
|
||||
"job":"node-exporter",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"node-exporter-nxpld",
|
||||
"service":"node-exporter"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-main",
|
||||
"instance":"10.233.99.18:8443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-metrics",
|
||||
"instance":"192.168.2.2:10250",
|
||||
"job":"kubelet",
|
||||
"namespace":"kube-system",
|
||||
"node":"ks-allinone",
|
||||
"service":"kubelet"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-self",
|
||||
"instance":"10.233.99.18:9443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"mysql-exporter",
|
||||
"instance":"10.233.99.71:9104",
|
||||
"job":"mysql-sz197k-prometheus-mysql-exporter",
|
||||
"namespace":"exporter",
|
||||
"pod":"mysql-sz197k-prometheus-mysql-exporter-5d58bc7d94-dh6r9",
|
||||
"service":"mysql-sz197k-prometheus-mysql-exporter"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"web",
|
||||
"instance":"10.233.99.22:9090",
|
||||
"job":"prometheus-k8s-system",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"prometheus-k8s-system-0",
|
||||
"service":"prometheus-k8s-system"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
"1.123456"
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
"1.123456"
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
208
pkg/simple/client/monitoring/prometheus/testdata/metrics-matrix-type-res.json
vendored
Normal file
208
pkg/simple/client/monitoring/prometheus/testdata/metrics-matrix-type-res.json
vendored
Normal file
@@ -0,0 +1,208 @@
|
||||
[
|
||||
{
|
||||
"metric_name":"cluster_cpu_utilisation",
|
||||
"data":{
|
||||
"resultType":"matrix",
|
||||
"result":[
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https",
|
||||
"instance":"192.168.2.2:9100",
|
||||
"job":"node-exporter",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"node-exporter-nxpld",
|
||||
"service":"node-exporter"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-main",
|
||||
"instance":"10.233.99.18:8443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-metrics",
|
||||
"instance":"192.168.2.2:10250",
|
||||
"job":"kubelet",
|
||||
"namespace":"kube-system",
|
||||
"node":"ks-allinone",
|
||||
"service":"kubelet"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-self",
|
||||
"instance":"10.233.99.18:9443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"mysql-exporter",
|
||||
"instance":"10.233.99.71:9104",
|
||||
"job":"mysql-sz197k-prometheus-mysql-exporter",
|
||||
"namespace":"exporter",
|
||||
"pod":"mysql-sz197k-prometheus-mysql-exporter-5d58bc7d94-dh6r9",
|
||||
"service":"mysql-sz197k-prometheus-mysql-exporter"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"web",
|
||||
"instance":"10.233.99.22:9090",
|
||||
"job":"prometheus-k8s-system",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"prometheus-k8s-system-0",
|
||||
"service":"prometheus-k8s-system"
|
||||
},
|
||||
"values":[
|
||||
[
|
||||
1585743925,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744045,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744165,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744285,
|
||||
1.123456
|
||||
],
|
||||
[
|
||||
1585744405,
|
||||
1.123456
|
||||
]
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
68
pkg/simple/client/monitoring/prometheus/testdata/metrics-vector-type-prom.json
vendored
Normal file
68
pkg/simple/client/monitoring/prometheus/testdata/metrics-vector-type-prom.json
vendored
Normal file
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"status":"success",
|
||||
"data":{
|
||||
"resultType":"vector",
|
||||
"result":[
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https",
|
||||
"instance":"192.168.2.2:9100",
|
||||
"job":"node-exporter",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"node-exporter-nxpld",
|
||||
"service":"node-exporter"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
"1.123456"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-main",
|
||||
"instance":"10.233.99.18:8443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
"1.123456"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-metrics",
|
||||
"instance":"192.168.2.2:10250",
|
||||
"job":"kubelet",
|
||||
"namespace":"kube-system",
|
||||
"node":"ks-allinone",
|
||||
"service":"kubelet"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
"1.123456"
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-self",
|
||||
"instance":"10.233.99.18:9443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
"1.123456"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
70
pkg/simple/client/monitoring/prometheus/testdata/metrics-vector-type-res.json
vendored
Normal file
70
pkg/simple/client/monitoring/prometheus/testdata/metrics-vector-type-res.json
vendored
Normal file
@@ -0,0 +1,70 @@
|
||||
[
|
||||
{
|
||||
"metric_name":"cluster_cpu_utilisation",
|
||||
"data":{
|
||||
"resultType":"vector",
|
||||
"result":[
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https",
|
||||
"instance":"192.168.2.2:9100",
|
||||
"job":"node-exporter",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"node-exporter-nxpld",
|
||||
"service":"node-exporter"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
1.123456
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-main",
|
||||
"instance":"10.233.99.18:8443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
1.123456
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-metrics",
|
||||
"instance":"192.168.2.2:10250",
|
||||
"job":"kubelet",
|
||||
"namespace":"kube-system",
|
||||
"node":"ks-allinone",
|
||||
"service":"kubelet"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
1.123456
|
||||
]
|
||||
},
|
||||
{
|
||||
"metric":{
|
||||
"__name__":"up",
|
||||
"endpoint":"https-self",
|
||||
"instance":"10.233.99.18:9443",
|
||||
"job":"kube-state-metrics",
|
||||
"namespace":"kubesphere-monitoring-system",
|
||||
"pod":"kube-state-metrics-566cdbcb48-98brh",
|
||||
"service":"kube-state-metrics"
|
||||
},
|
||||
"value":[
|
||||
1585743854.077,
|
||||
1.123456
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
23
pkg/simple/client/monitoring/prometheus/testdata/promqls.go
vendored
Normal file
23
pkg/simple/client/monitoring/prometheus/testdata/promqls.go
vendored
Normal file
@@ -0,0 +1,23 @@
|
||||
package testdata
|
||||
|
||||
var PromQLs = map[string]string{
|
||||
"cluster_cpu_utilisation": `:node_cpu_utilisation:avg1m`,
|
||||
"node_cpu_utilisation": `node:node_cpu_utilisation:avg1m{node="i-2dazc1d6"}`,
|
||||
"node_cpu_total": `node:node_num_cpu:sum{node=~"i-2dazc1d6|i-ezjb7gsk"}`,
|
||||
"workspace_cpu_usage": `round(sum by (label_kubesphere_io_workspace) (namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", label_kubesphere_io_workspace="system-workspace"}), 0.001)`,
|
||||
"workspace_memory_usage": `sum by (label_kubesphere_io_workspace) (namespace:container_memory_usage_bytes:sum{namespace!="", label_kubesphere_io_workspace=~"system-workspace|demo", label_kubesphere_io_workspace!=""})`,
|
||||
"namespace_cpu_usage": `round(namespace:container_cpu_usage_seconds_total:sum_rate{namespace!="", namespace="kube-system"}, 0.001)`,
|
||||
"namespace_memory_usage": `namespace:container_memory_usage_bytes:sum{namespace!="", namespace=~"kube-system|default"}`,
|
||||
"namespace_memory_usage_wo_cache": `namespace:container_memory_usage_bytes_wo_cache:sum{namespace!="", label_kubesphere_io_workspace="system-workspace", namespace=~"kube-system|default"}`,
|
||||
"workload_cpu_usage": `round(namespace:workload_cpu_usage:sum{namespace="default", workload=~"Deployment:apiserver|coredns"}, 0.001)`,
|
||||
"workload_deployment_replica_available": `label_join(sum (label_join(label_replace(kube_deployment_status_replicas_available{namespace="default"}, "owner_kind", "Deployment", "", ""), "workload", "", "deployment")) by (namespace, owner_kind, workload), "workload", ":", "owner_kind", "workload")`,
|
||||
"pod_cpu_usage": `round(label_join(sum by (namespace, pod_name) (irate(container_cpu_usage_seconds_total{job="kubelet", pod_name!="", image!=""}[5m])), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{owner_kind="ReplicaSet", owner_name=~"^deployment-[^-]{1,10}$"} * on (namespace, pod) group_left(node) kube_pod_info{pod=~"elasticsearch-0", namespace="default"}, 0.001)`,
|
||||
"pod_memory_usage": `label_join(sum by (namespace, pod_name) (container_memory_usage_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{} * on (namespace, pod) group_left(node) kube_pod_info{pod="elasticsearch-12345", namespace="default"}`,
|
||||
"pod_memory_usage_wo_cache": `label_join(sum by (namespace, pod_name) (container_memory_working_set_bytes{job="kubelet", pod_name!="", image!=""}), "pod", "", "pod_name") * on (namespace, pod) group_left(owner_kind, owner_name) kube_pod_owner{} * on (namespace, pod) group_left(node) kube_pod_info{pod="elasticsearch-12345", node="i-2dazc1d6"}`,
|
||||
"container_cpu_usage": `round(sum by (namespace, pod_name, container_name) (irate(container_cpu_usage_seconds_total{job="kubelet", container_name!="POD", container_name!="", image!="", pod_name="elasticsearch-12345", namespace="default", container_name="syscall"}[5m])), 0.001)`,
|
||||
"container_memory_usage": `sum by (namespace, pod_name, container_name) (container_memory_usage_bytes{job="kubelet", container_name!="POD", container_name!="", image!="", pod_name="elasticsearch-12345", namespace="default", container_name=~"syscall"})`,
|
||||
"pvc_inodes_available": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_free) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{namespace="default", persistentvolumeclaim="db-123"}`,
|
||||
"pvc_inodes_used": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes_used) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{namespace="default", persistentvolumeclaim=~"db-123"}`,
|
||||
"pvc_inodes_total": `max by (namespace, persistentvolumeclaim) (kubelet_volume_stats_inodes) * on (namespace, persistentvolumeclaim) group_left (storageclass) kube_persistentvolumeclaim_info{storageclass="default", persistentvolumeclaim=~"db-123"}`,
|
||||
"etcd_server_list": `label_replace(up{job="etcd"}, "node_ip", "$1", "instance", "(.*):.*")`,
|
||||
}
|
||||
Reference in New Issue
Block a user