Change metering exported format to csv.

Signed-off-by: Rao Yunkun <yunkunrao@yunify.com>

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.1
 	github.com/json-iterator/go v1.1.10
+	github.com/jszwec/csvutil v1.5.0
 	github.com/kelseyhightower/envconfig v1.4.0 // indirect
 	github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0
 	github.com/kubesphere/sonargo v0.0.2
@@ -98,12 +99,12 @@ require (
 	k8s.io/kubectl v0.18.6
 	k8s.io/metrics v0.18.6
 	k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
+	kubesphere.io/client-go v0.0.0
 	openpitrix.io/openpitrix v0.4.9-0.20200611125425-ae07f141e797
 	sigs.k8s.io/application v0.8.4-0.20201016185654-c8e2959e57a0
 	sigs.k8s.io/controller-runtime v0.6.4
 	sigs.k8s.io/controller-tools v0.4.0
 	sigs.k8s.io/kubefed v0.4.0
-	kubesphere.io/client-go v0.0.0
 )
 
 replace (
@@ -735,6 +736,8 @@ replace (
 	k8s.io/kubectl => k8s.io/kubectl v0.18.6
 	k8s.io/metrics => k8s.io/metrics v0.18.6
 	k8s.io/utils => k8s.io/utils v0.0.0-20200603063816-c1c6865ac451
+
+	kubesphere.io/client-go => ./staging/src/kubesphere.io/client-go
 	kubesphere.io/im => kubesphere.io/im v0.1.0
 	openpitrix.io/iam => openpitrix.io/iam v0.1.0
 	openpitrix.io/libqueue => openpitrix.io/libqueue v0.4.1
@@ -758,6 +761,4 @@ replace (
 	sigs.k8s.io/yaml => sigs.k8s.io/yaml v1.2.0
 	sourcegraph.com/sourcegraph/appdash => sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0
 	vbom.ml/util => vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc
-
-	kubesphere.io/client-go => ./staging/src/kubesphere.io/client-go
 )

go.sum

@@ -432,6 +432,8 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o=
+github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM=
+github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=

pkg/kapis/monitoring/v1alpha3/helper.go

@@ -19,8 +19,8 @@ package v1alpha3
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
+	"github.com/jszwec/csvutil"
 	"io"
 	"strconv"
 	"strings"
@@ -388,7 +388,7 @@ func ExportMetrics(resp *restful.Response, metrics model.Metrics) {
 		}
 	}
 
-	resBytes, err := json.MarshalIndent(metrics, "", " ")
+	resBytes, err := csvutil.Marshal(metrics.Results)
 	if err != nil {
 		api.HandleBadRequest(resp, nil, err)
 		return

pkg/kapis/tenant/v1alpha2/metering.go

@@ -64,16 +64,17 @@ func (h *tenantHandler) QueryMeteringsHierarchy(req *restful.Request, resp *rest
 func (h *tenantHandler) HandlePriceInfoQuery(req *restful.Request, resp *restful.Response) {
 
 	var priceInfoResponse metering.PriceInfo
+	priceInfoResponse.Init()
 
 	meterConfig, err := monitoring.LoadYaml()
 	if err != nil {
-		klog.Error(err)
+		klog.Warning(err)
 		resp.WriteAsJson(priceInfoResponse)
 		return
 	}
 
 	priceInfo := meterConfig.GetPriceInfo()
-	priceInfoResponse.Currency = "CNY"
+	priceInfoResponse.Currency = priceInfo.CurrencyUnit
 	priceInfoResponse.CpuPerCorePerHour = priceInfo.CpuPerCorePerHour
 	priceInfoResponse.MemPerGigabytesPerHour = priceInfo.MemPerGigabytesPerHour
 	priceInfoResponse.IngressNetworkTrafficPerGiagabytesPerHour = priceInfo.IngressNetworkTrafficPerGiagabytesPerHour

pkg/models/metering/type.go

@@ -9,6 +9,16 @@ type PriceInfo struct {
 	PvcPerGigabytesPerHour                    float64 `json:"pvc_per_gigabytes_per_hour,omitempty" description:"pvc price"`
 }
 
+// currently init method fill illegal value to hint that metering config file was not mounted yet
+func (p *PriceInfo) Init() {
+	p.Currency = ""
+	p.CpuPerCorePerHour = -1
+	p.MemPerGigabytesPerHour = -1
+	p.IngressNetworkTrafficPerGiagabytesPerHour = -1
+	p.EgressNetworkTrafficPerGiagabytesPerHour = -1
+	p.PvcPerGigabytesPerHour = -1
+}
+
 type PodStatistic struct {
 	CPUUsage            float64 `json:"cpu_usage" description:"cpu_usage"`
 	MemoryUsageWoCache  float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"`

pkg/models/monitoring/utils.go

@@ -19,6 +19,14 @@ const (
 	meteringConfig = "/etc/kubesphere/metering/ks-metering.yaml"
 )
 
+var meterResourceUnitMap = map[int]string{
+	METER_RESOURCE_TYPE_CPU:         "cores",
+	METER_RESOURCE_TYPE_MEM:         "bytes",
+	METER_RESOURCE_TYPE_NET_INGRESS: "bytes",
+	METER_RESOURCE_TYPE_NET_EGRESS:  "bytes",
+	METER_RESOURCE_TYPE_PVC:         "bytes",
+}
+
 var MeterResourceMap = map[string]int{
 	"meter_cluster_cpu_usage":                 METER_RESOURCE_TYPE_CPU,
 	"meter_cluster_memory_usage":              METER_RESOURCE_TYPE_MEM,
@@ -67,6 +75,7 @@ type PriceInfo struct {
 	IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingressNetworkTrafficPerGiagabytesPerHour" yaml:"ingressNetworkTrafficPerGiagabytesPerHour"`
 	EgressNetworkTrafficPerGigabytesPerHour   float64 `json:"egressNetworkTrafficPerGigabytesPerHour" yaml:"egressNetworkTrafficPerGigabytesPerHour"`
 	PvcPerGigabytesPerHour                    float64 `json:"pvcPerGigabytesPerHour" yaml:"pvcPerGigabytesPerHour"`
+	CurrencyUnit                              string  `json:"currencyUnit" yaml:"currencyUnit"`
 }
 
 type Billing struct {
@@ -143,6 +152,25 @@ func getAvgPointValue(points []monitoring.Point) float64 {
 	return getSumPointValue(points) / float64(len(points))
 }
 
+func getCurrencyUnit() string {
+	meterConfig, err := LoadYaml()
+	if err != nil {
+		klog.Error(err)
+		return ""
+	}
+
+	return meterConfig.GetPriceInfo().CurrencyUnit
+}
+
+func getResourceUnit(meterName string) string {
+	if resourceType, ok := MeterResourceMap[meterName]; !ok {
+		klog.Errorf("invlaid meter %v", meterName)
+		return ""
+	} else {
+		return meterResourceUnitMap[resourceType]
+	}
+}
+
 func getFeeWithMeterName(meterName string, sum float64) float64 {
 
 	meterConfig, err := LoadYaml()
@@ -216,6 +244,8 @@ func updateMetricStatData(metric monitoring.Metric, scalingMap map[string]float6
 			metricData.MetricValues[index].SumValue = sum
 			metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum)
 		}
+		metricData.MetricValues[index].CurrencyUnit = getCurrencyUnit()
+		metricData.MetricValues[index].ResourceUnit = getResourceUnit(metricName)
 
 	}
 	return metricData

pkg/simple/client/monitoring/types.go

@@ -20,7 +20,9 @@ import (
 	"errors"
 	"fmt"
 	"github.com/json-iterator/go"
+	"github.com/jszwec/csvutil"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -36,14 +38,46 @@ type Metadata struct {
 }
 
 type Metric struct {
-	MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum"`
+	MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum" csv:"metric_name"`
 	MetricData `json:"data,omitempty" description:"actual metric result"`
-	Error      string `json:"error,omitempty"`
+	Error      string `json:"error,omitempty" csv:"-"`
+}
+
+type MetricValues []MetricValue
+
+func (m MetricValues) MarshalCSV() ([]byte, error) {
+
+	var ret []string
+	for _, v := range m {
+		tmp, err := v.MarshalCSV()
+		if err != nil {
+			return nil, err
+		}
+
+		ret = append(ret, string(tmp))
+	}
+
+	return []byte(strings.Join(ret, "||")), nil
 }
 
 type MetricData struct {
-	MetricType   string        `json:"resultType,omitempty" description:"result type, one of matrix, vector"`
-	MetricValues []MetricValue `json:"result,omitempty" description:"metric data including labels, time series and values"`
+	MetricType   string `json:"resultType,omitempty" description:"result type, one of matrix, vector" csv:"metric_type"`
+	MetricValues `json:"result,omitempty" description:"metric data including labels, time series and values" csv:"metric_values"`
+}
+
+func (m MetricData) MarshalCSV() ([]byte, error) {
+	var ret []byte
+
+	for _, v := range m.MetricValues {
+		tmp, err := csvutil.Marshal(&v)
+		if err != nil {
+			return nil, err
+		}
+
+		ret = append(ret, tmp...)
+	}
+
+	return ret, nil
 }
 
 // The first element is the timestamp, the second is the metric value.
@@ -60,11 +94,48 @@ type MetricValue struct {
 	ExportSample   *ExportPoint  `json:"exported_value,omitempty" description:"exported time series, values of vector type"`
 	ExportedSeries []ExportPoint `json:"exported_values,omitempty" description:"exported time series, values of matrix type"`
 
-	MinValue float64 `json:"min_value" description:"minimum value from monitor points"`
-	MaxValue float64 `json:"max_value" description:"maximum value from monitor points"`
-	AvgValue float64 `json:"avg_value" description:"average value from monitor points"`
-	SumValue float64 `json:"sum_value" description:"sum value from monitor points"`
-	Fee      float64 `json:"fee" description:"resource fee"`
+	MinValue     float64 `json:"min_value" description:"minimum value from monitor points"`
+	MaxValue     float64 `json:"max_value" description:"maximum value from monitor points"`
+	AvgValue     float64 `json:"avg_value" description:"average value from monitor points"`
+	SumValue     float64 `json:"sum_value" description:"sum value from monitor points"`
+	Fee          float64 `json:"fee" description:"resource fee"`
+	ResourceUnit string  `json:"resource_unit"`
+	CurrencyUnit string  `json:"currency_unit"`
+}
+
+func (mv MetricValue) MarshalCSV() ([]byte, error) {
+	// metric value format:
+	// 	target,stats value(include fees),exported_value,exported_values
+	// for example:
+	// 	{workspace:demo-ws},,2021-02-23 01:00:00 AM 0|2021-02-23 02:00:00 AM 0|...
+	var metricValueCSVTemplate = "{%s},unit:%s|min:%.3f|max:%.3f|avg:%.3f|sum:%.3f|fee:%.2f %s,%s,%s"
+
+	var targetList []string
+	for k, v := range mv.Metadata {
+		targetList = append(targetList, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	exportedSampleStr := ""
+	if mv.ExportSample != nil {
+		exportedSampleStr = mv.ExportSample.Format()
+	}
+
+	exportedSeriesStrList := []string{}
+	for _, v := range mv.ExportedSeries {
+		exportedSeriesStrList = append(exportedSeriesStrList, v.Format())
+	}
+
+	return []byte(fmt.Sprintf(metricValueCSVTemplate,
+		strings.Join(targetList, "|"),
+		mv.ResourceUnit,
+		mv.MinValue,
+		mv.MaxValue,
+		mv.AvgValue,
+		mv.SumValue,
+		mv.Fee,
+		mv.CurrencyUnit,
+		exportedSampleStr,
+		exportedSeriesStrList)), nil
 }
 
 func (mv *MetricValue) TransferToExportedMetricValue() {
@@ -152,16 +223,6 @@ func (p ExportPoint) Value() float64 {
 	return p[1]
 }
 
-// MarshalJSON implements json.Marshaler. It will be called when writing JSON to HTTP response
-// Inspired by prometheus/client_golang
-func (p ExportPoint) MarshalJSON() ([]byte, error) {
-	t, err := jsoniter.Marshal(p.Timestamp())
-	if err != nil {
-		return nil, err
-	}
-	v, err := jsoniter.Marshal(strconv.FormatFloat(p.Value(), 'f', -1, 64))
-	if err != nil {
-		return nil, err
-	}
-	return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil
+func (p ExportPoint) Format() string {
+	return p.Timestamp() + " " + strconv.FormatFloat(p.Value(), 'f', -1, 64)
 }