diff --git a/go.mod b/go.mod index b683c8bdb..650815ccd 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.1 github.com/json-iterator/go v1.1.10 + github.com/jszwec/csvutil v1.5.0 github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/kubernetes-csi/external-snapshotter/client/v3 v3.0.0 github.com/kubesphere/sonargo v0.0.2 @@ -98,12 +99,12 @@ require ( k8s.io/kubectl v0.18.6 k8s.io/metrics v0.18.6 k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 + kubesphere.io/client-go v0.0.0 openpitrix.io/openpitrix v0.4.9-0.20200611125425-ae07f141e797 sigs.k8s.io/application v0.8.4-0.20201016185654-c8e2959e57a0 sigs.k8s.io/controller-runtime v0.6.4 sigs.k8s.io/controller-tools v0.4.0 sigs.k8s.io/kubefed v0.4.0 - kubesphere.io/client-go v0.0.0 ) replace ( @@ -735,6 +736,8 @@ replace ( k8s.io/kubectl => k8s.io/kubectl v0.18.6 k8s.io/metrics => k8s.io/metrics v0.18.6 k8s.io/utils => k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 + + kubesphere.io/client-go => ./staging/src/kubesphere.io/client-go kubesphere.io/im => kubesphere.io/im v0.1.0 openpitrix.io/iam => openpitrix.io/iam v0.1.0 openpitrix.io/libqueue => openpitrix.io/libqueue v0.4.1 @@ -758,6 +761,4 @@ replace ( sigs.k8s.io/yaml => sigs.k8s.io/yaml v1.2.0 sourcegraph.com/sourcegraph/appdash => sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 vbom.ml/util => vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc - - kubesphere.io/client-go => ./staging/src/kubesphere.io/client-go ) diff --git a/go.sum b/go.sum index ba15f7f70..b3ba4ff05 100644 --- a/go.sum +++ b/go.sum @@ -432,6 +432,8 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= +github.com/jszwec/csvutil v1.5.0 h1:ErLnF1Qzzt9svk8CUY7CyLl/W9eET+KWPIZWkE1o6JM= +github.com/jszwec/csvutil v1.5.0/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= diff --git a/pkg/kapis/monitoring/v1alpha3/helper.go b/pkg/kapis/monitoring/v1alpha3/helper.go index 7fee0477a..eb1d08a4e 100644 --- a/pkg/kapis/monitoring/v1alpha3/helper.go +++ b/pkg/kapis/monitoring/v1alpha3/helper.go @@ -19,8 +19,8 @@ package v1alpha3 import ( "bytes" "context" - "encoding/json" "fmt" + "github.com/jszwec/csvutil" "io" "strconv" "strings" @@ -388,7 +388,7 @@ func ExportMetrics(resp *restful.Response, metrics model.Metrics) { } } - resBytes, err := json.MarshalIndent(metrics, "", " ") + resBytes, err := csvutil.Marshal(metrics.Results) if err != nil { api.HandleBadRequest(resp, nil, err) return diff --git a/pkg/kapis/tenant/v1alpha2/metering.go b/pkg/kapis/tenant/v1alpha2/metering.go index 48060ea1b..c68d92a82 100644 --- a/pkg/kapis/tenant/v1alpha2/metering.go +++ b/pkg/kapis/tenant/v1alpha2/metering.go @@ -64,16 +64,17 @@ func (h *tenantHandler) QueryMeteringsHierarchy(req *restful.Request, resp *rest func (h *tenantHandler) HandlePriceInfoQuery(req *restful.Request, resp *restful.Response) { var priceInfoResponse metering.PriceInfo + priceInfoResponse.Init() meterConfig, err := monitoring.LoadYaml() if err != nil { - klog.Error(err) + klog.Warning(err) resp.WriteAsJson(priceInfoResponse) return } priceInfo := meterConfig.GetPriceInfo() - priceInfoResponse.Currency = "CNY" + priceInfoResponse.Currency = priceInfo.CurrencyUnit priceInfoResponse.CpuPerCorePerHour = priceInfo.CpuPerCorePerHour priceInfoResponse.MemPerGigabytesPerHour = priceInfo.MemPerGigabytesPerHour priceInfoResponse.IngressNetworkTrafficPerGiagabytesPerHour = priceInfo.IngressNetworkTrafficPerGiagabytesPerHour diff --git a/pkg/models/metering/type.go b/pkg/models/metering/type.go index ef6c228fc..8e3ff1631 100644 --- a/pkg/models/metering/type.go +++ b/pkg/models/metering/type.go @@ -9,6 +9,16 @@ type PriceInfo struct { PvcPerGigabytesPerHour float64 `json:"pvc_per_gigabytes_per_hour,omitempty" description:"pvc price"` } +// currently init method fill illegal value to hint that metering config file was not mounted yet +func (p *PriceInfo) Init() { + p.Currency = "" + p.CpuPerCorePerHour = -1 + p.MemPerGigabytesPerHour = -1 + p.IngressNetworkTrafficPerGiagabytesPerHour = -1 + p.EgressNetworkTrafficPerGiagabytesPerHour = -1 + p.PvcPerGigabytesPerHour = -1 +} + type PodStatistic struct { CPUUsage float64 `json:"cpu_usage" description:"cpu_usage"` MemoryUsageWoCache float64 `json:"memory_usage_wo_cache" description:"memory_usage_wo_cache"` diff --git a/pkg/models/monitoring/utils.go b/pkg/models/monitoring/utils.go index daf12409e..762a5746e 100644 --- a/pkg/models/monitoring/utils.go +++ b/pkg/models/monitoring/utils.go @@ -19,6 +19,14 @@ const ( meteringConfig = "/etc/kubesphere/metering/ks-metering.yaml" ) +var meterResourceUnitMap = map[int]string{ + METER_RESOURCE_TYPE_CPU: "cores", + METER_RESOURCE_TYPE_MEM: "bytes", + METER_RESOURCE_TYPE_NET_INGRESS: "bytes", + METER_RESOURCE_TYPE_NET_EGRESS: "bytes", + METER_RESOURCE_TYPE_PVC: "bytes", +} + var MeterResourceMap = map[string]int{ "meter_cluster_cpu_usage": METER_RESOURCE_TYPE_CPU, "meter_cluster_memory_usage": METER_RESOURCE_TYPE_MEM, @@ -67,6 +75,7 @@ type PriceInfo struct { IngressNetworkTrafficPerGiagabytesPerHour float64 `json:"ingressNetworkTrafficPerGiagabytesPerHour" yaml:"ingressNetworkTrafficPerGiagabytesPerHour"` EgressNetworkTrafficPerGigabytesPerHour float64 `json:"egressNetworkTrafficPerGigabytesPerHour" yaml:"egressNetworkTrafficPerGigabytesPerHour"` PvcPerGigabytesPerHour float64 `json:"pvcPerGigabytesPerHour" yaml:"pvcPerGigabytesPerHour"` + CurrencyUnit string `json:"currencyUnit" yaml:"currencyUnit"` } type Billing struct { @@ -143,6 +152,25 @@ func getAvgPointValue(points []monitoring.Point) float64 { return getSumPointValue(points) / float64(len(points)) } +func getCurrencyUnit() string { + meterConfig, err := LoadYaml() + if err != nil { + klog.Error(err) + return "" + } + + return meterConfig.GetPriceInfo().CurrencyUnit +} + +func getResourceUnit(meterName string) string { + if resourceType, ok := MeterResourceMap[meterName]; !ok { + klog.Errorf("invlaid meter %v", meterName) + return "" + } else { + return meterResourceUnitMap[resourceType] + } +} + func getFeeWithMeterName(meterName string, sum float64) float64 { meterConfig, err := LoadYaml() @@ -216,6 +244,8 @@ func updateMetricStatData(metric monitoring.Metric, scalingMap map[string]float6 metricData.MetricValues[index].SumValue = sum metricData.MetricValues[index].Fee = getFeeWithMeterName(metricName, sum) } + metricData.MetricValues[index].CurrencyUnit = getCurrencyUnit() + metricData.MetricValues[index].ResourceUnit = getResourceUnit(metricName) } return metricData diff --git a/pkg/simple/client/monitoring/types.go b/pkg/simple/client/monitoring/types.go index 3def940d3..f892c42f7 100644 --- a/pkg/simple/client/monitoring/types.go +++ b/pkg/simple/client/monitoring/types.go @@ -20,7 +20,9 @@ import ( "errors" "fmt" "github.com/json-iterator/go" + "github.com/jszwec/csvutil" "strconv" + "strings" "time" ) @@ -36,14 +38,46 @@ type Metadata struct { } type Metric struct { - MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum"` + MetricName string `json:"metric_name,omitempty" description:"metric name, eg. scheduler_up_sum" csv:"metric_name"` MetricData `json:"data,omitempty" description:"actual metric result"` - Error string `json:"error,omitempty"` + Error string `json:"error,omitempty" csv:"-"` +} + +type MetricValues []MetricValue + +func (m MetricValues) MarshalCSV() ([]byte, error) { + + var ret []string + for _, v := range m { + tmp, err := v.MarshalCSV() + if err != nil { + return nil, err + } + + ret = append(ret, string(tmp)) + } + + return []byte(strings.Join(ret, "||")), nil } type MetricData struct { - MetricType string `json:"resultType,omitempty" description:"result type, one of matrix, vector"` - MetricValues []MetricValue `json:"result,omitempty" description:"metric data including labels, time series and values"` + MetricType string `json:"resultType,omitempty" description:"result type, one of matrix, vector" csv:"metric_type"` + MetricValues `json:"result,omitempty" description:"metric data including labels, time series and values" csv:"metric_values"` +} + +func (m MetricData) MarshalCSV() ([]byte, error) { + var ret []byte + + for _, v := range m.MetricValues { + tmp, err := csvutil.Marshal(&v) + if err != nil { + return nil, err + } + + ret = append(ret, tmp...) + } + + return ret, nil } // The first element is the timestamp, the second is the metric value. @@ -60,11 +94,48 @@ type MetricValue struct { ExportSample *ExportPoint `json:"exported_value,omitempty" description:"exported time series, values of vector type"` ExportedSeries []ExportPoint `json:"exported_values,omitempty" description:"exported time series, values of matrix type"` - MinValue float64 `json:"min_value" description:"minimum value from monitor points"` - MaxValue float64 `json:"max_value" description:"maximum value from monitor points"` - AvgValue float64 `json:"avg_value" description:"average value from monitor points"` - SumValue float64 `json:"sum_value" description:"sum value from monitor points"` - Fee float64 `json:"fee" description:"resource fee"` + MinValue float64 `json:"min_value" description:"minimum value from monitor points"` + MaxValue float64 `json:"max_value" description:"maximum value from monitor points"` + AvgValue float64 `json:"avg_value" description:"average value from monitor points"` + SumValue float64 `json:"sum_value" description:"sum value from monitor points"` + Fee float64 `json:"fee" description:"resource fee"` + ResourceUnit string `json:"resource_unit"` + CurrencyUnit string `json:"currency_unit"` +} + +func (mv MetricValue) MarshalCSV() ([]byte, error) { + // metric value format: + // target,stats value(include fees),exported_value,exported_values + // for example: + // {workspace:demo-ws},,2021-02-23 01:00:00 AM 0|2021-02-23 02:00:00 AM 0|... + var metricValueCSVTemplate = "{%s},unit:%s|min:%.3f|max:%.3f|avg:%.3f|sum:%.3f|fee:%.2f %s,%s,%s" + + var targetList []string + for k, v := range mv.Metadata { + targetList = append(targetList, fmt.Sprintf("%s=%s", k, v)) + } + + exportedSampleStr := "" + if mv.ExportSample != nil { + exportedSampleStr = mv.ExportSample.Format() + } + + exportedSeriesStrList := []string{} + for _, v := range mv.ExportedSeries { + exportedSeriesStrList = append(exportedSeriesStrList, v.Format()) + } + + return []byte(fmt.Sprintf(metricValueCSVTemplate, + strings.Join(targetList, "|"), + mv.ResourceUnit, + mv.MinValue, + mv.MaxValue, + mv.AvgValue, + mv.SumValue, + mv.Fee, + mv.CurrencyUnit, + exportedSampleStr, + exportedSeriesStrList)), nil } func (mv *MetricValue) TransferToExportedMetricValue() { @@ -152,16 +223,6 @@ func (p ExportPoint) Value() float64 { return p[1] } -// MarshalJSON implements json.Marshaler. It will be called when writing JSON to HTTP response -// Inspired by prometheus/client_golang -func (p ExportPoint) MarshalJSON() ([]byte, error) { - t, err := jsoniter.Marshal(p.Timestamp()) - if err != nil { - return nil, err - } - v, err := jsoniter.Marshal(strconv.FormatFloat(p.Value(), 'f', -1, 64)) - if err != nil { - return nil, err - } - return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil +func (p ExportPoint) Format() string { + return p.Timestamp() + " " + strconv.FormatFloat(p.Value(), 'f', -1, 64) }