kubesphere/vendor/kubesphere.io/monitoring-dashboard/tools/converter/dashboard_converter.go

package converter

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"strconv"

	yamlConverter "github.com/ghodss/yaml"
	"github.com/grafana-tools/sdk"
	"github.com/mitchellh/mapstructure"
	v1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
	ansModel "kubesphere.io/monitoring-dashboard/api/v1alpha2/annotations"
	panelsModel "kubesphere.io/monitoring-dashboard/api/v1alpha2/panels"
	templatingsModel "kubesphere.io/monitoring-dashboard/api/v1alpha2/templatings"
)

type k8sDashboard struct {
	APIVersion string                  `json:"apiVersion" yaml:"apiVersion"`
	Kind       string                  `json:"kind" yaml:"kind"`
	Metadata   map[string]string       `json:"metadata" yaml:"metadata"`
	Spec       *v1alpha2.DashboardSpec `json:"spec" yaml:"spec"`
}

// Converter struct: this struct has a log property, so other newly added methods can access this log
type Converter struct {
	OutputJson []byte
	OutputYaml []byte
}

// NewConverter: new a Converter struct object with a logger object
func NewConverter() *Converter {
	return &Converter{}
}

// ConvertToDashboard converts the input json content to Dashboard model
func (converter *Converter) ConvertToDashboard(content []byte, isClusterCrd bool, ns string, name string) (*k8sDashboard, error) {
	// convert to a dashboard
	dashboard, err := converter.convert(content, isClusterCrd)
	if err != nil {
		return nil, fmt.Errorf("could parse input: %s", err.Error())
	}

	apiVersion := v1alpha2.GroupVersion.Group + "/" + v1alpha2.GroupVersion.Version

	kind := "Dashboard"
	if isClusterCrd {
		kind = "ClusterDashboard"
	}

	metadata := make(map[string]string)
	if ns == "" {
		ns = "default"
	}
	if !isClusterCrd {
		metadata["namespace"] = ns
	}
	metadata["name"] = name

	return &k8sDashboard{
		APIVersion: apiVersion,
		Kind:       kind,
		Metadata:   metadata,
		Spec:       dashboard,
	}, nil

}

// ConvertDashboardToJson converts the input json content to json bytes content
func (converter *Converter) ConvertDashboardToJson(content []byte, isClusterCrd bool, ns string, name string) error {
	manifest, err := converter.ConvertToDashboard(content, isClusterCrd, ns, name)
	if err != nil {
		return fmt.Errorf("could not convert json content to dashboard: %s", err.Error())
	}
	convertedJson, err := json.Marshal(manifest)
	if err != nil {
		return fmt.Errorf("could not marshal dashboard to json: %s", err.Error())
	}

	converter.OutputJson = convertedJson
	return nil
}

// ConvertDashboardToYaml converts the input json content to yaml bytes content
func (converter *Converter) ConvertDashboardToYaml(content []byte, isClusterCrd bool, ns string, name string) error {
	err := converter.ConvertDashboardToJson(content, isClusterCrd, ns, name)
	if err != nil {
		return fmt.Errorf("could not marshal dashboard to json: %s", err.Error())
	}
	convertedYaml, err := yamlConverter.JSONToYAML(converter.OutputJson)
	if err != nil {
		return fmt.Errorf("could not convert json to yaml: %s", err.Error())
	}

	converter.OutputYaml = convertedYaml
	return nil
}

// ConvertFromFile converts the input json file to yaml/json bytes content
func (converter *Converter) ConvertFromFile(input io.Reader, isClusterCrd bool, ns string, name string) error {
	content, err := ioutil.ReadAll(input)
	if err != nil {
		return fmt.Errorf("could not read input: %s", err.Error())
	}

	err = converter.ConvertDashboardToYaml(content, isClusterCrd, ns, name)
	if err != nil {
		return fmt.Errorf("could not convert from input: %s", err.Error())
	}

	return nil
}

// ConvertToKubsphereDashboardManifests converts to a k8s mainfest file
func (converter *Converter) ConvertToKubsphereDashboardManifests(input io.Reader, output io.Writer, isClusterCrd bool, ns string, name string) error {

	err := converter.ConvertFromFile(input, isClusterCrd, ns, name)
	if err != nil {
		return err
	}
	_, err = output.Write([]byte(converter.OutputYaml))
	if err != nil {
		return err
	}
	return nil
}

// convert reads a input Converter file, then extract needed fields to the yaml model
func (converter *Converter) convert(content []byte, isClusterCrd bool) (*v1alpha2.DashboardSpec, error) {

	board := &sdk.Board{}
	if err := json.Unmarshal(content, board); err != nil {
		return nil, fmt.Errorf("could not unmarshall dashboard: %s", err.Error())
	}

	// a yaml model
	dashboard := &v1alpha2.DashboardSpec{}

	// starts to convert general settings
	converter.convertGeneralSettings(board, dashboard)
	// starts to convert templating variables
	converter.convertVariables(board.Templating.List, dashboard)
	// starts to convert annotations
	converter.convertAnnotations(board.Annotations.List, dashboard)
	// starts to convert pannels
	converter.convertPanels(board.Panels, dashboard, isClusterCrd)
	// starts to convert rows
	// some dashboards only include rows
	converter.convertRows(board.Rows, dashboard, isClusterCrd)

	return dashboard, nil
}

// convert GeneralSettings
func (converter *Converter) convertGeneralSettings(board *sdk.Board, dashboard *v1alpha2.DashboardSpec) {
	dashboard.Title = board.Title
	dashboard.Editable = board.Editable
	dashboard.SharedCrosshair = board.SharedCrosshair
	dashboard.Tags = board.Tags
	dashboard.Time.From = board.Time.From
	dashboard.Time.To = board.Time.To
	dashboard.Timezone = board.Timezone

	if board.Refresh != nil {
		dashboard.AutoRefresh = board.Refresh.Value
	}
}

// convert Annotations
func (converter *Converter) convertAnnotations(annotations []sdk.Annotation, dashboard *v1alpha2.DashboardSpec) {
	for _, annotation := range annotations {
		// grafana-sdk doesn't expose the "builtIn" field, so we work around that by skipping
		// the annotation we know to be built-in by its name
		if annotation.Name == "Annotations & Alerts" {
			continue
		}

		if annotation.Type != "tags" {
			continue
		}

		datasource := ""
		if annotation.Datasource != nil {
			datasource = *annotation.Datasource
		}

		dashboard.Annotations = append(dashboard.Annotations, ansModel.Annotation{
			Name:        annotation.Name,
			Datasource:  datasource,
			IconColor:   annotation.IconColor,
			Tags:        annotation.Tags,
			ShowLine:    annotation.ShowLine,
			LineColor:   annotation.LineColor,
			IconSize:    annotation.IconSize,
			Enable:      annotation.Enable,
			Query:       annotation.Query,
			Expr:        annotation.Expr,
			Step:        annotation.Step,
			TextField:   annotation.TextField,
			TextFormat:  annotation.TextFormat,
			TitleFormat: annotation.TitleFormat,
			TagsField:   annotation.TagsField,
			TagKeys:     annotation.TagKeys,
			Type:        annotation.Type,
		})
	}
}

// convert templating variables
func (converter *Converter) convertVariables(variables []sdk.TemplateVar, dashboard *v1alpha2.DashboardSpec) {
	for _, variable := range variables {
		if variable.Query == nil {
			continue
		}
		q, ok := variable.Query.(string)
		if !ok {
			continue
		}
		var options []templatingsModel.Option
		for _, op := range variable.Options {
			options = append(options, templatingsModel.Option{
				Text:     op.Text,
				Value:    op.Value,
				Selected: op.Selected,
			})
		}
		v := templatingsModel.TemplateVar{
			Name:        variable.Name,
			Type:        variable.Type,
			Auto:        variable.Auto,
			AutoCount:   variable.AutoCount,
			Datasource:  variable.Datasource,
			Options:     options,
			Query:       q,
			IncludeAll:  variable.IncludeAll,
			AllFormat:   variable.AllFormat,
			AllValue:    variable.AllValue,
			Multi:       variable.Multi,
			MultiFormat: variable.MultiFormat,
			Regex:       variable.Regex,
			Label:       variable.Label,
			Hide:        variable.Hide,
			Sort:        variable.Sort,
		}
		dashboard.Templatings = append(dashboard.Templatings, v)
	}
}

//convert rows
func (converter *Converter) convertPanels(panels []*sdk.Panel, dashboard *v1alpha2.DashboardSpec, isClusterCrd bool) {

	for _, panel := range panels {
		if panel.Type == "row" {
			for _, rowPanel := range panel.Panels {
				convertedPanel, ok := converter.convertDataPanel(rowPanel, isClusterCrd)
				if ok {
					dashboard.Panels = append(dashboard.Panels, convertedPanel)
				}
			}
		} else {
			convertedPanel, ok := converter.convertDataPanel(*panel, isClusterCrd)
			if ok {
				dashboard.Panels = append(dashboard.Panels, convertedPanel)
			}
		}
	}

}

//convert rows
func (converter *Converter) convertRows(rows []*sdk.Row, dashboard *v1alpha2.DashboardSpec, isClusterCrd bool) {

	for _, row := range rows {
		if row == nil {
			continue
		}
		panels := row.Panels
		if panels == nil || len(rows) == 0 {
			continue
		}
		for _, pl := range panels {
			convertedPanel, ok := converter.convertDataPanel(pl, isClusterCrd)
			if ok {
				dashboard.Panels = append(dashboard.Panels, convertedPanel)
			}
		}
	}
}

// convert different types of the given panel
func (converter *Converter) convertDataPanel(panel sdk.Panel, isClusterCrd bool) (*panelsModel.Panel, bool) {
	switch panel.Type {
	case "graph":
		return converter.convertGraph(panel, isClusterCrd), true
	case "singlestat":
		return converter.convertSingleStat(panel, isClusterCrd), true
	case "bargauge":
		return converter.convertBarGauge(panel, isClusterCrd), true
	case "table":
		return converter.convertTable(panel, isClusterCrd), true
	case "text":
		return converter.convertText(panel), true
	default:
		if panel.OfType == sdk.CustomType {
			return converter.convertCustom(panel, isClusterCrd), true
		}
	}
	return &panelsModel.Panel{}, false
}

// a graph panel
func (converter *Converter) convertGraph(panel sdk.Panel, isClusterCrd bool) *panelsModel.Panel {
	// filled with values of the given fields
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}
	graph := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        panel.Type,
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
			Colors:      defaultColors(),
		},
	}

	if panel.GraphPanel == nil {
		return graph
	}

	graph.CommonPanel.Decimals = uintpointToInt64point(panel.GraphPanel.Decimals)
	graph.CommonPanel.Legend = converter.convertLegend(panel.GraphPanel.Legend)

	graph.GraphPanel = &panelsModel.GraphPanel{
		Bars:  panel.GraphPanel.Bars,
		Lines: panel.GraphPanel.Lines,
		Stack: panel.GraphPanel.Stack,
		Xaxis: panelsModel.Axis{
			Format:   panel.GraphPanel.Xaxis.Format,
			Decimals: int64(panel.GraphPanel.Xaxis.Decimals),
		},
	}

	// converts target
	if panel.GraphPanel.Targets != nil && len(panel.GraphPanel.Targets) > 0 {
		for index, target := range panel.GraphPanel.Targets {
			graphTarget := converter.convertTarget(target, index)
			if graphTarget == nil {
				continue
			}

			graph.CommonPanel.Targets = append(graph.CommonPanel.Targets, *graphTarget)
		}

	}

	// converts yaxes
	for _, yaxis := range panel.GraphPanel.Yaxes {
		graph.GraphPanel.Yaxes = append(graph.GraphPanel.Yaxes, panelsModel.Axis{
			Format:   handleGraphFormat(yaxis.Format),
			Decimals: int64(yaxis.Decimals),
		})
		break
	}

	return graph
}

func (converter *Converter) convertLegend(sdkLegend sdk.Legend) []string {
	var legend []string

	if !sdkLegend.Show {
		legend = append(legend, "hide")
	}
	if sdkLegend.AlignAsTable {
		legend = append(legend, "as_table")
	}
	if sdkLegend.RightSide {
		legend = append(legend, "to_the_right")
	}
	if sdkLegend.Min {
		legend = append(legend, "min")
	}
	if sdkLegend.Max {
		legend = append(legend, "max")
	}
	if sdkLegend.Avg {
		legend = append(legend, "avg")
	}
	if sdkLegend.Current {
		legend = append(legend, "current")
	}
	if sdkLegend.Total {
		legend = append(legend, "total")
	}
	if sdkLegend.HideEmpty {
		legend = append(legend, "no_null_series")
	}
	if sdkLegend.HideZero {
		legend = append(legend, "no_zero_series")
	}

	return legend
}

// singlestat panel
func (converter *Converter) convertSingleStat(panel sdk.Panel, isClusterCrd bool) *panelsModel.Panel {
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}
	singleStat := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        panel.Type,
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
		},
	}

	if panel.SinglestatPanel == nil {
		return singleStat
	}

	singleStat.CommonPanel.Format = panel.SinglestatPanel.Format
	singleStat.CommonPanel.Decimals = intToInt64point(panel.SinglestatPanel.Decimals)

	singleStat.SinglestatPanel = &panelsModel.SinglestatPanel{
		ValueName: panel.SinglestatPanel.ValueName,
	}

	if len(panel.SinglestatPanel.Colors) == 3 {
		singleStat.CommonPanel.Colors = []string{
			panel.SinglestatPanel.Colors[0],
			panel.SinglestatPanel.Colors[1],
			panel.SinglestatPanel.Colors[2],
		}
	} else {
		singleStat.CommonPanel.Colors = defaultColors()
	}

	if panel.SinglestatPanel.SparkLine.Show && panel.SinglestatPanel.SparkLine.Full {
		singleStat.SparkLine = "full"
	}
	if panel.SinglestatPanel.SparkLine.Show && !panel.SinglestatPanel.SparkLine.Full {
		singleStat.SparkLine = "bottom"
	}

	// handles targets
	if panel.SinglestatPanel.Targets != nil && len(panel.SinglestatPanel.Targets) > 0 {
		for index, target := range panel.SinglestatPanel.Targets {
			target := converter.convertTarget(target, index)
			if target == nil {
				continue
			}

			singleStat.CommonPanel.Targets = append(singleStat.CommonPanel.Targets, *target)
		}

	}

	// handles gauge
	singleStat.Gauge = panelsModel.Gauge{
		MaxValue:         int64(panel.SinglestatPanel.Gauge.MaxValue),
		MinValue:         int64(panel.SinglestatPanel.Gauge.MinValue),
		Show:             panel.SinglestatPanel.Gauge.Show,
		ThresholdLabels:  panel.SinglestatPanel.Gauge.ThresholdLabels,
		ThresholdMarkers: panel.SinglestatPanel.Gauge.ThresholdMarkers,
	}

	return singleStat
}

// gauge
func (converter *Converter) convertCustom(panel sdk.Panel, isClusterCrd bool) *panelsModel.Panel {
	// set options
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}
	customPanel := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        "singlestat",
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
		},
	}

	if panel.CustomPanel == nil {
		return customPanel
	}

	var sdkTargets []sdk.Target

	custom := *panel.CustomPanel

	if err := mapstructure.Decode(custom["targets"], &sdkTargets); err != nil {
		return customPanel
	}

	var targets []panelsModel.Target

	for index, target := range sdkTargets {
		t := converter.convertTarget(target, index)
		if t == nil {
			continue
		}
		targets = append(targets, *t)
	}

	customPanel.CommonPanel.Targets = targets

	return customPanel
}

// bar gauge
func (converter *Converter) convertBarGauge(panel sdk.Panel, isClusterCrd bool) *panelsModel.Panel {
	// set options
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}
	barGaugePanel := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        panel.Type,
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
		},
	}

	if panel.BarGaugePanel == nil {
		return barGaugePanel
	}

	barGaugePanel.BarGaugePanel = &panelsModel.BarGaugePanel{
		Options: &panelsModel.BarGaugeOptions{
			Orientation: panel.BarGaugePanel.Options.Orientation,
			TextMode:    panel.BarGaugePanel.Options.TextMode,
			ColorMode:   panel.BarGaugePanel.Options.ColorMode,
			GraphMode:   panel.BarGaugePanel.Options.GraphMode,
			JustifyMode: panel.BarGaugePanel.Options.JustifyMode,
			DisplayMode: panel.BarGaugePanel.Options.DisplayMode,
			Content:     panel.BarGaugePanel.Options.Content,
			Mode:        panel.BarGaugePanel.Options.Mode,
		},
	}

	// handles targets
	if panel.BarGaugePanel.Targets != nil && len(panel.BarGaugePanel.Targets) > 0 {
		for index, target := range panel.BarGaugePanel.Targets {
			barGaugeTarget := converter.convertTarget(target, index)
			if barGaugeTarget == nil {
				continue
			}

			barGaugePanel.CommonPanel.Targets = append(barGaugePanel.CommonPanel.Targets, *barGaugeTarget)
		}

	}

	return barGaugePanel
}

// converts a table panel
func (converter *Converter) convertTable(panel sdk.Panel, isClusterCrd bool) *panelsModel.Panel {
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}
	tablePanel := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        panel.Type,
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
		},
	}

	if panel.TablePanel == nil {
		return tablePanel
	}

	tablePanel.TablePanel = &panelsModel.TablePanel{
		Scroll: panel.TablePanel.Scroll,
	}

	if panel.TablePanel.Targets != nil && len(panel.TablePanel.Targets) > 0 {
		for index, target := range panel.TablePanel.Targets {
			graphTarget := converter.convertTarget(target, index)
			if graphTarget == nil {
				continue
			}

			tablePanel.CommonPanel.Targets = append(tablePanel.CommonPanel.Targets, *graphTarget)
		}
	}

	if panel.TablePanel.Sort != nil {
		tablePanel.TablePanel.Sort = &panelsModel.Sort{
			Col:  panel.TablePanel.Sort.Col,
			Desc: panel.TablePanel.Sort.Desc,
		}
	}

	return tablePanel
}

// converts a text panel
func (converter *Converter) convertText(panel sdk.Panel) *panelsModel.Panel {
	var height *string
	if panel.Height != nil {
		var h = panel.Height.(string)
		height = &h
	}

	textPanel := &panelsModel.Panel{
		CommonPanel: panelsModel.CommonPanel{
			Title:       panel.Title,
			Id:          int64(panel.ID),
			Type:        panel.Type,
			Description: panel.CommonPanel.Description,
			Height:      height,
			Datasource:  panel.Datasource,
		},
	}

	if panel.TextPanel == nil {
		return textPanel
	}

	textPanel.TextPanel = &panelsModel.TextPanel{
		Mode:    panel.TextPanel.Mode,
		Content: panel.TextPanel.Content,
	}

	return textPanel
}

func (converter *Converter) convertTarget(target sdk.Target, index int) *panelsModel.Target {
	// looks like a prometheus target
	return converter.convertPrometheusTarget(target, index)
}

func (converter *Converter) convertPrometheusTarget(target sdk.Target, index int) *panelsModel.Target {
	t := &panelsModel.Target{
		// RefID: target.RefID,
		RefID:        int64(index) + 1,
		LegendFormat: handleLegendFormat(target.LegendFormat),
	}

	// adjusts the query expression to adapt to the ks cluster
	converedExpr := convertExpr(target.Expr)
	if converedExpr == "" {
		t.Expression = target.Expr
		return t
	}

	t.Expression = fmt.Sprintf("%s", converedExpr)
	t.Step = toString(target.Step)
	return t

}

func panelSpan(panel sdk.Panel) int64 {
	return int64(panel.ID)
}

func defaultOption(opt sdk.Current) string {
	if opt.Value == nil {
		return ""
	}

	return opt.Value.(string)
}

func handleZimu(z string) int64 {
	n, _ := strconv.Atoi(z)
	return int64(n) + 1
}

func toString(step int) string {
	// number := int(step / 60)
	// if number == 0 {
	// 	return strconv.Itoa(step) + "s"
	// }
	// if number > 60 {
	// 	return strconv.Itoa(number/60) + "h"
	// }
	// return strconv.Itoa(number) + "m"
	return "1m"
}

func handleGraphFormat(f string) string {

	if f == "bytes" || f == "Bps" {
		f = "Byte"
	} else if f == "percent" || f == "percentunit" {
		f = "percent (0.0-1.0)"
	} else {
		f = "none"
	}
	return f
}

func handleLegendFormat(l string) string {
	badPat := regexp.MustCompile(`\{(\s+\w+\s+)\}`)
	if match := badPat.Match([]byte(l)); match {
		f := func(s string) string {
			stripReg := regexp.MustCompile(`\s+`)
			return stripReg.ReplaceAllString(s, "")
		}
		return badPat.ReplaceAllStringFunc(l, f)
	}
	return l
}

func pointToString(des *string) string {
	d := ""
	if des != nil {
		d = *des
	}
	return d
}

func uintpointToInt64point(up *uint) *int64 {
	if up != nil {
		var t = int64(*up)
		return &t
	}
	return nil
}

func intToInt64point(o int) *int64 {
	var c = int64(o)
	return &c
}

func defaultColors() []string {
	return []string{"#60acfc", "#23c2db", "#64d5b2", "#d5ec5a", "#ffb64e", "#fb816d", "#d15c7f"}
}

func convertExpr(expr string) string {

	// free the door if don't match a `[{}]` regex style
	pat := regexp.MustCompile(`[\{\}]`)
	if !pat.Match([]byte(expr)) {
		return ""
	}
	// handles $interval or $__interval
	pat1 := regexp.MustCompile(`\$_{0,2}interval`)
	if pat1.Match([]byte(expr)) {
		expr = pat1.ReplaceAllString(expr, "3m")
	}

	// if contains irate/rate/count func, just removes `\{.*\}`
	pat2 := regexp.MustCompile(`\{.*?\}`)
	if matchCommon := pat2.Match([]byte(expr)); matchCommon {
		expr = pat2.ReplaceAllString(expr, "")
	}

	// if contains count, removes `>\d+`
	pat3 := regexp.MustCompile(`>\d+`)
	if matchCount := pat3.Match([]byte(expr)); matchCount {
		expr = pat3.ReplaceAllString(expr, "")
	}
	return expr

}