[v3.2] Add grafana dashboard importing API (#11)

* Add API to import grafana templates to kubesphere dashboard
* Merge and fix the latest codes from kubesphere #2501

Signed-off-by: zhu733756 <talonzhu@yunify.com>

pkg/apis/addtoscheme_dashboard_v1alpha2.go

@@ -0,0 +1,7 @@
+package apis
+
+import monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
+
+func init() {
+	AddToSchemes = append(AddToSchemes, monitoringdashboardv1alpha2.SchemeBuilder.AddToScheme)
+}

pkg/constants/constants.go

@@ -129,6 +129,8 @@ const (
 	NotificationTag             = "Notification"
 	NotificationSecretNamespace = "kubesphere-monitoring-federated"
 	NotificationManagedLabel    = "notification.kubesphere.io/managed"
+
+	DashboardTag = "Dashboard"
 )
 
 var (

pkg/kapis/monitoring/v1alpha3/dashboard_template_test.sh

@@ -0,0 +1,3 @@
+#! /bin/bash
+
+curl -d '{"grafanaDashboardName":"test2","grafanaDashboardUrl":"https://grafana.com/api/dashboards/7362/revisions/5/download"}' -H "Content-Type: application/json" localhost:9090/kapis/monitoring.kubesphere.io/v1alpha3/dashboard/template

pkg/kapis/monitoring/v1alpha3/handler.go

@@ -19,12 +19,19 @@
 package v1alpha3
 
 import (
+	"context"
+	"encoding/json"
 	"errors"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"regexp"
 	"strings"
 
 	"k8s.io/klog"
 
+	converter "kubesphere.io/monitoring-dashboard/tools/converter"
+
 	openpitrixoptions "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 	"kubesphere.io/kubesphere/pkg/simple/client/s3"
 
@@ -32,8 +39,11 @@ import (
 	"kubesphere.io/kubesphere/pkg/models/openpitrix"
 
 	"github.com/emicklei/go-restful"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
+
 	"kubesphere.io/kubesphere/pkg/api"
 	"kubesphere.io/kubesphere/pkg/informers"
 	model "kubesphere.io/kubesphere/pkg/models/monitoring"
@@ -302,3 +312,116 @@ func (h handler) handleAdhocQuery(req *restful.Request, resp *restful.Response)
 		resp.WriteAsJson(res)
 	}
 }
+
+// handleGrafanaDashboardImport imports Grafana template and converts it to KubeSphere dashboard.
+// The description of the Parameters:
+// grafanaDashboardName: the name of this Grafana template needed to convert.
+// grafanaDashboardUrl: the link to download this Grafana template.
+// grafanaDashboardContent: the whole JSON content needed to convert.
+// Note that the parameter grafanaDashboardName is indispensable,
+// and the requested parameter grafanaDashboardUrl and grafanaDashboardContent cannot be empty at the same time.
+func (h handler) handleGrafanaDashboardImport(req *restful.Request, resp *restful.Response) {
+	var entity monitoring.DashboardEntity
+	err := req.ReadEntity(&entity)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	if entity.GrafanaDashboardName == "" {
+		err := errors.New("the requested parameter grafanaDashboardName cannot be empty")
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+	if entity.GrafanaDashboardUrl == "" && entity.GrafanaDashboardContent == "" {
+		err := errors.New("the requested parameter grafanaDashboardUrl and grafanaDashboardContent cannot be empty at the same time")
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	grafanaDashboardContent := []byte(entity.GrafanaDashboardContent)
+	if entity.GrafanaDashboardUrl != "" {
+		c, err := func(u string) ([]byte, error) {
+			_, err := url.ParseRequestURI(u)
+			if err != nil {
+				return nil, err
+			}
+			client := &http.Client{}
+			req, err := http.NewRequest("GET", u, nil)
+			if err != nil {
+				return nil, err
+			}
+
+			r, err := client.Do(req)
+			if err != nil {
+				return nil, err
+			}
+
+			defer r.Body.Close()
+
+			c, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				return nil, err
+			}
+			return c, nil
+		}(entity.GrafanaDashboardUrl)
+
+		if err != nil {
+			api.HandleBadRequest(resp, nil, err)
+			return
+		}
+
+		grafanaDashboardContent = []byte(c)
+	}
+
+	c := converter.NewConverter()
+	convertedDashboard, err := c.ConvertToDashboard(grafanaDashboardContent, true, "", entity.GrafanaDashboardName)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	dashboard := monitoringdashboardv1alpha2.ClusterDashboard{
+		TypeMeta: v1.TypeMeta{
+			APIVersion: convertedDashboard.APIVersion,
+			Kind:       convertedDashboard.Kind,
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name: convertedDashboard.Metadata["name"],
+		},
+		Spec: *convertedDashboard.Spec,
+	}
+
+	jsonDashbaord, err := json.Marshal(dashboard)
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	// a dashboard with the same name cannot post.
+	ctx := context.TODO()
+	_, err = h.k.Discovery().RESTClient().
+		Get().
+		AbsPath("/apis/monitoring.kubesphere.io/v1alpha2/clusterdashboards/" + dashboard.Name).
+		DoRaw(ctx)
+
+	if err == nil {
+		api.HandleBadRequest(resp, nil, errors.New("a dashboard with the same name already exists!"))
+		return
+	}
+
+	// create this dashboard
+	_, err = h.k.Discovery().RESTClient().
+		Post().
+		AbsPath("/apis/monitoring.kubesphere.io/v1alpha2/clusterdashboards").
+		Body(jsonDashbaord).
+		DoRaw(ctx)
+
+	if err != nil {
+		api.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	resp.WriteAsJson(dashboard)
+
+}

pkg/kapis/monitoring/v1alpha3/register.go

@@ -20,6 +20,8 @@ package v1alpha3
 import (
 	"net/http"
 
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
+
 	openpitrixoptions "kubesphere.io/kubesphere/pkg/simple/client/openpitrix"
 
 	"kubesphere.io/kubesphere/pkg/client/clientset/versioned"
@@ -504,6 +506,14 @@ func AddToContainer(c *restful.Container, k8sClient kubernetes.Interface, monito
 		Returns(http.StatusOK, respOK, monitoring.Metric{})).
 		Produces(restful.MIME_JSON)
 
+	ws.Route(ws.POST("/dashboard/template").
+		To(h.handleGrafanaDashboardImport).
+		Doc("Convert Grafana templates to KubeSphere dashboards.").
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.DashboardTag}).
+		Writes(monitoringdashboardv1alpha2.ClusterDashboard{}).
+		Returns(http.StatusOK, respOK, monitoringdashboardv1alpha2.ClusterDashboard{})).
+		Produces(restful.MIME_JSON)
+
 	c.Add(ws)
 	return nil
 }

pkg/models/resources/v1alpha3/clusterdashboard/clusterdashboard.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
-	monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"kubesphere.io/kubesphere/pkg/api"
@@ -39,7 +39,7 @@ func New(c client.Reader) v1alpha3.Interface {
 }
 
 func (d *dashboardGetter) Get(_, name string) (runtime.Object, error) {
-	dashboard := monitoringdashboardv1alpha1.ClusterDashboard{}
+	dashboard := monitoringdashboardv1alpha2.ClusterDashboard{}
 	err := d.c.Get(context.Background(), types.NamespacedName{Name: name}, &dashboard)
 	if err != nil {
 		klog.Error(err)
@@ -49,7 +49,7 @@ func (d *dashboardGetter) Get(_, name string) (runtime.Object, error) {
 }
 
 func (d *dashboardGetter) List(_ string, query *query.Query) (*api.ListResult, error) {
-	dashboards := monitoringdashboardv1alpha1.ClusterDashboardList{}
+	dashboards := monitoringdashboardv1alpha2.ClusterDashboardList{}
 	err := d.c.List(context.Background(), &dashboards, &client.ListOptions{LabelSelector: query.Selector()})
 	if err != nil {
 		klog.Error(err)
@@ -65,12 +65,12 @@ func (d *dashboardGetter) List(_ string, query *query.Query) (*api.ListResult, e
 
 func (d *dashboardGetter) compare(left runtime.Object, right runtime.Object, field query.Field) bool {
 
-	leftClusterDashboard, ok := left.(*monitoringdashboardv1alpha1.ClusterDashboard)
+	leftClusterDashboard, ok := left.(*monitoringdashboardv1alpha2.ClusterDashboard)
 	if !ok {
 		return false
 	}
 
-	rightClusterDashboard, ok := right.(*monitoringdashboardv1alpha1.ClusterDashboard)
+	rightClusterDashboard, ok := right.(*monitoringdashboardv1alpha2.ClusterDashboard)
 	if !ok {
 		return false
 	}
@@ -79,7 +79,7 @@ func (d *dashboardGetter) compare(left runtime.Object, right runtime.Object, fie
 }
 
 func (d *dashboardGetter) filter(object runtime.Object, filter query.Filter) bool {
-	dashboard, ok := object.(*monitoringdashboardv1alpha1.ClusterDashboard)
+	dashboard, ok := object.(*monitoringdashboardv1alpha2.ClusterDashboard)
 	if !ok {
 		return false
 	}

pkg/models/resources/v1alpha3/clusterdashboard/clusterdashboard_test.go

@@ -23,7 +23,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
-	monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
@@ -32,8 +32,8 @@ import (
 
 var c client.Client
 
-func compare(actual *monitoringdashboardv1alpha1.ClusterDashboard,
-	expects ...*monitoringdashboardv1alpha1.ClusterDashboard) bool {
+func compare(actual *monitoringdashboardv1alpha2.ClusterDashboard,
+	expects ...*monitoringdashboardv1alpha2.ClusterDashboard) bool {
 	for _, app := range expects {
 		if actual.Name == app.Name && reflect.DeepEqual(actual.Labels, app.Labels) {
 			return true
@@ -44,7 +44,7 @@ func compare(actual *monitoringdashboardv1alpha1.ClusterDashboard,
 
 func TestGetListClusterDashboards(t *testing.T) {
 	sch := scheme.Scheme
-	if err := monitoringdashboardv1alpha1.AddToScheme(sch); err != nil {
+	if err := monitoringdashboardv1alpha2.AddToScheme(sch); err != nil {
 		t.Fatalf("unable add APIs to scheme: %v", err)
 	}
 
@@ -53,7 +53,7 @@ func TestGetListClusterDashboards(t *testing.T) {
 	var labelSet1 = map[string]string{"foo-1": "bar-1"}
 	var labelSet2 = map[string]string{"foo-2": "bar-2"}
 
-	testCases := []*monitoringdashboardv1alpha1.ClusterDashboard{
+	testCases := []*monitoringdashboardv1alpha2.ClusterDashboard{
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:   "clusterdashboard-1",
@@ -92,7 +92,7 @@ func TestGetListClusterDashboards(t *testing.T) {
 	}
 
 	for _, dashboard := range results.Items {
-		dashboard, err := dashboard.(*monitoringdashboardv1alpha1.ClusterDashboard)
+		dashboard, err := dashboard.(*monitoringdashboardv1alpha2.ClusterDashboard)
 		if !err {
 			t.Fatal(err)
 		}
@@ -106,7 +106,7 @@ func TestGetListClusterDashboards(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dashboard := result.(*monitoringdashboardv1alpha1.ClusterDashboard)
+	dashboard := result.(*monitoringdashboardv1alpha2.ClusterDashboard)
 	if !compare(dashboard, testCases...) {
 		t.Errorf("The results %v not match testcases %v", result, testCases)
 	}

pkg/models/resources/v1alpha3/dashboard/dashboard.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog"
-	monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"kubesphere.io/kubesphere/pkg/api"
@@ -39,7 +39,7 @@ func New(c client.Reader) v1alpha3.Interface {
 }
 
 func (d *dashboardGetter) Get(namespace, name string) (runtime.Object, error) {
-	dashboard := monitoringdashboardv1alpha1.Dashboard{}
+	dashboard := monitoringdashboardv1alpha2.Dashboard{}
 	err := d.c.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, &dashboard)
 	if err != nil {
 		klog.Error(err)
@@ -49,7 +49,7 @@ func (d *dashboardGetter) Get(namespace, name string) (runtime.Object, error) {
 }
 
 func (d *dashboardGetter) List(namespace string, query *query.Query) (*api.ListResult, error) {
-	dashboards := monitoringdashboardv1alpha1.DashboardList{}
+	dashboards := monitoringdashboardv1alpha2.DashboardList{}
 	err := d.c.List(context.Background(), &dashboards, &client.ListOptions{Namespace: namespace, LabelSelector: query.Selector()})
 	if err != nil {
 		klog.Error(err)
@@ -65,12 +65,12 @@ func (d *dashboardGetter) List(namespace string, query *query.Query) (*api.ListR
 
 func (d *dashboardGetter) compare(left runtime.Object, right runtime.Object, field query.Field) bool {
 
-	leftDashboard, ok := left.(*monitoringdashboardv1alpha1.Dashboard)
+	leftDashboard, ok := left.(*monitoringdashboardv1alpha2.Dashboard)
 	if !ok {
 		return false
 	}
 
-	rightDashboard, ok := right.(*monitoringdashboardv1alpha1.Dashboard)
+	rightDashboard, ok := right.(*monitoringdashboardv1alpha2.Dashboard)
 	if !ok {
 		return false
 	}
@@ -79,7 +79,7 @@ func (d *dashboardGetter) compare(left runtime.Object, right runtime.Object, fie
 }
 
 func (d *dashboardGetter) filter(object runtime.Object, filter query.Filter) bool {
-	dashboard, ok := object.(*monitoringdashboardv1alpha1.Dashboard)
+	dashboard, ok := object.(*monitoringdashboardv1alpha2.Dashboard)
 	if !ok {
 		return false
 	}

pkg/models/resources/v1alpha3/dashboard/dashboard_test.go

@@ -23,7 +23,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes/scheme"
-	monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 
@@ -32,7 +32,7 @@ import (
 
 var c client.Client
 
-func compare(actual *monitoringdashboardv1alpha1.Dashboard, expects ...*monitoringdashboardv1alpha1.Dashboard) bool {
+func compare(actual *monitoringdashboardv1alpha2.Dashboard, expects ...*monitoringdashboardv1alpha2.Dashboard) bool {
 	for _, app := range expects {
 		if actual.Name == app.Name && actual.Namespace == app.Namespace && reflect.DeepEqual(actual.Labels, app.Labels) {
 			return true
@@ -43,7 +43,7 @@ func compare(actual *monitoringdashboardv1alpha1.Dashboard, expects ...*monitori
 
 func TestGetListDashboards(t *testing.T) {
 	sch := scheme.Scheme
-	if err := monitoringdashboardv1alpha1.AddToScheme(sch); err != nil {
+	if err := monitoringdashboardv1alpha2.AddToScheme(sch); err != nil {
 		t.Fatalf("unable add APIs to scheme: %v", err)
 	}
 
@@ -53,7 +53,7 @@ func TestGetListDashboards(t *testing.T) {
 	var labelSet2 = map[string]string{"foo-2": "bar-2"}
 
 	var ns = "ns-1"
-	testCases := []*monitoringdashboardv1alpha1.Dashboard{
+	testCases := []*monitoringdashboardv1alpha2.Dashboard{
 		{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      "dashboard-1",
@@ -94,7 +94,7 @@ func TestGetListDashboards(t *testing.T) {
 	}
 
 	for _, dashboard := range results.Items {
-		dashboard, err := dashboard.(*monitoringdashboardv1alpha1.Dashboard)
+		dashboard, err := dashboard.(*monitoringdashboardv1alpha2.Dashboard)
 		if !err {
 			t.Fatal(err)
 		}
@@ -108,7 +108,7 @@ func TestGetListDashboards(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	dashboard := result.(*monitoringdashboardv1alpha1.Dashboard)
+	dashboard := result.(*monitoringdashboardv1alpha2.Dashboard)
 	if !compare(dashboard, testCases...) {
 		t.Errorf("The results %v not match testcases %v", result, testCases)
 	}

pkg/models/resources/v1alpha3/resource/resource.go

@@ -23,7 +23,7 @@ import (
 	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	monitoringdashboardv1alpha1 "kubesphere.io/monitoring-dashboard/api/v1alpha1"
+	monitoringdashboardv1alpha2 "kubesphere.io/monitoring-dashboard/api/v1alpha2"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 
 	clusterv1alpha1 "kubesphere.io/api/cluster/v1alpha1"
@@ -136,7 +136,7 @@ func NewResourceGetter(factory informers.InformerFactory, cache cache.Cache) *Re
 	clusterResourceGetters[clusterv1alpha1.SchemeGroupVersion.WithResource(clusterv1alpha1.ResourcesPluralCluster)] = cluster.New(factory.KubeSphereSharedInformerFactory())
 	clusterResourceGetters[notificationv2beta1.SchemeGroupVersion.WithResource(notificationv2beta1.ResourcesPluralConfig)] = notification.NewNotificationConfigGetter(factory.KubeSphereSharedInformerFactory())
 	clusterResourceGetters[notificationv2beta1.SchemeGroupVersion.WithResource(notificationv2beta1.ResourcesPluralReceiver)] = notification.NewNotificationReceiverGetter(factory.KubeSphereSharedInformerFactory())
-	clusterResourceGetters[monitoringdashboardv1alpha1.GroupVersion.WithResource("clusterdashboards")] = clusterdashboard.New(cache)
+	clusterResourceGetters[monitoringdashboardv1alpha2.GroupVersion.WithResource("clusterdashboards")] = clusterdashboard.New(cache)
 
 	// federated resources
 	namespacedResourceGetters[typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedNamespace)] = federatednamespace.New(factory.KubeSphereSharedInformerFactory())
@@ -148,7 +148,7 @@ func NewResourceGetter(factory informers.InformerFactory, cache cache.Cache) *Re
 	namespacedResourceGetters[typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedPersistentVolumeClaim)] = federatedpersistentvolumeclaim.New(factory.KubeSphereSharedInformerFactory())
 	namespacedResourceGetters[typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedStatefulSet)] = federatedstatefulset.New(factory.KubeSphereSharedInformerFactory())
 	namespacedResourceGetters[typesv1beta1.SchemeGroupVersion.WithResource(typesv1beta1.ResourcePluralFederatedIngress)] = federatedingress.New(factory.KubeSphereSharedInformerFactory())
-	namespacedResourceGetters[monitoringdashboardv1alpha1.GroupVersion.WithResource("dashboards")] = dashboard.New(cache)
+	namespacedResourceGetters[monitoringdashboardv1alpha2.GroupVersion.WithResource("dashboards")] = dashboard.New(cache)
 
 	return &ResourceGetter{
 		namespacedResourceGetters: namespacedResourceGetters,

pkg/simple/client/monitoring/types.go

@@ -49,6 +49,12 @@ type MetricData struct {
 	MetricValues `json:"result,omitempty" description:"metric data including labels, time series and values" csv:"metric_values"`
 }
 
+type DashboardEntity struct {
+	GrafanaDashboardName    string `json:"grafanaDashboardName"`
+	GrafanaDashboardUrl     string `json:"grafanaDashboardUrl,omitempty"`
+	GrafanaDashboardContent string `json:"grafanaDashboardContent,omitempty"`
+}
+
 // The first element is the timestamp, the second is the metric value.
 // eg, [1585658599.195, 0.528]
 type Point [2]float64