custom alerting

Signed-off-by: junotx <junotx@126.com>

pkg/api/customalerting/v1alpha1/types.go

@@ -0,0 +1,389 @@
+/*
+Copyright 2020 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/emicklei/go-restful"
+	"github.com/pkg/errors"
+	prommodel "github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/promql/parser"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+const (
+	RuleLevelCluster   RuleLevel = "cluster"
+	RuleLevelNamespace RuleLevel = "namespace"
+)
+
+var (
+	ErrThanosRulerNotEnabled     = errors.New("The request operation to custom alerting rule could not be done because thanos ruler is not enabled")
+	ErrAlertingRuleNotFound      = errors.New("The alerting rule was not found")
+	ErrAlertingRuleAlreadyExists = errors.New("The alerting rule already exists")
+
+	ruleLabelNameMatcher = regexp.MustCompile(`[a-zA-Z_][a-zA-Z0-9_]*`)
+)
+
+type RuleLevel string
+
+type AlertingRuleQualifier struct {
+	Id     string    `json:"id,omitempty" description:"rule id is mainly for the builtin rules"`
+	Name   string    `json:"name,omitempty" description:"rule name which should be uniq for custom rules in the specified namespace"`
+	Level  RuleLevel `json:"level,omitempty" description:"rule level is only for the custom rules and its value is one of cluster, namespace"`
+	Custom bool      `json:"custom" description:"whether to be a custom rule. The builtin rules are not custom and can only be viewed"`
+}
+
+type AlertingRuleProps struct {
+	Query       string            `json:"query,omitempty" description:"prometheus query expression, grammars of which may be referred to https://prometheus.io/docs/prometheus/latest/querying/basics/"`
+	Duration    string            `json:"duration,omitempty" description:"duration an alert transitions from Pending to Firing state, which must match ^([0-9]+)(y|w|d|h|m|s|ms)$"`
+	Labels      map[string]string `json:"labels,omitempty" description:"extra labels to attach to the resulting alert sample vectors (the key string has to match [a-zA-Z_][a-zA-Z0-9_]*). eg: a typical label called severity, whose value may be info, warning, error, critical, is usually used to indicate the severity of an alert"`
+	Annotations map[string]string `json:"annotations,omitempty" description:"non-identifying key/value pairs. summary, message, description are the commonly used annotation names"`
+}
+
+type PostableAlertingRule struct {
+	Name        string `json:"name,omitempty" description:"rule name which should be uniq for custom rules in the specified namespace"`
+	Alias       string `json:"alias,omitempty" description:"alias for the rule"`
+	Description string `json:"description,omitempty" description:"description for the rule"`
+
+	AlertingRuleProps `json:",omitempty"`
+}
+
+func (r *PostableAlertingRule) Validate() error {
+	errs := []error{}
+
+	if r.Name == "" {
+		errs = append(errs, errors.New("name can not be empty"))
+	}
+	if _, err := parser.ParseExpr(r.Query); err != nil {
+		errs = append(errs, errors.Wrapf(err, "query is invalid: %s", r.Query))
+	}
+	if r.Duration != "" {
+		if _, err := prommodel.ParseDuration(r.Duration); err != nil {
+			errs = append(errs, errors.Wrapf(err, "duration is invalid: %s", r.Duration))
+		}
+	}
+
+	if len(r.Labels) > 0 {
+		for name, _ := range r.Labels {
+			if !ruleLabelNameMatcher.MatchString(name) || strings.HasPrefix(name, "__") {
+				errs = append(errs, errors.Errorf(
+					"label name (%s) is not valid. The name must match [a-zA-Z_][a-zA-Z0-9_]* and has not the __ prefix (label names with this prefix are for internal use)", name))
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errs)
+}
+
+type GettableAlertingRule struct {
+	AlertingRuleQualifier `json:",omitempty"`
+	AlertingRuleProps     `json:",omitempty"`
+
+	Alias       string `json:"alias,omitempty" description:"alias for the rule"`
+	Description string `json:"description,omitempty" description:"description for the rule"`
+
+	State                     string     `json:"state,omitempty" description:"state of a rule based on its alerts, one of firing, pending, inactive"`
+	Health                    string     `json:"health,omitempty" description:"health state of a rule based on the last execution, one of ok, err, unknown"`
+	LastError                 string     `json:"lastError,omitempty" description:"error for the last execution"`
+	EvaluationDurationSeconds float64    `json:"evaluationTime,omitempty" description:"taken seconds for evaluation of query expression"`
+	LastEvaluation            *time.Time `json:"lastEvaluation,omitempty" description:"time for last evaluation of query expression"`
+	Alerts                    []*Alert   `json:"alerts,omitempty" description:"alerts"`
+}
+
+type GettableAlertingRuleList struct {
+	Items []*GettableAlertingRule `json:"items"`
+	Total int                     `json:"total"`
+}
+
+type Alert struct {
+	ActiveAt    *time.Time        `json:"activeAt,omitempty" description:"time when alert is active"`
+	Annotations map[string]string `json:"annotations,omitempty" description:"annotations"`
+	Labels      map[string]string `json:"labels,omitempty" description:"labels"`
+	State       string            `json:"state,omitempty" description:"state"`
+	Value       string            `json:"value,omitempty" description:"the value at the last evaluation of the query expression"`
+
+	Rule *AlertingRuleQualifier `json:"rule,omitempty" description:"rule triggering the alert"`
+}
+
+type AlertList struct {
+	Items []*Alert `json:"items"`
+	Total int      `json:"total"`
+}
+
+type AlertingRuleQueryParams struct {
+	NameContainFilter   string
+	State               string
+	Health              string
+	LabelEqualFilters   map[string]string
+	LabelContainFilters map[string]string
+
+	Offset    int
+	Limit     int
+	SortField string
+	SortType  string
+}
+
+func (q *AlertingRuleQueryParams) Filter(rules []*GettableAlertingRule) []*GettableAlertingRule {
+	var ret []*GettableAlertingRule
+	for _, rule := range rules {
+		if rule == nil {
+			continue
+		}
+		if q == nil || q.matches(rule) {
+			ret = append(ret, rule)
+		}
+	}
+	return ret
+}
+
+func (q *AlertingRuleQueryParams) matches(rule *GettableAlertingRule) bool {
+	if q.NameContainFilter != "" && !strings.Contains(rule.Name, q.NameContainFilter) {
+		return false
+	}
+	if q.State != "" && q.State != rule.State {
+		return false
+	}
+	if q.Health != "" && q.Health != rule.Health {
+		return false
+	}
+	if len(rule.Labels) == 0 {
+		return len(q.LabelEqualFilters) == 0 && len(q.LabelContainFilters) == 0
+	}
+	for k, v := range q.LabelEqualFilters {
+		if fv, ok := rule.Labels[k]; !ok || fv != v {
+			return false
+		}
+	}
+	for k, v := range q.LabelContainFilters {
+		if fv, ok := rule.Labels[k]; !ok || !strings.Contains(fv, v) {
+			return false
+		}
+	}
+	return true
+}
+
+// AlertingRuleIdCompare defines the default order for the alerting rules.
+// For the alerting rule list, it guarantees a stable sort. For the custom alerting rules with possible same names
+// and the builtin alerting rules with possible same ids, it guarantees the stability of get operations.
+func AlertingRuleIdCompare(leftId, rightId string) bool {
+	// default to ascending order of id
+	return leftId <= rightId
+}
+
+func (q *AlertingRuleQueryParams) Sort(rules []*GettableAlertingRule) {
+	idCompare := func(left, right *GettableAlertingRule) bool {
+		return AlertingRuleIdCompare(left.Id, right.Id)
+	}
+	var compare = idCompare
+	if q != nil {
+		reverse := q.SortType == "desc"
+		switch q.SortField {
+		case "name":
+			compare = func(left, right *GettableAlertingRule) bool {
+				if c := strings.Compare(left.Name, right.Name); c != 0 {
+					if reverse {
+						return c > 0
+					}
+					return c < 0
+				}
+				return idCompare(left, right)
+			}
+		case "lastEvaluation":
+			compare = func(left, right *GettableAlertingRule) bool {
+				if left.LastEvaluation == nil {
+					if right.LastEvaluation != nil {
+						return false
+					}
+				} else {
+					if right.LastEvaluation == nil {
+						return true
+					} else if left.LastEvaluation.Equal(*right.LastEvaluation) {
+						if reverse {
+							return left.LastEvaluation.After(*right.LastEvaluation)
+						}
+						return left.LastEvaluation.Before(*right.LastEvaluation)
+					}
+				}
+				return idCompare(left, right)
+			}
+		case "evaluationTime":
+			compare = func(left, right *GettableAlertingRule) bool {
+				if left.EvaluationDurationSeconds != right.EvaluationDurationSeconds {
+					if reverse {
+						return left.EvaluationDurationSeconds > right.EvaluationDurationSeconds
+					}
+					return left.EvaluationDurationSeconds < right.EvaluationDurationSeconds
+				}
+				return idCompare(left, right)
+			}
+		}
+	}
+	sort.Slice(rules, func(i, j int) bool {
+		return compare(rules[i], rules[j])
+	})
+}
+
+func (q *AlertingRuleQueryParams) Sub(rules []*GettableAlertingRule) []*GettableAlertingRule {
+	start, stop := 0, 10
+	if q != nil {
+		start, stop = q.Offset, q.Offset+q.Limit
+	}
+	total := len(rules)
+	if start < total {
+		if stop > total {
+			stop = total
+		}
+		return rules[start:stop]
+	}
+	return nil
+}
+
+type AlertQueryParams struct {
+	State               string
+	LabelEqualFilters   map[string]string
+	LabelContainFilters map[string]string
+
+	Offset int
+	Limit  int
+}
+
+func (q *AlertQueryParams) Filter(alerts []*Alert) []*Alert {
+	var ret []*Alert
+	for _, alert := range alerts {
+		if alert == nil {
+			continue
+		}
+		if q == nil || q.matches(alert) {
+			ret = append(ret, alert)
+		}
+	}
+	return ret
+}
+
+func (q *AlertQueryParams) matches(alert *Alert) bool {
+	if q.State != "" && q.State != alert.State {
+		return false
+	}
+	if len(alert.Labels) == 0 {
+		return len(q.LabelEqualFilters) == 0 && len(q.LabelContainFilters) == 0
+	}
+	for k, v := range q.LabelEqualFilters {
+		if fv, ok := alert.Labels[k]; !ok || fv != v {
+			return false
+		}
+	}
+	for k, v := range q.LabelContainFilters {
+		if fv, ok := alert.Labels[k]; !ok || !strings.Contains(fv, v) {
+			return false
+		}
+	}
+	return true
+}
+
+func (q *AlertQueryParams) Sort(alerts []*Alert) {
+	compare := func(left, right *Alert) bool {
+		if left.ActiveAt == nil {
+			if right.ActiveAt != nil {
+				return false
+			}
+		} else {
+			if right.ActiveAt == nil {
+				return true
+			} else if !left.ActiveAt.Equal(*right.ActiveAt) {
+				return left.ActiveAt.After(*right.ActiveAt)
+			}
+		}
+		return prommodel.LabelsToSignature(left.Labels) <= prommodel.LabelsToSignature(right.Labels)
+	}
+	sort.Slice(alerts, func(i, j int) bool {
+		return compare(alerts[i], alerts[j])
+	})
+}
+
+func (q *AlertQueryParams) Sub(alerts []*Alert) []*Alert {
+	start, stop := 0, 10
+	if q != nil {
+		start, stop = q.Offset, q.Offset+q.Limit
+	}
+	total := len(alerts)
+	if start < total {
+		if stop > total {
+			stop = total
+		}
+		return alerts[start:stop]
+	}
+	return nil
+}
+
+func ParseAlertingRuleQueryParams(req *restful.Request) (*AlertingRuleQueryParams, error) {
+	var (
+		q   = &AlertingRuleQueryParams{}
+		err error
+	)
+
+	q.NameContainFilter = req.QueryParameter("name")
+	q.State = req.QueryParameter("state")
+	q.Health = req.QueryParameter("health")
+	q.Offset, _ = strconv.Atoi(req.QueryParameter("offset"))
+	q.Limit, err = strconv.Atoi(req.QueryParameter("limit"))
+	if err != nil {
+		q.Limit = 10
+		err = nil
+	}
+	q.LabelEqualFilters, q.LabelContainFilters = parseLabelFilters(req)
+	q.SortField = req.QueryParameter("sort_field")
+	q.SortType = req.QueryParameter("sort_type")
+	return q, err
+}
+
+func ParseAlertQueryParams(req *restful.Request) (*AlertQueryParams, error) {
+	var (
+		q   = &AlertQueryParams{}
+		err error
+	)
+
+	q.State = req.QueryParameter("state")
+	q.Offset, _ = strconv.Atoi(req.QueryParameter("offset"))
+	q.Limit, err = strconv.Atoi(req.QueryParameter("limit"))
+	if err != nil {
+		q.Limit = 10
+		err = nil
+	}
+	q.LabelEqualFilters, q.LabelContainFilters = parseLabelFilters(req)
+	return q, err
+}
+
+func parseLabelFilters(req *restful.Request) (map[string]string, map[string]string) {
+	var (
+		labelEqualFilters   = make(map[string]string)
+		labelContainFilters = make(map[string]string)
+		labelFiltersString  = req.QueryParameter("label_filters")
+	)
+	for _, filter := range strings.Split(labelFiltersString, ",") {
+		if i := strings.Index(filter, "="); i > 0 && len(filter) > i+1 {
+			labelEqualFilters[filter[:i]] = filter[i+1:]
+		} else if i := strings.Index(filter, "~"); i > 0 && len(filter) > i+1 {
+			labelContainFilters[filter[:i]] = filter[i+1:]
+		}
+	}
+	return labelEqualFilters, labelContainFilters
+}

pkg/apiserver/apiserver.go

@@ -51,6 +51,7 @@ import (
 	alertingv1 "kubesphere.io/kubesphere/pkg/kapis/alerting/v1"
 	clusterkapisv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/cluster/v1alpha1"
 	configv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/config/v1alpha2"
+	customalertingv1alpha1 "kubesphere.io/kubesphere/pkg/kapis/customalerting/v1alpha1"
 	devopsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha2"
 	devopsv1alpha3 "kubesphere.io/kubesphere/pkg/kapis/devops/v1alpha3"
 	iamapi "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2"
@@ -74,6 +75,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/models/resources/v1alpha3/user"
 	"kubesphere.io/kubesphere/pkg/simple/client/auditing"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops"
 	"kubesphere.io/kubesphere/pkg/simple/client/events"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
@@ -144,6 +146,8 @@ type APIServer struct {
 
 	AuditingClient auditing.Client
 
+	CustomAlertingClient customalerting.RuleClient
+
 	// controller-runtime cache
 	RuntimeCache runtimecache.Cache
 }
@@ -236,6 +240,8 @@ func (s *APIServer) installKubeSphereAPIs() {
 	urlruntime.Must(notificationv1.AddToContainer(s.container, s.Config.NotificationOptions.Endpoint))
 	urlruntime.Must(alertingv1.AddToContainer(s.container, s.Config.AlertingOptions.Endpoint))
 	urlruntime.Must(version.AddToContainer(s.container, s.KubernetesClient.Discovery()))
+	urlruntime.Must(customalertingv1alpha1.AddToContainer(s.container, s.InformerFactory,
+		s.KubernetesClient.Prometheus(), s.CustomAlertingClient, s.Config.CustomAlertingOptions))
 }
 
 func (s *APIServer) Run(stopCh <-chan struct{}) (err error) {
@@ -503,6 +509,26 @@ func (s *APIServer) waitForResourceSync(stopCh <-chan struct{}) error {
 	apiextensionsInformerFactory.Start(stopCh)
 	apiextensionsInformerFactory.WaitForCacheSync(stopCh)
 
+	if promFactory := s.InformerFactory.PrometheusSharedInformerFactory(); promFactory != nil {
+		prometheusGVRs := []schema.GroupVersionResource{
+			{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheuses"},
+			{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"},
+			{Group: "monitoring.coreos.com", Version: "v1", Resource: "thanosrulers"},
+		}
+		for _, gvr := range prometheusGVRs {
+			if isResourceExists(gvr) {
+				_, err = promFactory.ForResource(gvr)
+				if err != nil {
+					return err
+				}
+			} else {
+				klog.Warningf("resource %s not exists in the cluster", gvr)
+			}
+		}
+		promFactory.Start(stopCh)
+		promFactory.WaitForCacheSync(stopCh)
+	}
+
 	// controller runtime cache for resources
 	go s.RuntimeCache.Start(stopCh)
 	s.RuntimeCache.WaitForCacheSync(stopCh)

pkg/apiserver/auditing/types_test.go

@@ -56,7 +56,7 @@ func TestGetAuditLevel(t *testing.T) {
 
 	ksClient := fake.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	a := auditing{
 		webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
@@ -85,7 +85,7 @@ func TestAuditing_Enabled(t *testing.T) {
 
 	ksClient := fake.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	a := auditing{
 		webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
@@ -115,7 +115,7 @@ func TestAuditing_K8sAuditingEnabled(t *testing.T) {
 
 	ksClient := fake.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	a := auditing{
 		webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
@@ -145,7 +145,7 @@ func TestAuditing_LogRequestObject(t *testing.T) {
 
 	ksClient := fake.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	a := auditing{
 		webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),
@@ -236,7 +236,7 @@ func TestAuditing_LogResponseObject(t *testing.T) {
 
 	ksClient := fake.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	a := auditing{
 		webhookLister: fakeInformerFactory.KubeSphereSharedInformerFactory().Auditing().V1alpha1().Webhooks().Lister(),

pkg/apiserver/authorization/rbac/rbac_test.go

@@ -857,7 +857,7 @@ func newMockRBACAuthorizer(staticRoles *StaticRoles) (*RBACAuthorizer, error) {
 
 	ksClient := fakeks.NewSimpleClientset()
 	k8sClient := fakek8s.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, nil, nil, nil, nil)
 
 	k8sInformerFactory := fakeInformerFactory.KubernetesSharedInformerFactory()
 	ksInformerFactory := fakeInformerFactory.KubeSphereSharedInformerFactory()

pkg/apiserver/config/config.go

@@ -25,6 +25,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
 	auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
@@ -97,6 +98,7 @@ type Config struct {
 	AuditingOptions       *auditingclient.Options                    `json:"auditing,omitempty" yaml:"auditing,omitempty" mapstructure:"auditing"`
 	AlertingOptions       *alerting.Options                          `json:"alerting,omitempty" yaml:"alerting,omitempty" mapstructure:"alerting"`
 	NotificationOptions   *notification.Options                      `json:"notification,omitempty" yaml:"notification,omitempty" mapstructure:"notification"`
+	CustomAlertingOptions *customalerting.Options                    `json:"customalerting,omitempty" yaml:"customalerting,omitempty" mapstructure:"customalerting"`
 }
 
 // newConfig creates a default non-empty Config
@@ -120,6 +122,7 @@ func New() *Config {
 		MultiClusterOptions:   multicluster.NewOptions(),
 		EventsOptions:         eventsclient.NewElasticSearchOptions(),
 		AuditingOptions:       auditingclient.NewElasticSearchOptions(),
+		CustomAlertingOptions: customalerting.NewOptions(),
 	}
 }
 

pkg/apiserver/config/config_test.go

@@ -28,6 +28,7 @@ import (
 	"kubesphere.io/kubesphere/pkg/simple/client/alerting"
 	auditingclient "kubesphere.io/kubesphere/pkg/simple/client/auditing/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/cache"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
 	"kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins"
 	eventsclient "kubesphere.io/kubesphere/pkg/simple/client/events/elasticsearch"
 	"kubesphere.io/kubesphere/pkg/simple/client/k8s"
@@ -156,6 +157,11 @@ func newTestConfig() (*Config, error) {
 			IndexPrefix: "ks-logstash-auditing",
 			Version:     "6",
 		},
+		CustomAlertingOptions: &customalerting.Options{
+			PrometheusEndpoint:       "http://prometheus-operated.kubesphere-monitoring-system.svc",
+			ThanosRulerEndpoint:      "http://thanos-ruler-operated.kubesphere-monitoring-system.svc",
+			ThanosRuleResourceLabels: "thanosruler=thanos-ruler,role=thanos-alerting-rules",
+		},
 	}
 	return conf, nil
 }

pkg/constants/constants.go

@@ -104,6 +104,8 @@ const (
 	LogQueryTag      = "Log Query"
 	EventsQueryTag   = "Events Query"
 	AuditingQueryTag = "Auditing Query"
+
+	CustomAlertingTag = "Custom Alerting"
 )
 
 var (

pkg/informers/informers.go

@@ -19,6 +19,8 @@ package informers
 import (
 	snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned"
 	snapshotinformer "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions"
+	prominformers "github.com/prometheus-operator/prometheus-operator/pkg/client/informers/externalversions"
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	istioclient "istio.io/client-go/pkg/clientset/versioned"
 	istioinformers "istio.io/client-go/pkg/informers/externalversions"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -41,6 +43,7 @@ type InformerFactory interface {
 	IstioSharedInformerFactory() istioinformers.SharedInformerFactory
 	SnapshotSharedInformerFactory() snapshotinformer.SharedInformerFactory
 	ApiExtensionSharedInformerFactory() apiextensionsinformers.SharedInformerFactory
+	PrometheusSharedInformerFactory() prominformers.SharedInformerFactory
 
 	// Start shared informer factory one by one if they are not nil
 	Start(stopCh <-chan struct{})
@@ -52,10 +55,12 @@ type informerFactories struct {
 	istioInformerFactory         istioinformers.SharedInformerFactory
 	snapshotInformerFactory      snapshotinformer.SharedInformerFactory
 	apiextensionsInformerFactory apiextensionsinformers.SharedInformerFactory
+	prometheusInformerFactory    prominformers.SharedInformerFactory
 }
 
 func NewInformerFactories(client kubernetes.Interface, ksClient versioned.Interface, istioClient istioclient.Interface,
-	snapshotClient snapshotclient.Interface, apiextensionsClient apiextensionsclient.Interface) InformerFactory {
+	snapshotClient snapshotclient.Interface, apiextensionsClient apiextensionsclient.Interface,
+	prometheusClient promresourcesclient.Interface) InformerFactory {
 	factory := &informerFactories{}
 
 	if client != nil {
@@ -78,6 +83,10 @@ func NewInformerFactories(client kubernetes.Interface, ksClient versioned.Interf
 		factory.apiextensionsInformerFactory = apiextensionsinformers.NewSharedInformerFactory(apiextensionsClient, defaultResync)
 	}
 
+	if prometheusClient != nil {
+		factory.prometheusInformerFactory = prominformers.NewSharedInformerFactory(prometheusClient, defaultResync)
+	}
+
 	return factory
 }
 
@@ -101,6 +110,10 @@ func (f *informerFactories) ApiExtensionSharedInformerFactory() apiextensionsinf
 	return f.apiextensionsInformerFactory
 }
 
+func (f *informerFactories) PrometheusSharedInformerFactory() prominformers.SharedInformerFactory {
+	return f.prometheusInformerFactory
+}
+
 func (f *informerFactories) Start(stopCh <-chan struct{}) {
 	if f.informerFactory != nil {
 		f.informerFactory.Start(stopCh)
@@ -121,4 +134,8 @@ func (f *informerFactories) Start(stopCh <-chan struct{}) {
 	if f.apiextensionsInformerFactory != nil {
 		f.apiextensionsInformerFactory.Start(stopCh)
 	}
+
+	if f.prometheusInformerFactory != nil {
+		f.prometheusInformerFactory.Start(stopCh)
+	}
 }

pkg/informers/null_informers.go

@@ -17,6 +17,8 @@ limitations under the License.
 package informers
 
 import (
+	prominformers "github.com/prometheus-operator/prometheus-operator/pkg/client/informers/externalversions"
+	promfake "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/fake"
 	snapshotinformer "github.com/kubernetes-csi/external-snapshotter/client/v3/informers/externalversions"
 	istioinformers "istio.io/client-go/pkg/informers/externalversions"
 	apiextensionsinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
@@ -30,6 +32,7 @@ import (
 type nullInformerFactory struct {
 	fakeK8sInformerFactory informers.SharedInformerFactory
 	fakeKsInformerFactory  ksinformers.SharedInformerFactory
+	fakePrometheusFactory  prominformers.SharedInformerFactory
 }
 
 func NewNullInformerFactory() InformerFactory {
@@ -39,9 +42,13 @@ func NewNullInformerFactory() InformerFactory {
 	fakeKsClient := ksfake.NewSimpleClientset()
 	fakeKsInformerFactory := ksinformers.NewSharedInformerFactory(fakeKsClient, time.Minute*10)
 
+	fakePrometheusClient := promfake.NewSimpleClientset()
+	fakePrometheusFactory := prominformers.NewSharedInformerFactory(fakePrometheusClient, time.Minute*10)
+
 	return &nullInformerFactory{
 		fakeK8sInformerFactory: fakeInformerFactory,
 		fakeKsInformerFactory:  fakeKsInformerFactory,
+		fakePrometheusFactory:  fakePrometheusFactory,
 	}
 }
 
@@ -65,5 +72,9 @@ func (n nullInformerFactory) ApiExtensionSharedInformerFactory() apiextensionsin
 	return nil
 }
 
+func (n *nullInformerFactory) PrometheusSharedInformerFactory() prominformers.SharedInformerFactory {
+	return n.fakePrometheusFactory
+}
+
 func (n nullInformerFactory) Start(stopCh <-chan struct{}) {
 }

pkg/kapis/cluster/v1alpha1/handler_test.go

@@ -125,7 +125,7 @@ func TestGeranteAgentDeployment(t *testing.T) {
 	k8sclient := k8sfake.NewSimpleClientset(service)
 	ksclient := fake.NewSimpleClientset(cluster)
 
-	informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil)
+	informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil, nil)
 
 	informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service)
 	informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster)
@@ -233,7 +233,7 @@ func TestValidateKubeConfig(t *testing.T) {
 	k8sclient := k8sfake.NewSimpleClientset(service)
 	ksclient := fake.NewSimpleClientset(cluster)
 
-	informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil)
+	informersFactory := informers.NewInformerFactories(k8sclient, ksclient, nil, nil, nil, nil)
 
 	informersFactory.KubernetesSharedInformerFactory().Core().V1().Services().Informer().GetIndexer().Add(service)
 	informersFactory.KubeSphereSharedInformerFactory().Cluster().V1alpha1().Clusters().Informer().GetIndexer().Add(cluster)

pkg/kapis/customalerting/v1alpha1/handler.go

@@ -0,0 +1,285 @@
+/*
+Copyright 2020 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	"github.com/emicklei/go-restful"
+	"k8s.io/klog"
+	ksapi "kubesphere.io/kubesphere/pkg/api"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/informers"
+	customalertingmodels "kubesphere.io/kubesphere/pkg/models/customalerting"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+type handler struct {
+	operator customalertingmodels.Operator
+}
+
+func newHandler(informers informers.InformerFactory,
+	promResourceClient promresourcesclient.Interface, ruleClient customalerting.RuleClient,
+	option *customalerting.Options) *handler {
+	return &handler{
+		operator: customalertingmodels.NewOperator(
+			informers, promResourceClient, ruleClient, option),
+	}
+}
+
+func (h *handler) handleListCustomAlertingRules(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	query, err := v1alpha1.ParseAlertingRuleQueryParams(req)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	rules, err := h.operator.ListCustomAlertingRules(req.Request.Context(), namespace, query)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+	resp.WriteEntity(rules)
+}
+
+func (h *handler) handleListCustomRulesAlerts(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	query, err := v1alpha1.ParseAlertQueryParams(req)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	alerts, err := h.operator.ListCustomRulesAlerts(req.Request.Context(), namespace, query)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+	resp.WriteEntity(alerts)
+}
+
+func (h *handler) handleGetCustomAlertingRule(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	ruleName := req.PathParameter("rule_name")
+
+	rule, err := h.operator.GetCustomAlertingRule(req.Request.Context(), namespace, ruleName)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+	if rule == nil {
+		ksapi.HandleNotFound(resp, nil, err)
+		return
+	}
+	resp.WriteEntity(rule)
+}
+
+func (h *handler) handleListCustomSpecifiedRuleAlerts(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	ruleName := req.PathParameter("rule_name")
+
+	alerts, err := h.operator.ListCustomSpecifiedRuleAlerts(req.Request.Context(), namespace, ruleName)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+	resp.WriteEntity(alerts)
+}
+
+func (h *handler) handleCreateCustomAlertingRule(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+
+	var rule v1alpha1.PostableAlertingRule
+	if err := req.ReadEntity(&rule); err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+	if err := rule.Validate(); err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	err := h.operator.CreateCustomAlertingRule(req.Request.Context(), namespace, &rule)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		case err == v1alpha1.ErrAlertingRuleAlreadyExists:
+			ksapi.HandleConflict(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+}
+
+func (h *handler) handleUpdateCustomAlertingRule(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	ruleName := req.PathParameter("rule_name")
+
+	var rule v1alpha1.PostableAlertingRule
+	if err := req.ReadEntity(&rule); err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+	if err := rule.Validate(); err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	err := h.operator.UpdateCustomAlertingRule(req.Request.Context(), namespace, ruleName, &rule)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+}
+
+func (h *handler) handleDeleteCustomAlertingRule(req *restful.Request, resp *restful.Response) {
+	namespace := req.PathParameter("namespace")
+	name := req.PathParameter("rule_name")
+
+	err := h.operator.DeleteCustomAlertingRule(req.Request.Context(), namespace, name)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrThanosRulerNotEnabled:
+			ksapi.HandleBadRequest(resp, nil, err)
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+}
+
+func (h *handler) handleListBuiltinAlertingRules(req *restful.Request, resp *restful.Response) {
+	query, err := v1alpha1.ParseAlertingRuleQueryParams(req)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	rules, err := h.operator.ListBuiltinAlertingRules(req.Request.Context(), query)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleInternalError(resp, nil, err)
+		return
+	}
+	resp.WriteEntity(rules)
+}
+
+func (h *handler) handleListBuiltinRulesAlerts(req *restful.Request, resp *restful.Response) {
+	query, err := v1alpha1.ParseAlertQueryParams(req)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleBadRequest(resp, nil, err)
+		return
+	}
+
+	alerts, err := h.operator.ListBuiltinRulesAlerts(req.Request.Context(), query)
+	if err != nil {
+		klog.Error(err)
+		ksapi.HandleInternalError(resp, nil, err)
+		return
+	}
+	resp.WriteEntity(alerts)
+}
+
+func (h *handler) handleGetBuiltinAlertingRule(req *restful.Request, resp *restful.Response) {
+	ruleId := req.PathParameter("rule_id")
+
+	rule, err := h.operator.GetBuiltinAlertingRule(req.Request.Context(), ruleId)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+	if rule == nil {
+		ksapi.HandleNotFound(resp, nil, err)
+		return
+	}
+
+	resp.WriteEntity(rule)
+}
+
+func (h *handler) handleListBuiltinSpecifiedRuleAlerts(req *restful.Request, resp *restful.Response) {
+	ruleId := req.PathParameter("rule_id")
+
+	alerts, err := h.operator.ListBuiltinSpecifiedRuleAlerts(req.Request.Context(), ruleId)
+	if err != nil {
+		klog.Error(err)
+		switch {
+		case err == v1alpha1.ErrAlertingRuleNotFound:
+			ksapi.HandleNotFound(resp, nil, err)
+		default:
+			ksapi.HandleInternalError(resp, nil, err)
+		}
+		return
+	}
+
+	resp.WriteEntity(alerts)
+}

pkg/kapis/customalerting/v1alpha1/register.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2020 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"net/http"
+
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	"github.com/emicklei/go-restful"
+	restfulspec "github.com/emicklei/go-restful-openapi"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	ksapi "kubesphere.io/kubesphere/pkg/api"
+	customalertingv1alpha1 "kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+const (
+	groupName = "custom.alerting.kubesphere.io"
+)
+
+var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha1"}
+
+func AddToContainer(container *restful.Container, informers informers.InformerFactory,
+	promResourceClient promresourcesclient.Interface, ruleClient customalerting.RuleClient,
+	option *customalerting.Options) error {
+
+	handler := newHandler(informers, promResourceClient, ruleClient, option)
+
+	ws := runtime.NewWebService(GroupVersion)
+
+	ws.Route(ws.GET("/rules").
+		To(handler.handleListCustomAlertingRules).
+		Doc("list the cluster-level custom alerting rules").
+		Param(ws.QueryParameter("name", "rule name")).
+		Param(ws.QueryParameter("state", "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("health", "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("sort_field", "sort field, one of `name`, `lastEvaluation`, `evaluationTime`")).
+		Param(ws.QueryParameter("sort_type", "sort type, one of `asc`, `desc`")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRuleList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/alerts").
+		To(handler.handleListCustomRulesAlerts).
+		Doc("list the alerts of the cluster-level custom alerting rules").
+		Param(ws.QueryParameter("state", "state, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.AlertList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/rules/{rule_name}").
+		To(handler.handleGetCustomAlertingRule).
+		Doc("get the cluster-level custom alerting rule with the specified name").
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRule{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/rules/{rule_name}/alerts").
+		To(handler.handleListCustomSpecifiedRuleAlerts).
+		Doc("list the alerts of the cluster-level custom alerting rule with the specified name").
+		Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.POST("/rules").
+		To(handler.handleCreateCustomAlertingRule).
+		Doc("create a cluster-level custom alerting rule").
+		Reads(customalertingv1alpha1.PostableAlertingRule{}).
+		Returns(http.StatusOK, ksapi.StatusOK, nil).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.PUT("/rules/{rule_name}").
+		To(handler.handleUpdateCustomAlertingRule).
+		Doc("update the cluster-level custom alerting rule with the specified name").
+		Reads(customalertingv1alpha1.PostableAlertingRule{}).
+		Returns(http.StatusOK, ksapi.StatusOK, nil).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.DELETE("/rules/{rule_name}").
+		To(handler.handleDeleteCustomAlertingRule).
+		Doc("delete the cluster-level custom alerting rule with the specified name").
+		Returns(http.StatusOK, ksapi.StatusOK, nil).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/rules").
+		To(handler.handleListCustomAlertingRules).
+		Doc("list the custom alerting rules in the specified namespace").
+		Param(ws.QueryParameter("name", "rule name")).
+		Param(ws.QueryParameter("state", "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("health", "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("sort_field", "sort field, one of `name`, `lastEvaluation`, `evaluationTime`")).
+		Param(ws.QueryParameter("sort_type", "sort type, one of `asc`, `desc`")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRuleList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/alerts").
+		To(handler.handleListCustomRulesAlerts).
+		Doc("list the alerts of the custom alerting rules in the specified namespace.").
+		Param(ws.QueryParameter("state", "state, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.AlertList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/rules/{rule_name}").
+		To(handler.handleGetCustomAlertingRule).
+		Doc("get the custom alerting rule with the specified name in the specified namespace").
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRule{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/namespaces/{namespace}/rules/{rule_name}/alerts").
+		To(handler.handleListCustomSpecifiedRuleAlerts).
+		Doc("get the alerts of the custom alerting rule with the specified name in the specified namespace").
+		Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.POST("/namespaces/{namespace}/rules").
+		To(handler.handleCreateCustomAlertingRule).
+		Doc("create a custom alerting rule in the specified namespace").
+		Reads(customalertingv1alpha1.PostableAlertingRule{}).
+		Returns(http.StatusOK, ksapi.StatusOK, "").
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.PUT("/namespaces/{namespace}/rules/{rule_name}").
+		To(handler.handleUpdateCustomAlertingRule).
+		Doc("update the custom alerting rule with the specified name in the specified namespace").
+		Reads(customalertingv1alpha1.PostableAlertingRule{}).
+		Returns(http.StatusOK, ksapi.StatusOK, "").
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.DELETE("/namespaces/{namespace}/rules/{rule_name}").
+		To(handler.handleDeleteCustomAlertingRule).
+		Doc("delete the custom alerting rule with the specified rule name in the specified namespace").
+		Returns(http.StatusOK, ksapi.StatusOK, nil).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/builtin/rules").
+		To(handler.handleListBuiltinAlertingRules).
+		Doc("list the builtin(non-custom) alerting rules").
+		Param(ws.QueryParameter("name", "rule name")).
+		Param(ws.QueryParameter("state", "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("health", "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("sort_field", "sort field, one of `name`, `lastEvaluation`, `evaluationTime`")).
+		Param(ws.QueryParameter("sort_type", "sort type, one of `asc`, `desc`")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRuleList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/builtin/alerts").
+		To(handler.handleListBuiltinRulesAlerts).
+		Doc("list the alerts of the builtin(non-custom) rules").
+		Param(ws.QueryParameter("state", "state, one of `firing`, `pending`, `inactive`")).
+		Param(ws.QueryParameter("label_filters", "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
+		Param(ws.QueryParameter("offset", "offset of the result set").DataType("integer").DefaultValue("0")).
+		Param(ws.QueryParameter("limit", "limit size of the result set").DataType("integer").DefaultValue("10")).
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.AlertList{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/builtin/rules/{rule_id}").
+		To(handler.handleGetBuiltinAlertingRule).
+		Doc("get the builtin(non-custom) alerting rule with specified id").
+		Returns(http.StatusOK, ksapi.StatusOK, customalertingv1alpha1.GettableAlertingRule{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	ws.Route(ws.GET("/builtin/rules/{rule_id}/alerts").
+		To(handler.handleListBuiltinSpecifiedRuleAlerts).
+		Doc("list the alerts of the builtin(non-custom) alerting rule with the specified id").
+		Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
+		Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
+
+	container.Add(ws)
+
+	return nil
+}

pkg/kapis/customalerting/v1alpha1/test_test.go

@@ -0,0 +1,147 @@
+package v1alpha1
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/emicklei/go-restful"
+	restfulspec "github.com/emicklei/go-restful-openapi"
+	"github.com/go-openapi/spec"
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	"github.com/prometheus/prometheus/promql/parser"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"kubesphere.io/kubesphere/pkg/apiserver/runtime"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+func TestTest(t *testing.T) {
+	expr := `increase((max by(job) (etcd_server_leader_changes_seen_total{job=~".*etcd.*"}) or 0 * absent(etcd_server_leader_changes_seen_total{job=~".*etcd.*"}))[15m:1m]) >= 3`
+	//expr = `absent_over_time(sum(nonexistent{job="myjob"})[1h:])`
+
+	pexpr, err := parser.ParseExpr(expr)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Println(pexpr.String())
+}
+
+func TestApis(t *testing.T) {
+	kubeconfig := "D:/ks/conf/ks3-config"
+
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	if err != nil {
+		panic(err)
+	}
+
+	k8sClient := kubernetes.NewForConfigOrDie(config)
+	promResourcesClient := promresourcesclient.NewForConfigOrDie(config)
+
+	option := &customalerting.Options{
+		PrometheusEndpoint: "http://139.198.112.79:39090/",
+		ThanosRulerEndpoint: "http://139.198.112.79:39091/",
+		ThanosRuleResourceLabels: "role=thanos-alerting-rules,thanosruler:thanos-ruler",
+	}
+	ruleClient, err := customalerting.NewRuleClient(option)
+	if err != nil {
+		panic(err)
+	}
+
+	stopCh := make(chan struct{})
+
+	informerFactory := informers.NewInformerFactories(k8sClient, nil, nil, nil, nil, promResourcesClient)
+	k8sGVRs := []schema.GroupVersionResource{
+		{Group: "", Version: "v1", Resource: "namespaces"},
+	}
+	for _, gvr := range k8sGVRs {
+		_, err = informerFactory.KubernetesSharedInformerFactory().ForResource(gvr)
+		if err != nil {
+			panic(err)
+		}
+	}
+	prometheusGVRs := []schema.GroupVersionResource{
+		{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheuses"},
+		{Group: "monitoring.coreos.com", Version: "v1", Resource: "prometheusrules"},
+		{Group: "monitoring.coreos.com", Version: "v1", Resource: "thanosrulers"},
+	}
+	for _, gvr := range prometheusGVRs {
+		_, err = informerFactory.PrometheusSharedInformerFactory().ForResource(gvr)
+		if err != nil {
+			panic(err)
+		}
+	}
+	informerFactory.Start(stopCh)
+	informerFactory.KubernetesSharedInformerFactory().WaitForCacheSync(stopCh)
+	informerFactory.PrometheusSharedInformerFactory().WaitForCacheSync(stopCh)
+	informerFactory.Start(stopCh)
+
+	container := restful.NewContainer()
+	AddToContainer(container, informerFactory, promResourcesClient, ruleClient, option)
+	server := &http.Server{}
+	server.Handler = container
+	if err := server.ListenAndServe(); err != nil {
+		panic(err)
+	}
+}
+
+func TestGenSwaggerJson(t *testing.T) {
+	container := runtime.Container
+
+	informerFactory := informers.NewNullInformerFactory()
+
+	AddToContainer(container, informerFactory, nil, nil, nil)
+
+	swagger := restfulspec.BuildSwagger(restfulspec.Config{
+		WebServices:                   container.RegisteredWebServices(),
+		PostBuildSwaggerObjectHandler: enrichSwaggerObject,
+	})
+
+	swagger.Info.Extensions = make(spec.Extensions)
+	swagger.Info.Extensions.Add("x-tagGroups", []struct {
+		Name string   `json:"name"`
+		Tags []string `json:"tags"`
+	}{
+		{
+			Name: "Custom Alerting",
+			Tags: []string{constants.CustomAlertingTag},
+		},
+	})
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "\t")
+	enc.Encode(swagger)
+}
+
+func enrichSwaggerObject(swo *spec.Swagger) {
+	swo.Info = &spec.Info{
+		InfoProps: spec.InfoProps{
+			Title:       "KubeSphere",
+			Description: "KubeSphere OpenAPI",
+			Contact: &spec.ContactInfo{
+				ContactInfoProps: spec.ContactInfoProps{
+					Name:  "KubeSphere",
+					URL:   "https://kubesphere.io/",
+					Email: "kubesphere@yunify.com",
+				},
+			},
+			License: &spec.License{
+				LicenseProps: spec.LicenseProps{
+					Name: "Apache 2.0",
+					URL:  "https://www.apache.org/licenses/LICENSE-2.0.html",
+				},
+			},
+			Version: "0.1.0",
+		}}
+
+	// setup security definitions
+	swo.SecurityDefinitions = map[string]*spec.SecurityScheme{
+		"jwt": spec.APIKeyAuth("Authorization", "header"),
+	}
+	swo.Security = []map[string][]string{{"jwt": []string{}}}
+}

pkg/kapis/monitoring/v1alpha3/helper_test.go

@@ -216,7 +216,7 @@ func TestParseRequestParams(t *testing.T) {
 	for i, tt := range tests {
 		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
 			client := fake.NewSimpleClientset(&tt.namespace)
-			fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil)
+			fakeInformerFactory := informers.NewInformerFactories(client, nil, nil, nil, nil, nil)
 			handler := newHandler(client, nil, fakeInformerFactory, nil)
 
 			result, err := handler.makeQueryOptions(tt.params, tt.lvl)

pkg/kapis/resources/v1alpha3/handler_test.go

@@ -186,7 +186,7 @@ func prepare() (informers.InformerFactory, error) {
 	snapshotClient := fakesnapshot.NewSimpleClientset()
 	apiextensionsClient := fakeapiextensions.NewSimpleClientset()
 
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, apiextensionsClient)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, apiextensionsClient, nil)
 
 	k8sInformerFactory := fakeInformerFactory.KubernetesSharedInformerFactory()
 

pkg/models/customalerting/customalerting.go

@@ -0,0 +1,632 @@
+package customalerting
+
+import (
+	"context"
+	"sort"
+	"strings"
+
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	prominformersv1 "github.com/prometheus-operator/prometheus-operator/pkg/client/informers/externalversions/monitoring/v1"
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	coreinformersv1 "k8s.io/client-go/informers/core/v1"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/constants"
+	"kubesphere.io/kubesphere/pkg/informers"
+	"kubesphere.io/kubesphere/pkg/models/customalerting/rules"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+const (
+	rulerNamespace                  = constants.KubeSphereMonitoringNamespace
+	customRuleGroupDefault          = "alerting.custom.defaults"
+	customRuleResourceLabelKeyLevel = "custom-alerting-rule-level"
+)
+
+var (
+	maxSecretSize        = corev1.MaxSecretSize
+	maxConfigMapDataSize = int(float64(maxSecretSize) * 0.45)
+)
+
+// Operator contains all operations to alerting rules. The operations may involve manipulations of prometheusrule
+// custom resources where the rules are persisted, and querying the rules state from prometheus endpoint and
+// thanos ruler endpoint.
+// For the following apis, if namespace is empty, do operations to alerting rules with cluster level,
+// or do operations only to rules of the specified namespaces.
+// All custom rules will be configured for thanos ruler, so the operations to custom alerting rule can not be done
+// if thanos ruler is not enabled.
+type Operator interface {
+	// ListCustomAlertingRules lists the custom alerting rules.
+	ListCustomAlertingRules(ctx context.Context, namespace string,
+		queryParams *v1alpha1.AlertingRuleQueryParams) (*v1alpha1.GettableAlertingRuleList, error)
+	// ListCustomRulesAlerts lists the alerts of the custom alerting rules.
+	ListCustomRulesAlerts(ctx context.Context, namespace string,
+		queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
+	// GetCustomAlertingRule gets the custom alerting rule with the given name.
+	GetCustomAlertingRule(ctx context.Context, namespace, ruleName string) (*v1alpha1.GettableAlertingRule, error)
+	// ListCustomSpecifiedRuleAlerts lists the alerts of the custom alerting rule with the given name.
+	ListCustomSpecifiedRuleAlerts(ctx context.Context, namespace, ruleName string) ([]*v1alpha1.Alert, error)
+	// CreateCustomAlertingRule creates a custom alerting rule.
+	CreateCustomAlertingRule(ctx context.Context, namespace string, rule *v1alpha1.PostableAlertingRule) error
+	// UpdateCustomAlertingRule updates the custom alerting rule with the given name.
+	UpdateCustomAlertingRule(ctx context.Context, namespace, ruleName string, rule *v1alpha1.PostableAlertingRule) error
+	// DeleteCustomAlertingRule deletes the custom alerting rule with the given name.
+	DeleteCustomAlertingRule(ctx context.Context, namespace, ruleName string) error
+
+	// ListBuiltinAlertingRules lists the builtin(non-custom) alerting rules
+	ListBuiltinAlertingRules(ctx context.Context,
+		queryParams *v1alpha1.AlertingRuleQueryParams) (*v1alpha1.GettableAlertingRuleList, error)
+	// ListBuiltinRulesAlerts lists the alerts of the builtin(non-custom) alerting rules
+	ListBuiltinRulesAlerts(ctx context.Context,
+		queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
+	// GetBuiltinAlertingRule gets the builtin(non-custom) alerting rule with the given id
+	GetBuiltinAlertingRule(ctx context.Context, ruleId string) (*v1alpha1.GettableAlertingRule, error)
+	// ListBuiltinSpecifiedRuleAlerts lists the alerts of the builtin(non-custom) alerting rule with the given id
+	ListBuiltinSpecifiedRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error)
+}
+
+func NewOperator(informers informers.InformerFactory,
+	promResourceClient promresourcesclient.Interface, ruleClient customalerting.RuleClient,
+	option *customalerting.Options) Operator {
+	o := operator{
+		namespaceInformer: informers.KubernetesSharedInformerFactory().Core().V1().Namespaces(),
+
+		promResourceClient: promResourceClient,
+
+		prometheusInformer:   informers.PrometheusSharedInformerFactory().Monitoring().V1().Prometheuses(),
+		thanosRulerInformer:  informers.PrometheusSharedInformerFactory().Monitoring().V1().ThanosRulers(),
+		ruleResourceInformer: informers.PrometheusSharedInformerFactory().Monitoring().V1().PrometheusRules(),
+
+		ruleClient: ruleClient,
+
+		thanosRuleResourceLabels: make(map[string]string),
+	}
+
+	o.resourceRuleCache = rules.NewRuleCache(o.ruleResourceInformer)
+
+	if option != nil && len(option.ThanosRuleResourceLabels) != 0 {
+		lblStrings := strings.Split(option.ThanosRuleResourceLabels, ",")
+		for _, lblString := range lblStrings {
+			lbl := strings.Split(lblString, "=")
+			if len(lbl) == 2 {
+				o.thanosRuleResourceLabels[lbl[0]] = lbl[1]
+			}
+		}
+	}
+
+	return &o
+}
+
+type operator struct {
+	ruleClient customalerting.RuleClient
+
+	promResourceClient promresourcesclient.Interface
+
+	prometheusInformer   prominformersv1.PrometheusInformer
+	thanosRulerInformer  prominformersv1.ThanosRulerInformer
+	ruleResourceInformer prominformersv1.PrometheusRuleInformer
+
+	namespaceInformer coreinformersv1.NamespaceInformer
+
+	resourceRuleCache *rules.RuleCache
+
+	thanosRuleResourceLabels map[string]string
+}
+
+func (o *operator) ListCustomAlertingRules(ctx context.Context, namespace string,
+	queryParams *v1alpha1.AlertingRuleQueryParams) (*v1alpha1.GettableAlertingRuleList, error) {
+
+	var level v1alpha1.RuleLevel
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+	}
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	alertingRules, err := o.listCustomAlertingRules(ctx, ruleNamespace, level)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlertingRules(alertingRules, queryParams), nil
+}
+
+func (o *operator) ListCustomRulesAlerts(ctx context.Context, namespace string,
+	queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error) {
+
+	var level v1alpha1.RuleLevel
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+	}
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	alertingRules, err := o.listCustomAlertingRules(ctx, ruleNamespace, level)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlerts(alertingRules, queryParams), nil
+}
+
+func (o *operator) GetCustomAlertingRule(ctx context.Context, namespace, ruleName string) (
+	*v1alpha1.GettableAlertingRule, error) {
+
+	var level v1alpha1.RuleLevel
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+	}
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	return o.getCustomAlertingRule(ctx, ruleNamespace, ruleName, level)
+}
+
+func (o *operator) ListCustomSpecifiedRuleAlerts(ctx context.Context, namespace, ruleName string) (
+	[]*v1alpha1.Alert, error) {
+
+	rule, err := o.GetCustomAlertingRule(ctx, namespace, ruleName)
+	if err != nil {
+		return nil, err
+	}
+	if rule == nil {
+		return nil, v1alpha1.ErrAlertingRuleNotFound
+	}
+	return rule.Alerts, nil
+}
+
+func (o *operator) ListBuiltinAlertingRules(ctx context.Context,
+	queryParams *v1alpha1.AlertingRuleQueryParams) (*v1alpha1.GettableAlertingRuleList, error) {
+
+	alertingRules, err := o.listBuiltinAlertingRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlertingRules(alertingRules, queryParams), nil
+}
+
+func (o *operator) ListBuiltinRulesAlerts(ctx context.Context,
+	queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error) {
+	alertingRules, err := o.listBuiltinAlertingRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlerts(alertingRules, queryParams), nil
+}
+
+func (o *operator) GetBuiltinAlertingRule(ctx context.Context, ruleId string) (
+	*v1alpha1.GettableAlertingRule, error) {
+
+	return o.getBuiltinAlertingRule(ctx, ruleId)
+}
+
+func (o *operator) ListBuiltinSpecifiedRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error) {
+	rule, err := o.getBuiltinAlertingRule(ctx, ruleId)
+	if err != nil {
+		return nil, err
+	}
+	if rule == nil {
+		return nil, v1alpha1.ErrAlertingRuleNotFound
+	}
+	return rule.Alerts, nil
+}
+
+func (o *operator) ListClusterAlertingRules(ctx context.Context, customFlag string,
+	queryParams *v1alpha1.AlertingRuleQueryParams) (*v1alpha1.GettableAlertingRuleList, error) {
+
+	namespace := rulerNamespace
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	alertingRules, err := o.listCustomAlertingRules(ctx, ruleNamespace, v1alpha1.RuleLevelCluster)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlertingRules(alertingRules, queryParams), nil
+}
+
+func (o *operator) ListClusterRulesAlerts(ctx context.Context,
+	queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error) {
+
+	namespace := rulerNamespace
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	alertingRules, err := o.listCustomAlertingRules(ctx, ruleNamespace, v1alpha1.RuleLevelCluster)
+	if err != nil {
+		return nil, err
+	}
+
+	return pageAlerts(alertingRules, queryParams), nil
+}
+
+func (o *operator) listCustomAlertingRules(ctx context.Context, ruleNamespace *corev1.Namespace,
+	level v1alpha1.RuleLevel) ([]*v1alpha1.GettableAlertingRule, error) {
+
+	ruler, err := o.getThanosRuler()
+	if err != nil {
+		return nil, err
+	}
+	if ruler == nil {
+		return nil, v1alpha1.ErrThanosRulerNotEnabled
+	}
+
+	resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace,
+		labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}))
+	if err != nil {
+		return nil, err
+	}
+
+	ruleGroups, err := o.ruleClient.ThanosRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return rules.MixAlertingRules(ruleNamespace.Name, &rules.ResourceRuleChunk{
+		ResourceRulesMap: resourceRulesMap,
+		Custom:           true,
+		Level:            level,
+	}, ruleGroups, ruler.ExternalLabels())
+}
+
+func (o *operator) getCustomAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	ruleName string, level v1alpha1.RuleLevel) (*v1alpha1.GettableAlertingRule, error) {
+
+	ruler, err := o.getThanosRuler()
+	if err != nil {
+		return nil, err
+	}
+	if ruler == nil {
+		return nil, v1alpha1.ErrThanosRulerNotEnabled
+	}
+
+	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace,
+		labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}), ruleName)
+	if err != nil {
+		return nil, err
+	}
+	if resourceRule == nil {
+		return nil, v1alpha1.ErrAlertingRuleNotFound
+	}
+
+	ruleGroups, err := o.ruleClient.ThanosRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	return rules.MixAlertingRule(ruleNamespace.Name, &rules.ResourceRuleSole{
+		ResourceRule: *resourceRule,
+		Custom:       true,
+		Level:        level,
+	}, ruleGroups, ruler.ExternalLabels())
+}
+
+func (o *operator) listBuiltinAlertingRules(ctx context.Context) (
+	[]*v1alpha1.GettableAlertingRule, error) {
+
+	ruler, err := o.getPrometheusRuler()
+	if err != nil {
+		return nil, err
+	}
+
+	ruleGroups, err := o.ruleClient.PrometheusRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if ruler == nil {
+		// for out-cluster prometheus
+		return rules.ParseAlertingRules(ruleGroups, false, v1alpha1.RuleLevelCluster,
+			func(group, id string, rule *customalerting.AlertingRule) bool {
+				return true
+			})
+	}
+
+	namespace := rulerNamespace
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return rules.MixAlertingRules(ruleNamespace.Name, &rules.ResourceRuleChunk{
+		ResourceRulesMap: resourceRulesMap,
+		Custom:           false,
+		Level:            v1alpha1.RuleLevelCluster,
+	}, ruleGroups, ruler.ExternalLabels())
+}
+
+func (o *operator) getBuiltinAlertingRule(ctx context.Context, ruleId string) (*v1alpha1.GettableAlertingRule, error) {
+
+	ruler, err := o.getPrometheusRuler()
+	if err != nil {
+		return nil, err
+	}
+
+	ruleGroups, err := o.ruleClient.PrometheusRules(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	if ruler == nil {
+		// for out-cluster prometheus
+		alertingRules, err := rules.ParseAlertingRules(ruleGroups, false, v1alpha1.RuleLevelCluster,
+			func(group, id string, rule *customalerting.AlertingRule) bool {
+				return ruleId == id
+			})
+		if err != nil {
+			return nil, err
+		}
+		if len(alertingRules) == 0 {
+			return nil, v1alpha1.ErrAlertingRuleNotFound
+		}
+		sort.Slice(alertingRules, func(i, j int) bool {
+			return v1alpha1.AlertingRuleIdCompare(alertingRules[i].Id, alertingRules[j].Id)
+		})
+		return alertingRules[0], nil
+	}
+
+	namespace := rulerNamespace
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return nil, err
+	}
+
+	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, nil, ruleId)
+	if err != nil {
+		return nil, err
+	}
+
+	if resourceRule == nil {
+		return nil, v1alpha1.ErrAlertingRuleNotFound
+	}
+
+	return rules.MixAlertingRule(ruleNamespace.Name, &rules.ResourceRuleSole{
+		ResourceRule: *resourceRule,
+		Custom:       false,
+		Level:        v1alpha1.RuleLevelCluster,
+	}, ruleGroups, ruler.ExternalLabels())
+}
+
+func (o *operator) CreateCustomAlertingRule(ctx context.Context, namespace string,
+	rule *v1alpha1.PostableAlertingRule) error {
+	ruler, err := o.getThanosRuler()
+	if err != nil {
+		return err
+	}
+	if ruler == nil {
+		return v1alpha1.ErrThanosRulerNotEnabled
+	}
+
+	var (
+		level              v1alpha1.RuleLevel
+		ruleResourceLabels = make(map[string]string)
+	)
+	for k, v := range o.thanosRuleResourceLabels {
+		ruleResourceLabels[k] = v
+	}
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+		expr, err := rules.InjectExprNamespaceLabel(rule.Query, namespace)
+		if err != nil {
+			return err
+		}
+		rule.Query = expr
+	}
+	ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return err
+	}
+
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
+	if err != nil {
+		return err
+	}
+	if resourceRule != nil {
+		return v1alpha1.ErrAlertingRuleAlreadyExists
+	}
+
+	return ruler.AddAlertingRule(ctx, ruleNamespace, extraRuleResourceSelector,
+		customRuleGroupDefault, parseToPrometheusRule(rule), ruleResourceLabels)
+}
+
+func (o *operator) UpdateCustomAlertingRule(ctx context.Context, namespace, name string,
+	rule *v1alpha1.PostableAlertingRule) error {
+
+	rule.Name = name
+
+	ruler, err := o.getThanosRuler()
+	if err != nil {
+		return err
+	}
+	if ruler == nil {
+		return v1alpha1.ErrThanosRulerNotEnabled
+	}
+
+	var (
+		level              v1alpha1.RuleLevel
+		ruleResourceLabels = make(map[string]string)
+	)
+	for k, v := range o.thanosRuleResourceLabels {
+		ruleResourceLabels[k] = v
+	}
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+		expr, err := rules.InjectExprNamespaceLabel(rule.Query, namespace)
+		if err != nil {
+			return err
+		}
+		rule.Query = expr
+	}
+	ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return err
+	}
+
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
+	if err != nil {
+		return err
+	}
+	if resourceRule == nil {
+		return v1alpha1.ErrAlertingRuleNotFound
+	}
+
+	return ruler.UpdateAlertingRule(ctx, ruleNamespace, extraRuleResourceSelector,
+		resourceRule.Group, parseToPrometheusRule(rule), ruleResourceLabels)
+}
+
+func (o *operator) DeleteCustomAlertingRule(ctx context.Context, namespace, name string) error {
+	ruler, err := o.getThanosRuler()
+	if err != nil {
+		return err
+	}
+	if ruler == nil {
+		return v1alpha1.ErrThanosRulerNotEnabled
+	}
+
+	var (
+		level v1alpha1.RuleLevel
+	)
+	if namespace == "" {
+		namespace = rulerNamespace
+		level = v1alpha1.RuleLevelCluster
+	} else {
+		level = v1alpha1.RuleLevelNamespace
+	}
+
+	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
+	if err != nil {
+		return err
+	}
+
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, name)
+	if err != nil {
+		return err
+	}
+	if resourceRule == nil {
+		return v1alpha1.ErrAlertingRuleNotFound
+	}
+
+	return ruler.DeleteAlertingRule(ctx, ruleNamespace, extraRuleResourceSelector, resourceRule.Group, name)
+}
+
+// getPrometheusRuler gets the cluster-in prometheus
+func (o *operator) getPrometheusRuler() (rules.Ruler, error) {
+	prometheuses, err := o.prometheusInformer.Lister().Prometheuses(rulerNamespace).List(labels.Everything())
+	if err != nil {
+		return nil, errors.Wrap(err, "error listing prometheuses")
+	}
+	if len(prometheuses) > 1 {
+		// it is not supported temporarily to have multiple prometheuses in the monitoring namespace
+		return nil, errors.Errorf(
+			"there is more than one prometheus custom resource in %s", rulerNamespace)
+	}
+	if len(prometheuses) == 0 {
+		return nil, nil
+	}
+
+	return rules.NewPrometheusRuler(prometheuses[0], o.ruleResourceInformer, o.promResourceClient), nil
+}
+
+func (o *operator) getThanosRuler() (rules.Ruler, error) {
+	thanosrulers, err := o.thanosRulerInformer.Lister().ThanosRulers(rulerNamespace).List(labels.Everything())
+	if err != nil {
+		return nil, errors.Wrap(err, "error listing thanosrulers: ")
+	}
+	if len(thanosrulers) > 1 {
+		// it is not supported temporarily to have multiple thanosrulers in the monitoring namespace
+		return nil, errors.Errorf(
+			"there is more than one thanosruler custom resource in %s", rulerNamespace)
+	}
+	if len(thanosrulers) == 0 {
+		// if there is no thanos ruler, custom rules will not be supported
+		return nil, nil
+	}
+
+	return rules.NewThanosRuler(thanosrulers[0], o.ruleResourceInformer, o.promResourceClient), nil
+}
+
+func parseToPrometheusRule(rule *v1alpha1.PostableAlertingRule) *promresourcesv1.Rule {
+	lbls := rule.Labels
+	lbls[rules.LabelKeyInternalRuleAlias] = rule.Alias
+	lbls[rules.LabelKeyInternalRuleDescription] = rule.Description
+	return &promresourcesv1.Rule{
+		Alert:       rule.Name,
+		Expr:        intstr.FromString(rule.Query),
+		For:         rule.Duration,
+		Labels:      lbls,
+		Annotations: rule.Annotations,
+	}
+}
+
+func pageAlertingRules(alertingRules []*v1alpha1.GettableAlertingRule,
+	queryParams *v1alpha1.AlertingRuleQueryParams) *v1alpha1.GettableAlertingRuleList {
+
+	alertingRules = queryParams.Filter(alertingRules)
+	queryParams.Sort(alertingRules)
+
+	return &v1alpha1.GettableAlertingRuleList{
+		Total: len(alertingRules),
+		Items: queryParams.Sub(alertingRules),
+	}
+}
+
+func pageAlerts(alertingRules []*v1alpha1.GettableAlertingRule,
+	queryParams *v1alpha1.AlertQueryParams) *v1alpha1.AlertList {
+
+	var alerts []*v1alpha1.Alert
+	for _, rule := range alertingRules {
+		alerts = append(alerts, queryParams.Filter(rule.Alerts)...)
+	}
+	queryParams.Sort(alerts)
+
+	return &v1alpha1.AlertList{
+		Total: len(alerts),
+		Items: queryParams.Sub(alerts),
+	}
+}

pkg/models/customalerting/rules/cache.go

@@ -0,0 +1,246 @@
+package rules
+
+import (
+	"sort"
+	"sync"
+
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	prominformersv1 "github.com/prometheus-operator/prometheus-operator/pkg/client/informers/externalversions/monitoring/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/client-go/tools/cache"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/server/errors"
+)
+
+// RuleCache caches all rules from the prometheusrule custom resources
+type RuleCache struct {
+	lock       sync.RWMutex
+	namespaces map[string]*namespaceRuleCache
+}
+
+func NewRuleCache(ruleResourceInformer prominformersv1.PrometheusRuleInformer) *RuleCache {
+	rc := RuleCache{
+		namespaces: make(map[string]*namespaceRuleCache),
+	}
+	ruleResourceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: rc.addCache,
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			rc.addCache(newObj)
+		},
+		DeleteFunc: rc.deleteCache,
+	})
+	return &rc
+}
+
+func (c *RuleCache) addCache(referObj interface{}) {
+	pr, ok := referObj.(*promresourcesv1.PrometheusRule)
+	if !ok {
+		return
+	}
+	cr := parseRuleResource(pr)
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	cn, ok := c.namespaces[pr.Namespace]
+	if !ok || cn == nil {
+		cn = &namespaceRuleCache{
+			namespace: pr.Namespace,
+			resources: make(map[string]*resourceRuleCache),
+		}
+		c.namespaces[pr.Namespace] = cn
+	}
+	cn.resources[pr.Name] = cr
+}
+
+func (c *RuleCache) deleteCache(referObj interface{}) {
+	pr, ok := referObj.(*promresourcesv1.PrometheusRule)
+	if !ok {
+		return
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+
+	cn, ok := c.namespaces[pr.Namespace]
+	if !ok {
+		return
+	}
+	delete(cn.resources, pr.Name)
+	if len(cn.resources) == 0 {
+		delete(c.namespaces, pr.Namespace)
+	}
+}
+
+func (c *RuleCache) getResourceRuleCaches(ruler Ruler, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector) (map[string]*resourceRuleCache, error) {
+
+	selected, err := ruleNamespaceSelected(ruler, ruleNamespace)
+	if err != nil {
+		return nil, err
+	}
+	if !selected {
+		return nil, nil
+	}
+	rSelector, err := ruler.RuleResourceSelector(extraRuleResourceSelector)
+	if err != nil {
+		return nil, err
+	}
+	var m = make(map[string]*resourceRuleCache)
+
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+
+	cn, ok := c.namespaces[ruleNamespace.Name]
+	if ok && cn != nil {
+		for _, cr := range cn.resources {
+			if rSelector.Matches(labels.Set(cr.Labels)) {
+				m[cr.Name] = cr
+			}
+		}
+	}
+	return m, nil
+}
+
+func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector, idOrName string) (*ResourceRule, error) {
+
+	caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
+	if err != nil {
+		return nil, err
+	}
+	if len(caches) == 0 {
+		return nil, nil
+	}
+
+	var rules []*ResourceRule
+	switch ruler.(type) {
+	case *PrometheusRuler:
+		for rn, rc := range caches {
+			if rule, ok := rc.IdRules[idOrName]; ok {
+				rules = append(rules, &ResourceRule{
+					Group:        rule.Group,
+					Id:           rule.Id,
+					Rule:         rule.Rule.DeepCopy(),
+					ResourceName: rn,
+				})
+			}
+		}
+	case *ThanosRuler:
+		for rn, rc := range caches {
+			if nrules, ok := rc.NameRules[idOrName]; ok {
+				for _, nrule := range nrules {
+					rules = append(rules, &ResourceRule{
+						Group:        nrule.Group,
+						Id:           nrule.Id,
+						Rule:         nrule.Rule.DeepCopy(),
+						ResourceName: rn,
+					})
+				}
+			}
+		}
+	default:
+		return nil, errors.New("unsupported ruler type")
+	}
+
+	if l := len(rules); l == 0 {
+		return nil, nil
+	} else if l > 1 {
+		// guarantees the stability of the get operations.
+		sort.Slice(rules, func(i, j int) bool {
+			return v1alpha1.AlertingRuleIdCompare(rules[i].Id, rules[j].Id)
+		})
+	}
+	return rules[0], nil
+}
+
+func (c *RuleCache) ListRules(ruler Ruler, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector) (map[string]*ResourceRules, error) {
+
+	caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
+	if err != nil {
+		return nil, err
+	}
+	if len(caches) == 0 {
+		return nil, nil
+	}
+
+	ret := make(map[string]*ResourceRules)
+	for rn, rc := range caches {
+		rrs := &ResourceRules{
+			GroupSet:  make(map[string]struct{}),
+			IdRules:   make(map[string]*ResourceRule),
+			NameRules: make(map[string][]*ResourceRule),
+		}
+		for name, rules := range rc.NameRules {
+			for _, rule := range rules {
+				rrs.GroupSet[rule.Group] = struct{}{}
+				rr := &ResourceRule{
+					Group:        rule.Group,
+					Id:           rule.Id,
+					Rule:         rule.Rule.DeepCopy(),
+					ResourceName: rn,
+				}
+				rrs.IdRules[rr.Id] = rr
+				rrs.NameRules[name] = append(rrs.NameRules[name], rr)
+			}
+		}
+		if len(rrs.IdRules) > 0 {
+			ret[rn] = rrs
+		}
+	}
+
+	return ret, nil
+}
+
+type namespaceRuleCache struct {
+	namespace string
+	resources map[string]*resourceRuleCache
+}
+
+type resourceRuleCache struct {
+	Name      string
+	Labels    map[string]string
+	GroupSet  map[string]struct{}
+	IdRules   map[string]*cacheRule
+	NameRules map[string][]*cacheRule
+}
+
+type cacheRule struct {
+	Group string
+	Id    string
+	Rule  *promresourcesv1.Rule
+}
+
+func parseRuleResource(pr *promresourcesv1.PrometheusRule) *resourceRuleCache {
+	var (
+		groupSet  = make(map[string]struct{})
+		idRules   = make(map[string]*cacheRule)
+		nameRules = make(map[string][]*cacheRule)
+	)
+	for i := 0; i < len(pr.Spec.Groups); i++ {
+		g := pr.Spec.Groups[i]
+		for j := 0; j < len(g.Rules); j++ {
+			gr := g.Rules[j]
+			if gr.Alert == "" {
+				continue
+			}
+			groupSet[g.Name] = struct{}{}
+			cr := &cacheRule{
+				Group: g.Name,
+				Id:    GenResourceRuleIdIgnoreFormat(g.Name, &gr),
+				Rule:  &gr,
+			}
+			nameRules[cr.Rule.Alert] = append(nameRules[cr.Rule.Alert], cr)
+			idRules[cr.Id] = cr
+		}
+	}
+	return &resourceRuleCache{
+		Name:      pr.Name,
+		Labels:    pr.Labels,
+		GroupSet:  groupSet,
+		IdRules:   idRules,
+		NameRules: nameRules,
+	}
+}

pkg/models/customalerting/rules/rule.go

@@ -0,0 +1,31 @@
+package rules
+
+import (
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+)
+
+type ResourceRules struct {
+	GroupSet  map[string]struct{}
+	IdRules   map[string]*ResourceRule
+	NameRules map[string][]*ResourceRule
+}
+
+type ResourceRule struct {
+	ResourceName string
+	Group        string
+	Id           string
+	Rule         *promresourcesv1.Rule
+}
+
+type ResourceRuleSole struct {
+	Level  v1alpha1.RuleLevel
+	Custom bool
+	ResourceRule
+}
+
+type ResourceRuleChunk struct {
+	Level            v1alpha1.RuleLevel
+	Custom           bool
+	ResourceRulesMap map[string]*ResourceRules
+}

pkg/models/customalerting/rules/ruler.go

@@ -0,0 +1,501 @@
+package rules
+
+import (
+	"context"
+	"fmt"
+	"sort"
+
+	"github.com/ghodss/yaml"
+	"github.com/pkg/errors"
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	prominformersv1 "github.com/prometheus-operator/prometheus-operator/pkg/client/informers/externalversions/monitoring/v1"
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+)
+
+const (
+	customAlertingRuleResourcePrefix = "custom-alerting-rule-"
+)
+
+var (
+	maxSecretSize        = corev1.MaxSecretSize
+	maxConfigMapDataSize = int(float64(maxSecretSize) * 0.3)
+
+	errOutOfConfigMapSize = errors.New("out of config map size")
+)
+
+type Ruler interface {
+	Namespace() string
+	RuleResourceNamespaceSelector() (labels.Selector, error)
+	RuleResourceSelector(extraRuleResourceSelector labels.Selector) (labels.Selector, error)
+	ExternalLabels() func() map[string]string
+
+	ListRuleResources(ruleNamespace *corev1.Namespace, extraRuleResourceSelector labels.Selector) (
+		[]*promresourcesv1.PrometheusRule, error)
+
+	AddAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace, extraRuleResourceSelector labels.Selector,
+		group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error
+	UpdateAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace, extraRuleResourceSelector labels.Selector,
+		group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error
+	DeleteAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace, extraRuleResourceSelector labels.Selector,
+		group string, name string) error
+}
+
+type ruleResource promresourcesv1.PrometheusRule
+
+// deleteAlertingRule deletes the rules with the given name.
+// If the rule is deleted, return true to indicate the resource should be updated.
+func (r *ruleResource) deleteAlertingRule(name string) (bool, error) {
+	var (
+		nGroups []promresourcesv1.RuleGroup
+		ok      bool
+	)
+
+	for _, g := range r.Spec.Groups {
+		var rules []promresourcesv1.Rule
+		for _, gr := range g.Rules {
+			if gr.Alert != "" && gr.Alert == name {
+				ok = true
+				continue
+			}
+			rules = append(rules, gr)
+		}
+		if len(rules) > 0 {
+			nGroups = append(nGroups, promresourcesv1.RuleGroup{
+				Name:                    g.Name,
+				Interval:                g.Interval,
+				PartialResponseStrategy: g.PartialResponseStrategy,
+				Rules:                   rules,
+			})
+		}
+	}
+
+	if ok {
+		r.Spec.Groups = nGroups
+	}
+	return ok, nil
+}
+
+// updateAlertingRule updates the rule with the given group.
+// If the rule is updated, return true to indicate the resource should be updated.
+func (r *ruleResource) updateAlertingRule(groupName string, rule *promresourcesv1.Rule) (bool, error) {
+	var (
+		ok       bool
+		pr       = (promresourcesv1.PrometheusRule)(*r)
+		npr      = pr.DeepCopy()
+		groupMap = make(map[string]*promresourcesv1.RuleGroup)
+	)
+
+	for _, g := range npr.Spec.Groups {
+		var rules []promresourcesv1.Rule
+		for i, gr := range g.Rules {
+			if gr.Alert != "" && gr.Alert == rule.Alert {
+				ok = true
+				continue
+			}
+			rules = append(rules, g.Rules[i])
+		}
+		if len(rules) > 0 {
+			groupMap[g.Name] = &promresourcesv1.RuleGroup{
+				Name:                    g.Name,
+				Interval:                g.Interval,
+				PartialResponseStrategy: g.PartialResponseStrategy,
+				Rules:                   rules,
+			}
+		}
+	}
+
+	if ok {
+		if g, exist := groupMap[groupName]; exist {
+			g.Rules = append(g.Rules, *rule)
+		} else {
+			groupMap[groupName] = &promresourcesv1.RuleGroup{
+				Name:  groupName,
+				Rules: []promresourcesv1.Rule{*rule},
+			}
+		}
+
+		var groups []promresourcesv1.RuleGroup
+		for _, g := range groupMap {
+			groups = append(groups, *g)
+		}
+
+		npr.Spec.Groups = groups
+		content, err := yaml.Marshal(npr)
+		if err != nil {
+			return false, errors.Wrap(err, "failed to unmarshal content")
+		}
+
+		if len(string(content)) < maxConfigMapDataSize { // check size limit
+			r.Spec.Groups = groups
+			return true, nil
+		}
+		return false, errOutOfConfigMapSize
+	}
+	return false, nil
+}
+
+func (r *ruleResource) addAlertingRule(group string, rule *promresourcesv1.Rule) (bool, error) {
+	var (
+		err error
+		pr  = (promresourcesv1.PrometheusRule)(*r)
+		npr = pr.DeepCopy()
+		ok  bool
+	)
+
+	for i := 0; i < len(npr.Spec.Groups); i++ {
+		if npr.Spec.Groups[i].Name == group {
+			npr.Spec.Groups[i].Rules = append(npr.Spec.Groups[i].Rules, *rule)
+			ok = true
+			break
+		}
+	}
+	if !ok { // add a group when there is no group with the specified group name
+		npr.Spec.Groups = append(npr.Spec.Groups, promresourcesv1.RuleGroup{
+			Name:  group,
+			Rules: []promresourcesv1.Rule{*rule},
+		})
+	}
+
+	content, err := yaml.Marshal(npr)
+	if err != nil {
+		return false, errors.Wrap(err, "failed to unmarshal content")
+	}
+
+	if len(string(content)) < maxConfigMapDataSize { // check size limit
+		r.Spec.Groups = npr.Spec.Groups
+		return true, nil
+	} else {
+		return false, errOutOfConfigMapSize
+	}
+}
+
+func (r *ruleResource) commit(ctx context.Context, prometheusResourceClient promresourcesclient.Interface) error {
+	var pr = (promresourcesv1.PrometheusRule)(*r)
+	if len(pr.Spec.Groups) == 0 {
+		return prometheusResourceClient.MonitoringV1().PrometheusRules(r.Namespace).Delete(ctx, r.Name, metav1.DeleteOptions{})
+	}
+	newPr, err := prometheusResourceClient.MonitoringV1().PrometheusRules(r.Namespace).Update(ctx, &pr, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+	newPr.DeepCopyInto(&pr)
+	return nil
+}
+
+type PrometheusRuler struct {
+	resource *promresourcesv1.Prometheus
+	informer prominformersv1.PrometheusRuleInformer
+	client   promresourcesclient.Interface
+}
+
+func NewPrometheusRuler(resource *promresourcesv1.Prometheus, informer prominformersv1.PrometheusRuleInformer,
+	client promresourcesclient.Interface) Ruler {
+	return &PrometheusRuler{
+		resource: resource,
+		informer: informer,
+		client:   client,
+	}
+}
+
+func (r *PrometheusRuler) Namespace() string {
+	return r.resource.Namespace
+}
+
+func (r *PrometheusRuler) RuleResourceNamespaceSelector() (labels.Selector, error) {
+	if r.resource.Spec.RuleNamespaceSelector == nil {
+		return nil, nil
+	}
+	return metav1.LabelSelectorAsSelector(r.resource.Spec.RuleNamespaceSelector)
+}
+
+func (r *PrometheusRuler) RuleResourceSelector(extraRuleResourceSelector labels.Selector) (labels.Selector, error) {
+	rSelector, err := metav1.LabelSelectorAsSelector(r.resource.Spec.RuleSelector)
+	if err != nil {
+		return nil, err
+	}
+	if extraRuleResourceSelector != nil {
+		if requirements, ok := extraRuleResourceSelector.Requirements(); ok {
+			rSelector = rSelector.Add(requirements...)
+		}
+	}
+	return rSelector, nil
+}
+
+func (r *PrometheusRuler) ExternalLabels() func() map[string]string {
+	// ignoring the external labels because rules gotten from prometheus endpoint do not include them
+	return nil
+}
+
+func (r *PrometheusRuler) ListRuleResources(ruleNamespace *corev1.Namespace, extraRuleResourceSelector labels.Selector) (
+	[]*promresourcesv1.PrometheusRule, error) {
+	selected, err := ruleNamespaceSelected(r, ruleNamespace)
+	if err != nil {
+		return nil, err
+	}
+	if !selected {
+		return nil, nil
+	}
+	rSelector, err := r.RuleResourceSelector(extraRuleResourceSelector)
+	if err != nil {
+		return nil, err
+	}
+	return r.informer.Lister().PrometheusRules(ruleNamespace.Name).List(rSelector)
+}
+
+func (r *PrometheusRuler) AddAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector,
+	group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error {
+	return errors.New("not supported to add rules for prometheus")
+}
+
+func (r *PrometheusRuler) UpdateAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector,
+	group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error {
+	return errors.New("not supported to update rules for prometheus")
+}
+
+func (r *PrometheusRuler) DeleteAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector,
+	group string, name string) error {
+	return errors.New("not supported to update rules for prometheus")
+}
+
+type ThanosRuler struct {
+	resource *promresourcesv1.ThanosRuler
+	informer prominformersv1.PrometheusRuleInformer
+	client   promresourcesclient.Interface
+}
+
+func NewThanosRuler(resource *promresourcesv1.ThanosRuler, informer prominformersv1.PrometheusRuleInformer,
+	client promresourcesclient.Interface) Ruler {
+	return &ThanosRuler{
+		resource: resource,
+		informer: informer,
+		client:   client,
+	}
+}
+
+func (r *ThanosRuler) Namespace() string {
+	return r.resource.Namespace
+}
+
+func (r *ThanosRuler) RuleResourceNamespaceSelector() (labels.Selector, error) {
+	if r.resource.Spec.RuleNamespaceSelector == nil {
+		return nil, nil
+	}
+	return metav1.LabelSelectorAsSelector(r.resource.Spec.RuleNamespaceSelector)
+}
+
+func (r *ThanosRuler) RuleResourceSelector(extraRuleSelector labels.Selector) (labels.Selector, error) {
+	rSelector, err := metav1.LabelSelectorAsSelector(r.resource.Spec.RuleSelector)
+	if err != nil {
+		return nil, err
+	}
+	if extraRuleSelector != nil {
+		if requirements, ok := extraRuleSelector.Requirements(); ok {
+			rSelector = rSelector.Add(requirements...)
+		}
+	}
+	return rSelector, nil
+}
+
+func (r *ThanosRuler) ExternalLabels() func() map[string]string {
+	// rules gotten from thanos ruler endpoint include the labels
+	lbls := make(map[string]string)
+	if ls := r.resource.Spec.Labels; ls != nil {
+		for k, v := range ls {
+			lbls[k] = v
+		}
+	}
+	return func() map[string]string {
+		return lbls
+	}
+}
+
+func (r *ThanosRuler) ListRuleResources(ruleNamespace *corev1.Namespace, extraRuleSelector labels.Selector) (
+	[]*promresourcesv1.PrometheusRule, error) {
+	selected, err := ruleNamespaceSelected(r, ruleNamespace)
+	if err != nil {
+		return nil, err
+	}
+	if !selected {
+		return nil, nil
+	}
+	rSelector, err := r.RuleResourceSelector(extraRuleSelector)
+	if err != nil {
+		return nil, err
+	}
+	return r.informer.Lister().PrometheusRules(ruleNamespace.Name).List(rSelector)
+}
+
+func (r *ThanosRuler) AddAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector,
+	group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error {
+
+	prometheusRules, err := r.ListRuleResources(ruleNamespace, extraRuleResourceSelector)
+	if err != nil {
+		return err
+	}
+
+	return r.addAlertingRule(ctx, ruleNamespace, prometheusRules, nil, group, rule, ruleResourceLabels)
+}
+
+func (r *ThanosRuler) addAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	prometheusRules []*promresourcesv1.PrometheusRule, excludeRuleResources map[string]*ruleResource,
+	group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error {
+
+	sort.Slice(prometheusRules, func(i, j int) bool {
+		return len(fmt.Sprint(prometheusRules[i])) <= len(fmt.Sprint(prometheusRules[j]))
+	})
+
+	for _, prometheusRule := range prometheusRules {
+		if len(excludeRuleResources) > 0 {
+			if _, ok := excludeRuleResources[prometheusRule.Name]; ok {
+				continue
+			}
+		}
+		resource := ruleResource(*prometheusRule)
+		if ok, err := resource.addAlertingRule(group, rule); err != nil {
+			if err == errOutOfConfigMapSize {
+				break
+			}
+			return err
+		} else if ok {
+			if err = resource.commit(ctx, r.client); err != nil {
+				return err
+			}
+			return nil
+		}
+	}
+	// create a new rule resource and add rule into it when all existing rule resources are full.
+	newPromRule := promresourcesv1.PrometheusRule{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:    ruleNamespace.Name,
+			GenerateName: customAlertingRuleResourcePrefix,
+			Labels:       ruleResourceLabels,
+		},
+		Spec: promresourcesv1.PrometheusRuleSpec{
+			Groups: []promresourcesv1.RuleGroup{{
+				Name:  group,
+				Rules: []promresourcesv1.Rule{*rule},
+			}},
+		},
+	}
+	if _, err := r.client.MonitoringV1().
+		PrometheusRules(ruleNamespace.Name).Create(ctx, &newPromRule, metav1.CreateOptions{}); err != nil {
+		return errors.Wrapf(err, "error creating a prometheusrule resource %s/%s",
+			newPromRule.Namespace, newPromRule.Name)
+	}
+	return nil
+}
+
+func (r *ThanosRuler) UpdateAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector,
+	group string, rule *promresourcesv1.Rule, ruleResourceLabels map[string]string) error {
+
+	prometheusRules, err := r.ListRuleResources(ruleNamespace, extraRuleResourceSelector)
+	if err != nil {
+		return err
+	}
+
+	var (
+		found              bool
+		success            bool
+		resourcesToDelRule = make(map[string]*ruleResource)
+	)
+	for _, prometheusRule := range prometheusRules {
+		resource := ruleResource(*prometheusRule)
+		if success { // If the update has been successful, delete the possible same rule in other resources
+			if ok, err := resource.deleteAlertingRule(rule.Alert); err != nil {
+				return err
+			} else if ok {
+				if err = resource.commit(ctx, r.client); err != nil {
+					return err
+				}
+			}
+			continue
+		}
+		if ok, err := resource.updateAlertingRule(group, rule); err != nil {
+			if err == errOutOfConfigMapSize {
+				// updating the rule in the resource will oversize the size limit,
+				// so delete it and then add the new rule to a new resource.
+				resourcesToDelRule[resource.Name] = &resource
+				found = true
+			} else {
+				return err
+			}
+		} else if ok {
+			if err = resource.commit(ctx, r.client); err != nil {
+				return err
+			}
+			found = true
+			success = true
+		}
+	}
+
+	if !found {
+		return v1alpha1.ErrAlertingRuleNotFound
+	}
+
+	if !success {
+		err := r.addAlertingRule(ctx, ruleNamespace, prometheusRules, resourcesToDelRule, group, rule, ruleResourceLabels)
+		if err != nil {
+			return err
+		}
+	}
+	for _, resource := range resourcesToDelRule {
+		if ok, err := resource.deleteAlertingRule(rule.Alert); err != nil {
+			return err
+		} else if ok {
+			if err = resource.commit(ctx, r.client); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (r *ThanosRuler) DeleteAlertingRule(ctx context.Context, ruleNamespace *corev1.Namespace,
+	extraRuleResourceSelector labels.Selector, group string, name string) error {
+	prometheusRules, err := r.ListRuleResources(ruleNamespace, extraRuleResourceSelector)
+	if err != nil {
+		return err
+	}
+	var success bool
+	for _, prometheusRule := range prometheusRules {
+		resource := ruleResource(*prometheusRule)
+		if ok, err := resource.deleteAlertingRule(name); err != nil {
+			return err
+		} else if ok {
+			if err = resource.commit(ctx, r.client); err != nil {
+				return err
+			}
+			success = true
+		}
+	}
+	if !success {
+		return v1alpha1.ErrAlertingRuleNotFound
+	}
+	return nil
+}
+
+func ruleNamespaceSelected(r Ruler, ruleNamespace *corev1.Namespace) (bool, error) {
+	rnSelector, err := r.RuleResourceNamespaceSelector()
+	if err != nil {
+		return false, err
+	}
+	if rnSelector == nil { // refer to the comment of Prometheus.Spec.RuleResourceNamespaceSelector
+		if r.Namespace() != ruleNamespace.Name {
+			return false, nil
+		}
+	} else {
+		if !rnSelector.Matches(labels.Set(ruleNamespace.Labels)) {
+			return false, nil
+		}
+	}
+	return true, nil
+}

pkg/models/customalerting/rules/utils.go

@@ -0,0 +1,421 @@
+package rules
+
+import (
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/prometheus-community/prom-label-proxy/injectproxy"
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	prommodel "github.com/prometheus/common/model"
+	promlabels "github.com/prometheus/prometheus/pkg/labels"
+	"github.com/prometheus/prometheus/promql/parser"
+	"github.com/prometheus/prometheus/rules"
+	"k8s.io/klog"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+const (
+	ErrGenRuleId = "error generating rule id"
+
+	LabelKeyInternalRuleGroup       = "__rule_group__"
+	LabelKeyInternalRuleName        = "__rule_name__"
+	LabelKeyInternalRuleQuery       = "__rule_query__"
+	LabelKeyInternalRuleDuration    = "__rule_duration__"
+	LabelKeyInternalRuleAlias       = "__rule_alias__"
+	LabelKeyInternalRuleDescription = "__rule_description__"
+)
+
+func FormatExpr(expr string) (string, error) {
+	parsedExpr, err := parser.ParseExpr(expr)
+	if err == nil {
+		return parsedExpr.String(), nil
+	}
+	return "", errors.Wrapf(err, "failed to parse expr: %s", expr)
+}
+
+// InjectExprNamespaceLabel injects an label, whose key is "namespace" and whose value is the given namespace,
+// into the prometheus query expression, which will limit the query scope.
+func InjectExprNamespaceLabel(expr, namespace string) (string, error) {
+	parsedExpr, err := parser.ParseExpr(expr)
+	if err != nil {
+		return "", err
+	}
+	if err = injectproxy.NewEnforcer(&promlabels.Matcher{
+		Type:  promlabels.MatchEqual,
+		Name:  "namespace",
+		Value: namespace,
+	}).EnforceNode(parsedExpr); err == nil {
+		return parsedExpr.String(), nil
+	}
+	return "", err
+}
+
+func FormatDuration(for_ string) (string, error) {
+	var duration prommodel.Duration
+	var err error
+	if for_ != "" {
+		duration, err = prommodel.ParseDuration(for_)
+		if err != nil {
+			return "", errors.Wrapf(err, "failed to parse Duration string(\"%s\") to time.Duration", for_)
+		}
+	}
+	return duration.String(), nil
+}
+
+func parseDurationSeconds(durationSeconds float64) string {
+	return prommodel.Duration(int64(durationSeconds * float64(time.Second))).String()
+}
+
+func GenResourceRuleIdIgnoreFormat(group string, rule *promresourcesv1.Rule) string {
+	query, err := FormatExpr(rule.Expr.String())
+	if err != nil {
+		klog.Warning(errors.Wrapf(err, "invalid alerting rule(%s)", rule.Alert))
+		query = rule.Expr.String()
+	}
+	duration, err := FormatDuration(rule.For)
+	if err != nil {
+		klog.Warning(errors.Wrapf(err, "invalid alerting rule(%s)", rule.Alert))
+		duration = rule.For
+	}
+
+	lbls := make(map[string]string)
+	for k, v := range rule.Labels {
+		lbls[k] = v
+	}
+	lbls[LabelKeyInternalRuleGroup] = group
+	lbls[LabelKeyInternalRuleName] = rule.Alert
+	lbls[LabelKeyInternalRuleQuery] = query
+	lbls[LabelKeyInternalRuleDuration] = duration
+
+	return prommodel.Fingerprint(prommodel.LabelsToSignature(lbls)).String()
+}
+
+func GenEndpointRuleId(group string, epRule *customalerting.AlertingRule,
+	externalLabels func() map[string]string) (string, error) {
+	query, err := FormatExpr(epRule.Query)
+	if err != nil {
+		return "", err
+	}
+	duration := parseDurationSeconds(epRule.Duration)
+
+	var labelsMap map[string]string
+	if externalLabels == nil {
+		labelsMap = epRule.Labels
+	} else {
+		labelsMap = make(map[string]string)
+		extLabels := externalLabels()
+		for key, value := range epRule.Labels {
+			if v, ok := extLabels[key]; !(ok && value == v) {
+				labelsMap[key] = value
+			}
+		}
+	}
+
+	lbls := make(map[string]string)
+	for k, v := range labelsMap {
+		lbls[k] = v
+	}
+	lbls[LabelKeyInternalRuleGroup] = group
+	lbls[LabelKeyInternalRuleName] = epRule.Name
+	lbls[LabelKeyInternalRuleQuery] = query
+	lbls[LabelKeyInternalRuleDuration] = duration
+
+	return prommodel.Fingerprint(prommodel.LabelsToSignature(lbls)).String(), nil
+}
+
+// MixAlertingRules mix rules from prometheusrule custom resources and rules from endpoints.
+// Use rules from prometheusrule custom resources as the main reference.
+func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRuleGroups []*customalerting.RuleGroup,
+	extLabels func() map[string]string) ([]*v1alpha1.GettableAlertingRule, error) {
+
+	var (
+		idEpRules = make(map[string]*customalerting.AlertingRule)
+		nameIds   = make(map[string][]string)
+		ret       []*v1alpha1.GettableAlertingRule
+	)
+	for _, group := range epRuleGroups {
+		fileShort := strings.TrimSuffix(filepath.Base(group.File), filepath.Ext(group.File))
+		if !strings.HasPrefix(fileShort, ruleNamespace+"-") {
+			continue
+		}
+		resourceRules, ok := ruleChunk.ResourceRulesMap[strings.TrimPrefix(fileShort, ruleNamespace+"-")]
+		if !ok {
+			continue
+		}
+		if _, ok := resourceRules.GroupSet[group.Name]; !ok {
+			continue
+		}
+
+		for _, epRule := range group.Rules {
+			if eid, err := GenEndpointRuleId(group.Name, epRule, extLabels); err != nil {
+				return nil, errors.Wrap(err, ErrGenRuleId)
+			} else {
+				idEpRules[eid] = epRule
+				nameIds[epRule.Name] = append(nameIds[epRule.Name], eid)
+			}
+		}
+	}
+	if ruleChunk.Custom {
+		// guarantee the names of the custom alerting rules not to be repeated
+		var m = make(map[string][]*ResourceRule)
+		for _, resourceRules := range ruleChunk.ResourceRulesMap {
+			for name, rrArr := range resourceRules.NameRules {
+				m[name] = append(m[name], rrArr...)
+			}
+		}
+		for _, rrArr := range m {
+			if l := len(rrArr); l > 0 {
+				if l > 1 {
+					sort.Slice(rrArr, func(i, j int) bool {
+						return v1alpha1.AlertingRuleIdCompare(rrArr[i].Id, rrArr[j].Id)
+					})
+				}
+				resRule := rrArr[0]
+				epRule := idEpRules[resRule.Id]
+				if r := mixAlertingRule(resRule, epRule, ruleChunk.Custom, ruleChunk.Level); r != nil {
+					ret = append(ret, r)
+				}
+			}
+		}
+	} else {
+		// guarantee the ids of the builtin alerting rules not to be repeated
+		var m = make(map[string]*v1alpha1.GettableAlertingRule)
+		for _, resourceRules := range ruleChunk.ResourceRulesMap {
+			for id, rule := range resourceRules.IdRules {
+				if r := mixAlertingRule(rule, idEpRules[id], ruleChunk.Custom, ruleChunk.Level); r != nil {
+					m[id] = r
+				}
+			}
+		}
+		for _, r := range m {
+			ret = append(ret, r)
+		}
+	}
+
+	return ret, nil
+}
+
+func MixAlertingRule(ruleNamespace string, rule *ResourceRuleSole, epRuleGroups []*customalerting.RuleGroup,
+	extLabels func() map[string]string) (*v1alpha1.GettableAlertingRule, error) {
+
+	if rule == nil || rule.Rule == nil {
+		return nil, nil
+	}
+
+	var epRules = make(map[string]*customalerting.AlertingRule)
+	for _, group := range epRuleGroups {
+		fileShort := strings.TrimSuffix(filepath.Base(group.File), filepath.Ext(group.File))
+		if !strings.HasPrefix(fileShort, ruleNamespace+"-") {
+			continue
+		}
+		if strings.TrimPrefix(fileShort, ruleNamespace+"-") != rule.ResourceName {
+			continue
+		}
+
+		for _, epRule := range group.Rules {
+			if eid, err := GenEndpointRuleId(group.Name, epRule, extLabels); err != nil {
+				return nil, errors.Wrap(err, ErrGenRuleId)
+			} else {
+				if rule.Rule.Alert == epRule.Name {
+					epRules[eid] = epRule
+				}
+			}
+		}
+	}
+	var epRule *customalerting.AlertingRule
+	if rule.Custom {
+		// guarantees the stability of the get operations.
+		var ids []string
+		for k, _ := range epRules {
+			ids = append(ids, k)
+		}
+		if l := len(ids); l > 0 {
+			if l > 1 {
+				sort.Slice(ids, func(i, j int) bool {
+					return v1alpha1.AlertingRuleIdCompare(ids[i], ids[j])
+				})
+			}
+			epRule = epRules[ids[0]]
+		}
+	} else {
+		epRule = epRules[rule.Id]
+	}
+
+	return mixAlertingRule(&rule.ResourceRule, epRule, rule.Custom, rule.Level), nil
+}
+
+func mixAlertingRule(resRule *ResourceRule, epRule *customalerting.AlertingRule,
+	custom bool, level v1alpha1.RuleLevel) *v1alpha1.GettableAlertingRule {
+
+	if resRule == nil || resRule.Rule == nil {
+		return nil
+	}
+
+	var (
+		alias      string
+		descrption string
+		lbls       map[string]string
+	)
+	if len(resRule.Rule.Labels) > 0 {
+		lbls = make(map[string]string)
+		for k, v := range resRule.Rule.Labels {
+			switch k {
+			case LabelKeyInternalRuleAlias:
+				alias = v
+			case LabelKeyInternalRuleDescription:
+				descrption = v
+			default:
+				lbls[k] = v
+			}
+		}
+	}
+
+	rule := v1alpha1.GettableAlertingRule{
+		AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
+			Id:     resRule.Id,
+			Name:   resRule.Rule.Alert,
+			Custom: custom,
+			Level:  level,
+		},
+		AlertingRuleProps: v1alpha1.AlertingRuleProps{
+			Query:       resRule.Rule.Expr.String(),
+			Duration:    resRule.Rule.For,
+			Labels:      lbls,
+			Annotations: resRule.Rule.Annotations,
+		},
+		Alias:       alias,
+		Description: descrption,
+		State:       stateInactiveString,
+		Health:      string(rules.HealthUnknown),
+	}
+
+	if epRule != nil {
+		// The state information and alerts associated with the rule are from the rule from the endpoint.
+		if epRule.Health != "" {
+			rule.Health = epRule.Health
+		}
+		rule.LastError = epRule.LastError
+		rule.LastEvaluation = epRule.LastEvaluation
+		rule.EvaluationDurationSeconds = epRule.EvaluationTime
+
+		rState := strings.ToLower(epRule.State)
+		cliRuleStateEmpty := rState == ""
+		if !cliRuleStateEmpty {
+			rule.State = rState
+		}
+		for _, a := range epRule.Alerts {
+			aState := strings.ToLower(a.State)
+			if cliRuleStateEmpty {
+				// for the rules gotten from prometheus or thanos ruler with a lower version, they may not contain
+				// the state property, so compute the rule state by states of its alerts
+				if alertState(rState) < alertState(aState) {
+					rule.State = aState
+				}
+			}
+			var lbls map[string]string
+			if len(a.Labels) > 0 {
+				lbls = make(map[string]string)
+				for k, v := range a.Labels {
+					switch k {
+					case LabelKeyInternalRuleAlias, LabelKeyInternalRuleDescription:
+					default:
+						lbls[k] = v
+					}
+				}
+			}
+			rule.Alerts = append(rule.Alerts, &v1alpha1.Alert{
+				ActiveAt:    a.ActiveAt,
+				Labels:      lbls,
+				Annotations: a.Annotations,
+				State:       aState,
+				Value:       a.Value,
+
+				Rule: &rule.AlertingRuleQualifier,
+			})
+		}
+	}
+	return &rule
+}
+
+func ParseAlertingRules(epRuleGroups []*customalerting.RuleGroup, custom bool, level v1alpha1.RuleLevel,
+	filterFunc func(group, ruleId string, rule *customalerting.AlertingRule) bool) ([]*v1alpha1.GettableAlertingRule, error) {
+
+	var ret []*v1alpha1.GettableAlertingRule
+	for _, g := range epRuleGroups {
+		for _, r := range g.Rules {
+			id, err := GenEndpointRuleId(g.Name, r, nil)
+			if err != nil {
+				return nil, err
+			}
+			if filterFunc(g.Name, id, r) {
+				rule := &v1alpha1.GettableAlertingRule{
+					AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
+						Id:     id,
+						Name:   r.Name,
+						Custom: custom,
+						Level:  level,
+					},
+					AlertingRuleProps: v1alpha1.AlertingRuleProps{
+						Query:       r.Query,
+						Duration:    parseDurationSeconds(r.Duration),
+						Labels:      r.Labels,
+						Annotations: r.Annotations,
+					},
+					State:                     r.State,
+					Health:                    string(r.Health),
+					LastError:                 r.LastError,
+					LastEvaluation:            r.LastEvaluation,
+					EvaluationDurationSeconds: r.EvaluationTime,
+				}
+				if rule.Health != "" {
+					rule.Health = string(rules.HealthUnknown)
+				}
+				ruleStateEmpty := rule.State == ""
+				rule.State = stateInactiveString
+				for _, a := range r.Alerts {
+					aState := strings.ToLower(a.State)
+					if ruleStateEmpty {
+						// for the rules gotten from prometheus or thanos ruler with a lower version, they may not contain
+						// the state property, so compute the rule state by states of its alerts
+						if alertState(rule.State) < alertState(aState) {
+							rule.State = aState
+						}
+					}
+					rule.Alerts = append(rule.Alerts, &v1alpha1.Alert{
+						ActiveAt:    a.ActiveAt,
+						Labels:      a.Labels,
+						Annotations: a.Annotations,
+						State:       aState,
+						Value:       a.Value,
+
+						Rule: &rule.AlertingRuleQualifier,
+					})
+				}
+				ret = append(ret, rule)
+			}
+		}
+	}
+	return ret, nil
+}
+
+var (
+	statePendingString  = rules.StatePending.String()
+	stateFiringString   = rules.StateFiring.String()
+	stateInactiveString = rules.StateInactive.String()
+)
+
+func alertState(state string) rules.AlertState {
+	switch state {
+	case statePendingString:
+		return rules.StatePending
+	case stateFiringString:
+		return rules.StateFiring
+	case stateInactiveString:
+		return rules.StateInactive
+	}
+	return rules.StateInactive
+}

pkg/models/customalerting/rules/utils_test.go

@@ -0,0 +1,95 @@
+package rules
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	"github.com/prometheus/prometheus/rules"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
+	"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
+)
+
+func TestMixAlertingRules(t *testing.T) {
+	var tests = []struct {
+		description       string
+		ruleNamespace     string
+		resourceRuleChunk *ResourceRuleChunk
+		ruleGroups        []*customalerting.RuleGroup
+		extLabels         func() map[string]string
+		expected          []*v1alpha1.GettableAlertingRule
+	}{{
+		description:   "mix custom rules",
+		ruleNamespace: "test",
+		resourceRuleChunk: &ResourceRuleChunk{
+			Level:  v1alpha1.RuleLevelNamespace,
+			Custom: true,
+			ResourceRulesMap: map[string]*ResourceRules{
+				"custom-alerting-rule-jqbgn": &ResourceRules{
+					GroupSet: map[string]struct{}{"alerting.custom.defaults": struct{}{}},
+					NameRules: map[string][]*ResourceRule{
+						"f89836879157ca88": []*ResourceRule{{
+							ResourceName: "custom-alerting-rule-jqbgn",
+							Group:        "alerting.custom.defaults",
+							Id:           "f89836879157ca88",
+							Rule: &promresourcesv1.Rule{
+								Alert: "TestCPUUsageHigh",
+								Expr:  intstr.FromString(`namespace:workload_cpu_usage:sum{namespace="test"} > 1`),
+								For:   "1m",
+								Labels: map[string]string{
+									LabelKeyInternalRuleAlias:       "The alias is here",
+									LabelKeyInternalRuleDescription: "The description is here",
+								},
+							},
+						}},
+					},
+				},
+			},
+		},
+		ruleGroups: []*customalerting.RuleGroup{{
+			Name: "alerting.custom.defaults",
+			File: "/etc/thanos/rules/thanos-ruler-thanos-ruler-rulefiles-0/test-custom-alerting-rule-jqbgn.yaml",
+			Rules: []*customalerting.AlertingRule{{
+				Name:     "TestCPUUsageHigh",
+				Query:    `namespace:workload_cpu_usage:sum{namespace="test"} > 1`,
+				Duration: 60,
+				Health:   string(rules.HealthGood),
+				State:    stateInactiveString,
+				Labels: map[string]string{
+					LabelKeyInternalRuleAlias:       "The alias is here",
+					LabelKeyInternalRuleDescription: "The description is here",
+				},
+			}},
+		}},
+		expected: []*v1alpha1.GettableAlertingRule{{
+			AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
+				Id:     "f89836879157ca88",
+				Name:   "TestCPUUsageHigh",
+				Level:  v1alpha1.RuleLevelNamespace,
+				Custom: true,
+			},
+			AlertingRuleProps: v1alpha1.AlertingRuleProps{
+				Query:    `namespace:workload_cpu_usage:sum{namespace="test"} > 1`,
+				Duration: "1m",
+				Labels:   map[string]string{},
+			},
+			Alias:       "The alias is here",
+			Description: "The description is here",
+			Health:      string(rules.HealthGood),
+			State:       stateInactiveString,
+		}},
+	}}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			rules, err := MixAlertingRules(test.ruleNamespace, test.resourceRuleChunk, test.ruleGroups, test.extLabels)
+			if err != nil {
+				t.Fatal(err)
+			}
+			if diff := cmp.Diff(rules, test.expected); diff != "" {
+				t.Fatalf("%T differ (-got, +want): %s", test.expected, diff)
+			}
+		})
+	}
+}

pkg/models/resources/v1alpha2/resource/resource_test.go

@@ -213,7 +213,7 @@ func prepare() (informers.InformerFactory, error) {
 	k8sClient := fakek8s.NewSimpleClientset()
 	istioClient := fakeistio.NewSimpleClientset()
 	snapshotClient := fakesnapshot.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, nil, nil)
 
 	k8sInformerFactory := fakeInformerFactory.KubernetesSharedInformerFactory()
 

pkg/models/resources/v1alpha3/resource/resource_test.go

@@ -108,7 +108,7 @@ func prepare() *ResourceGetter {
 	istioClient := fakeistio.NewSimpleClientset()
 	snapshotClient := fakesnapshot.NewSimpleClientset()
 	apiextensionsClient := fakeapiextensions.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, apiextensionsClient)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, snapshotClient, apiextensionsClient, nil)
 
 	for _, namespace := range namespaces {
 		fakeInformerFactory.KubernetesSharedInformerFactory().Core().V1().

pkg/models/tenant/tenent_test.go

@@ -491,7 +491,7 @@ func prepare() Interface {
 	ksClient := fakeks.NewSimpleClientset([]runtime.Object{testWorkspace, systemWorkspace}...)
 	k8sClient := fakek8s.NewSimpleClientset([]runtime.Object{testNamespace, kubesphereSystem}...)
 	istioClient := fakeistio.NewSimpleClientset()
-	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, nil, nil)
+	fakeInformerFactory := informers.NewInformerFactories(k8sClient, ksClient, istioClient, nil, nil, nil)
 
 	for _, workspace := range workspaces {
 		fakeInformerFactory.KubeSphereSharedInformerFactory().Tenant().V1alpha1().

pkg/simple/client/customalerting/options.go

@@ -0,0 +1,66 @@
+/*
+Copyright 2020 KubeSphere Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customalerting
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/spf13/pflag"
+	"kubesphere.io/kubesphere/pkg/utils/reflectutils"
+)
+
+type Options struct {
+	PrometheusEndpoint       string `json:"prometheusEndpoint" yaml:"prometheusEndpoint"`
+	ThanosRulerEndpoint      string `json:"thanosRulerEndpoint" yaml:"thanosRulerEndpoint"`
+	ThanosRuleResourceLabels string `json:"thanosRuleResourceLabels" yaml:"thanosRuleResourceLabels"`
+}
+
+func NewOptions() *Options {
+	return &Options{}
+}
+
+func (o *Options) ApplyTo(options *Options) {
+	reflectutils.Override(options, o)
+}
+
+func (o *Options) Validate() []error {
+	errs := []error{}
+
+	if len(o.ThanosRuleResourceLabels) > 0 {
+		lblStrings := strings.Split(o.ThanosRuleResourceLabels, ",")
+		for _, lblString := range lblStrings {
+			if len(lblString) > 0 {
+				lbl := strings.Split(lblString, "=")
+				if len(lbl) != 2 {
+					errs = append(errs, fmt.Errorf("invalid thanos-rule-resource-labels arg: %s", o.ThanosRuleResourceLabels))
+				}
+			}
+		}
+	}
+
+	return errs
+}
+
+func (o *Options) AddFlags(fs *pflag.FlagSet, c *Options) {
+	fs.StringVar(&o.PrometheusEndpoint, "prometheus-endpoint", c.PrometheusEndpoint,
+		"Prometheus service endpoint from which built-in alerting rules are gotten.")
+	fs.StringVar(&o.ThanosRulerEndpoint, "thanos-ruler-endpoint", c.ThanosRulerEndpoint,
+		"Thanos ruler service endpoint from which custom alerting rules are gotten.")
+	fs.StringVar(&o.ThanosRuleResourceLabels, "thanos-rule-resource-labels", c.ThanosRuleResourceLabels,
+		"The labels will be added to prometheusrule custom resources to be selected by thanos ruler. eg: thanosruler=thanos-ruler,role=custom-alerting-rules")
+}

pkg/simple/client/customalerting/rule_client.go

@@ -0,0 +1,173 @@
+package customalerting
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/pkg/errors"
+	"net/http"
+)
+import "github.com/prometheus/client_golang/api"
+
+const (
+	apiPrefix      = "/api/v1"
+	epRules        = apiPrefix + "/rules"
+	statusAPIError = 422
+
+	statusSuccess status = "success"
+	statusError   status = "error"
+
+	ErrBadData     ErrorType = "bad_data"
+	ErrTimeout     ErrorType = "timeout"
+	ErrCanceled    ErrorType = "canceled"
+	ErrExec        ErrorType = "execution"
+	ErrBadResponse ErrorType = "bad_response"
+	ErrServer      ErrorType = "server_error"
+	ErrClient      ErrorType = "client_error"
+)
+
+type status string
+
+type ErrorType string
+
+type Error struct {
+	Type   ErrorType
+	Msg    string
+	Detail string
+}
+
+func (e *Error) Error() string {
+	return fmt.Sprintf("%s: %s", e.Type, e.Msg)
+}
+
+type response struct {
+	Status    status          `json:"status"`
+	Data      json.RawMessage `json:"data,omitempty"`
+	ErrorType ErrorType       `json:"errorType,omitempty"`
+	Error     string          `json:"error,omitempty"`
+	Warnings  []string        `json:"warnings,omitempty"`
+}
+
+type RuleClient interface {
+	PrometheusRules(ctx context.Context) ([]*RuleGroup, error)
+	ThanosRules(ctx context.Context) ([]*RuleGroup, error)
+}
+
+type ruleClient struct {
+	prometheus  api.Client
+	thanosruler api.Client
+}
+
+func (c *ruleClient) PrometheusRules(ctx context.Context) ([]*RuleGroup, error) {
+	if c.prometheus != nil {
+		return c.rules(c.prometheus, ctx)
+	}
+	return nil, nil
+}
+
+func (c *ruleClient) ThanosRules(ctx context.Context) ([]*RuleGroup, error) {
+	if c.thanosruler != nil {
+		return c.rules(c.thanosruler, ctx)
+	}
+	return nil, nil
+}
+
+func (c *ruleClient) rules(client api.Client, ctx context.Context) ([]*RuleGroup, error) {
+	u := client.URL(epRules, nil)
+	q := u.Query()
+	q.Add("type", "alert")
+	u.RawQuery = q.Encode()
+
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "error creating request: ")
+	}
+
+	_, body, _, err := c.do(client, ctx, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "error doing request: ")
+	}
+
+	var result struct {
+		Groups []*RuleGroup
+	}
+	err = json.Unmarshal(body, &result)
+	if err != nil {
+		return nil, errors.Wrap(err, "")
+	}
+	return result.Groups, nil
+}
+
+func (c *ruleClient) do(client api.Client, ctx context.Context, req *http.Request) (*http.Response, []byte, []string, error) {
+	resp, body, e := client.Do(ctx, req)
+	if e != nil {
+		return resp, body, nil, e
+	}
+
+	code := resp.StatusCode
+
+	if code/100 != 2 && !apiError(code) {
+		errorType, errorMsg := errorTypeAndMsgFor(resp)
+		return resp, body, nil, &Error{
+			Type:   errorType,
+			Msg:    errorMsg,
+			Detail: string(body),
+		}
+	}
+
+	var result response
+	if http.StatusNoContent != code {
+		if jsonErr := json.Unmarshal(body, &result); jsonErr != nil {
+			return resp, body, nil, &Error{
+				Type: ErrBadResponse,
+				Msg:  jsonErr.Error(),
+			}
+		}
+	}
+
+	var err error
+	if apiError(code) && result.Status == "success" {
+		err = &Error{
+			Type: ErrBadResponse,
+			Msg:  "inconsistent body for response code",
+		}
+	}
+	if result.Status == "error" {
+		err = &Error{
+			Type: result.ErrorType,
+			Msg:  result.Error,
+		}
+	}
+
+	return resp, []byte(result.Data), result.Warnings, err
+}
+
+func errorTypeAndMsgFor(resp *http.Response) (ErrorType, string) {
+	switch resp.StatusCode / 100 {
+	case 4:
+		return ErrClient, fmt.Sprintf("client error: %d", resp.StatusCode)
+	case 5:
+		return ErrServer, fmt.Sprintf("server error: %d", resp.StatusCode)
+	}
+	return ErrBadResponse, fmt.Sprintf("bad response code %d", resp.StatusCode)
+}
+
+func apiError(code int) bool {
+	// These are the codes that rule server sends when it returns an error.
+	return code == statusAPIError || code == http.StatusBadRequest ||
+		code == http.StatusServiceUnavailable || code == http.StatusInternalServerError
+}
+
+func NewRuleClient(options *Options) (RuleClient, error) {
+	var (
+		c ruleClient
+		e error
+	)
+	if options.PrometheusEndpoint != "" {
+		c.prometheus, e = api.NewClient(api.Config{Address: options.PrometheusEndpoint})
+	}
+	if options.ThanosRulerEndpoint != "" {
+		c.thanosruler, e = api.NewClient(api.Config{Address: options.ThanosRulerEndpoint})
+	}
+	return &c, e
+}

pkg/simple/client/customalerting/rule_client_test.go

@@ -0,0 +1,100 @@
+package customalerting
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestListRules(t *testing.T) {
+	var tests = []struct {
+		description string
+		fakeCode    int
+		fakeResp    string
+		expectError bool
+	}{{
+		description: "list alerting rules from prometheus endpoint",
+		expectError: false,
+		fakeCode:    200,
+		fakeResp: `
+{
+    "status": "success",
+    "data": {
+        "groups": [
+            {
+                "name": "kubernetes-resources",
+                "file": "/etc/prometheus/rules/prometheus-k8s-rulefiles-0/kubesphere-monitoring-system-prometheus-k8s-rules.yaml",
+                "rules": [
+                    {
+                        "state": "firing",
+                        "name": "KubeCPUOvercommit",
+                        "query": "sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum) / sum(kube_node_status_allocatable_cpu_cores) > (count(kube_node_status_allocatable_cpu_cores) - 1) / count(kube_node_status_allocatable_cpu_cores)",
+                        "duration": 300,
+                        "labels": {
+                            "severity": "warning"
+                        },
+                        "annotations": {
+                            "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.",
+                            "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
+                        },
+                        "alerts": [
+                            {
+                                "labels": {
+                                    "alertname": "KubeCPUOvercommit",
+                                    "severity": "warning"
+                                },
+                                "annotations": {
+                                    "message": "Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.",
+                                    "runbook_url": "https://github.com/ kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit"
+                                },
+                                "state": "firing",
+                                "activeAt": "2020-09-22T06:18:47.55260138Z",
+                                "value": "4.405e-01"
+                            }
+                        ],
+                        "health": "ok",
+                        "evaluationTime": 0.000894038,
+                        "lastEvaluation": "2020-09-22T08:57:17.566233983Z",
+                        "type": "alerting"
+                    }
+                ]
+            }
+        ]
+    }
+}
+`,
+	}}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			mock := MockService(epRules, test.fakeCode, test.fakeResp)
+			defer mock.Close()
+			c, e := NewRuleClient(&Options{PrometheusEndpoint: mock.URL})
+			if e != nil {
+				t.Fatal(e)
+			}
+			rgs, e := c.PrometheusRules(context.TODO())
+			if test.expectError {
+			} else {
+				if e != nil {
+					t.Fatal(e)
+				} else if len(rgs) == 1 && len(rgs[0].Rules) == 1 {
+
+				} else {
+					t.Fatalf("expect %d group and %d rule but got %d group and %d rule", 1, 1, len(rgs), len(rgs[0].Rules))
+				}
+			}
+
+		})
+	}
+}
+
+func MockService(pattern string, fakeCode int, fakeResp string) *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc(pattern, func(res http.ResponseWriter, req *http.Request) {
+		res.WriteHeader(fakeCode)
+		res.Write([]byte(fakeResp))
+	})
+	return httptest.NewServer(mux)
+}

pkg/simple/client/customalerting/types.go

@@ -0,0 +1,40 @@
+package customalerting
+
+import (
+	"time"
+)
+
+type RuleGroup struct {
+	Name           string          `json:"name"`
+	File           string          `json:"file"`
+	Rules          []*AlertingRule `json:"rules"`
+	Interval       float64         `json:"interval"`
+	EvaluationTime float64         `json:"evaluationTime"`
+	LastEvaluation *time.Time      `json:"lastEvaluation"`
+}
+
+type AlertingRule struct {
+	// State can be "pending", "firing", "inactive".
+	State       string            `json:"state"`
+	Name        string            `json:"name"`
+	Query       string            `json:"query"`
+	Duration    float64           `json:"duration"`
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	Alerts      []*Alert          `json:"alerts"`
+	// Health can be "ok", "err", "unknown".
+	Health         string     `json:"health"`
+	LastError      string     `json:"lastError,omitempty"`
+	EvaluationTime float64    `json:"evaluationTime"`
+	LastEvaluation *time.Time `json:"lastEvaluation"`
+	// Type of an alertingRule is always "alerting".
+	Type string `json:"type"`
+}
+
+type Alert struct {
+	Labels      map[string]string `json:"labels"`
+	Annotations map[string]string `json:"annotations"`
+	State       string            `json:"state"`
+	ActiveAt    *time.Time        `json:"activeAt,omitempty"`
+	Value       string            `json:"value"`
+}

pkg/simple/client/k8s/fake_client.go

@@ -18,6 +18,7 @@ package k8s
 
 import (
 	snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned"
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	istioclient "istio.io/client-go/pkg/clientset/versioned"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/discovery"
@@ -42,6 +43,8 @@ type FakeClient struct {
 
 	ApiExtensionClient apiextensionsclient.Interface
 
+	prometheusClient promresourcesclient.Interface
+
 	MasterURL string
 
 	KubeConfig *rest.Config
@@ -50,7 +53,8 @@ type FakeClient struct {
 func NewFakeClientSets(k8sClient kubernetes.Interface, discoveryClient *discovery.DiscoveryClient,
 	kubeSphereClient kubesphere.Interface,
 	istioClient istioclient.Interface, snapshotClient snapshotclient.Interface,
-	apiextensionsclient apiextensionsclient.Interface, masterURL string, kubeConfig *rest.Config) Client {
+	apiextensionsclient apiextensionsclient.Interface, prometheusClient promresourcesclient.Interface,
+	masterURL string, kubeConfig *rest.Config) Client {
 	return &FakeClient{
 		K8sClient:          k8sClient,
 		DiscoveryClient:    discoveryClient,
@@ -58,6 +62,7 @@ func NewFakeClientSets(k8sClient kubernetes.Interface, discoveryClient *discover
 		IstioClient:        istioClient,
 		SnapshotClient:     snapshotClient,
 		ApiExtensionClient: apiextensionsclient,
+		prometheusClient:   prometheusClient,
 		MasterURL:          masterURL,
 		KubeConfig:         kubeConfig,
 	}
@@ -87,6 +92,10 @@ func (n *FakeClient) Discovery() discovery.DiscoveryInterface {
 	return n.DiscoveryClient
 }
 
+func (n *FakeClient) Prometheus() promresourcesclient.Interface {
+	return n.prometheusClient
+}
+
 func (n *FakeClient) Master() string {
 	return n.MasterURL
 }

pkg/simple/client/k8s/kubernetes.go

@@ -17,6 +17,7 @@ limitations under the License.
 package k8s
 
 import (
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned"
 	istioclient "istio.io/client-go/pkg/clientset/versioned"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -35,6 +36,7 @@ type Client interface {
 	Snapshot() snapshotclient.Interface
 	ApiExtensions() apiextensionsclient.Interface
 	Discovery() discovery.DiscoveryInterface
+	Prometheus() promresourcesclient.Interface
 	Master() string
 	Config() *rest.Config
 }
@@ -55,6 +57,8 @@ type kubernetesClient struct {
 
 	apiextensions apiextensionsclient.Interface
 
+	prometheus promresourcesclient.Interface
+
 	master string
 
 	config *rest.Config
@@ -77,6 +81,7 @@ func NewKubernetesClientOrDie(options *KubernetesOptions) Client {
 		istio:           istioclient.NewForConfigOrDie(config),
 		snapshot:        snapshotclient.NewForConfigOrDie(config),
 		apiextensions:   apiextensionsclient.NewForConfigOrDie(config),
+		prometheus:      promresourcesclient.NewForConfigOrDie(config),
 		master:          config.Host,
 		config:          config,
 	}
@@ -135,6 +140,11 @@ func NewKubernetesClient(options *KubernetesOptions) (Client, error) {
 		return nil, err
 	}
 
+	k.prometheus, err = promresourcesclient.NewForConfig(config)
+	if err != nil {
+		return nil, err
+	}
+
 	k.master = options.Master
 	k.config = config
 
@@ -165,6 +175,10 @@ func (k *kubernetesClient) ApiExtensions() apiextensionsclient.Interface {
 	return k.apiextensions
 }
 
+func (k *kubernetesClient) Prometheus() promresourcesclient.Interface {
+	return k.prometheus
+}
+
 // master address used to generate kubeconfig for downloading
 func (k *kubernetesClient) Master() string {
 	return k.master

pkg/simple/client/k8s/null_client.go

@@ -17,6 +17,7 @@ limitations under the License.
 package k8s
 
 import (
+	promresourcesclient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
 	snapshotclient "github.com/kubernetes-csi/external-snapshotter/client/v3/clientset/versioned"
 	istio "istio.io/client-go/pkg/clientset/versioned"
 	apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -57,6 +58,10 @@ func (n nullClient) Discovery() discovery.DiscoveryInterface {
 	return nil
 }
 
+func (n *nullClient) Prometheus() promresourcesclient.Interface {
+	return nil
+}
+
 func (n nullClient) Master() string {
 	return ""
 }