fix notification triggering again after alerting rule changes

Signed-off-by: junot <junotxiang@yunify.com>

pkg/models/alerting/alerting.go

@@ -24,8 +24,6 @@ import (
 
 const (
 	rulerNamespace = constants.KubeSphereMonitoringNamespace
-
-	customRuleResourceLabelKeyLevel = "custom-alerting-rule-level"
 )
 
 // Operator contains all operations to alerting rules. The operations may involve manipulations of prometheusrule
@@ -254,7 +252,7 @@ func (o *operator) listCustomAlertingRules(ctx context.Context, ruleNamespace *c
 	}
 
 	resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace,
-		labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}))
+		labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}))
 	if err != nil {
 		return nil, err
 	}
@@ -287,7 +285,7 @@ func (o *operator) getCustomAlertingRule(ctx context.Context, ruleNamespace *cor
 	}
 
 	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace,
-		labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}), ruleName)
+		labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}), ruleName)
 	if err != nil {
 		return nil, err
 	}
@@ -426,14 +424,14 @@ func (o *operator) CreateCustomAlertingRule(ctx context.Context, namespace strin
 		}
 		rule.Query = expr
 	}
-	ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
+	ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
 
 	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
 	if err != nil {
 		return err
 	}
 
-	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
 	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
 	if err != nil {
 		return err
@@ -491,14 +489,14 @@ func (o *operator) UpdateCustomAlertingRule(ctx context.Context, namespace, name
 		}
 		rule.Query = expr
 	}
-	ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
+	ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
 
 	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
 	if err != nil {
 		return err
 	}
 
-	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
 	resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
 	if err != nil {
 		return err
@@ -550,7 +548,7 @@ func (o *operator) DeleteCustomAlertingRule(ctx context.Context, namespace, name
 		return err
 	}
 
-	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
 	resourceRules, err := o.resourceRuleCache.GetRuleByIdOrName(ruler, ruleNamespace, extraRuleResourceSelector, name)
 	if err != nil {
 		return err
@@ -602,13 +600,13 @@ func (o *operator) CreateOrUpdateCustomAlertingRules(ctx context.Context, namesp
 	} else {
 		level = v2alpha1.RuleLevelNamespace
 	}
-	ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
+	ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
 	ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
 	if err != nil {
 		return nil, err
 	}
 
-	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
 
 	resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector)
 	if err != nil {
@@ -766,7 +764,7 @@ func (o *operator) DeleteCustomAlertingRules(ctx context.Context, namespace stri
 		return nil, err
 	}
 
-	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
+	extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
 	resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector)
 	if err != nil {
 		return nil, err

pkg/models/alerting/rules/ruler.go

@@ -26,6 +26,8 @@ import (
 const (
 	customAlertingRuleResourcePrefix = "custom-alerting-rule-"
 
+	CustomRuleResourceLabelKeyLevel = "custom-alerting-rule-level"
+
 	customRuleGroupDefaultPrefix = "alerting.custom.defaults."
 	customRuleGroupSize          = 20
 )
@@ -144,6 +146,14 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error
 		}
 	}
 
+	var referNs = r.Namespace
+	if len(r.Labels) > 0 {
+		if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok &&
+			v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster {
+			referNs = ""
+		}
+	}
+
 	addRules := func(g *promresourcesv1.RuleGroup) bool {
 		var add bool
 		var num = customRuleGroupSize - len(g.Rules)
@@ -157,7 +167,7 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error
 					// Because Prometheus may migrate information such as alerts from the old rule into the new rule
 					// when updating a rule within a group.
 					if _, ok := gNames[g.Name]; !ok {
-						g.Rules = append(g.Rules, rule.Rule)
+						appendRules(g, referNs, rule.Rule)
 						num--
 						delete(ruleMap, name)
 						add = true
@@ -238,6 +248,14 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
 		spec = new(promresourcesv1.PrometheusRuleSpec)
 	}
 
+	var referNs = r.Namespace
+	if len(r.Labels) > 0 {
+		if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok &&
+			v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster {
+			referNs = ""
+		}
+	}
+
 	// For the rules that have specific group names, add them to the matched groups.
 	// For the rules that do not specify group names, add them to the automatically generated groups until the limit is reached.
 	for i, g := range spec.Groups {
@@ -253,7 +271,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
 
 		if !groupedRulesDrained {
 			if _, ok := groupedRules[gName]; ok {
-				spec.Groups[i].Rules = append(spec.Groups[i].Rules, groupedRules[gName]...)
+				appendRules(&spec.Groups[i], referNs, groupedRules[gName]...)
 				delete(groupedRules, gName)
 				commit = true
 			}
@@ -275,7 +293,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
 				if stop = cursor + num; stop > len(unGroupedRules) {
 					stop = len(unGroupedRules)
 				}
-				spec.Groups[i].Rules = append(spec.Groups[i].Rules, unGroupedRules[cursor:stop]...)
+				appendRules(&spec.Groups[i], referNs, unGroupedRules[cursor:stop]...)
 				cursor = stop
 				commit = true
 			}
@@ -288,7 +306,9 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
 		if len(rules) == 0 {
 			continue
 		}
-		spec.Groups = append(spec.Groups, promresourcesv1.RuleGroup{Name: gName, Rules: rules})
+		g := promresourcesv1.RuleGroup{Name: gName}
+		appendRules(&g, referNs, rules...)
+		spec.Groups = append(spec.Groups, g)
 		commit = true
 	}
 	for groupMax++; cursor < len(rules); groupMax++ {
@@ -297,7 +317,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
 		if stop = cursor + customRuleGroupSize; stop > len(unGroupedRules) {
 			stop = len(unGroupedRules)
 		}
-		g.Rules = append(g.Rules, unGroupedRules[cursor:stop]...)
+		appendRules(&g, referNs, unGroupedRules[cursor:stop]...)
 		spec.Groups = append(spec.Groups, g)
 		cursor = stop
 		commit = true
@@ -329,6 +349,25 @@ func (r *ruleResource) commit(ctx context.Context, prometheusResourceClient prom
 	return nil
 }
 
+func appendRules(group *promresourcesv1.RuleGroup, referNs string, rules ...promresourcesv1.Rule) {
+	if len(rules) == 0 {
+		return
+	}
+	for i := range rules {
+		r := &rules[i]
+		id := GenResourceRuleIdIgnoreFormat(group.Name, r)
+		if r.Labels == nil {
+			r.Labels = make(map[string]string)
+		}
+		var v = id
+		if referNs != "" {
+			v = referNs + "-" + id
+		}
+		r.Labels[LabelKeyRuleId] = v
+	}
+	group.Rules = append(group.Rules, rules...)
+}
+
 type PrometheusRuler struct {
 	resource *promresourcesv1.Prometheus
 	informer prominformersv1.PrometheusRuleInformer

pkg/models/alerting/rules/utils.go

@@ -30,6 +30,8 @@ const (
 	LabelKeyThanosRulerReplica = "thanos_ruler_replica"
 	LabelKeyPrometheusReplica  = "prometheus_replica"
 
+	LabelKeyRuleId = "rule_id"
+
 	LabelKeyAlertType   = "alerttype"
 	LabelValueAlertType = "metric"
 )
@@ -89,6 +91,9 @@ func GenResourceRuleIdIgnoreFormat(group string, rule *promresourcesv1.Rule) str
 
 	lbls := make(map[string]string)
 	for k, v := range rule.Labels {
+		if k == LabelKeyRuleId {
+			continue
+		}
 		lbls[k] = v
 	}
 	lbls[LabelKeyInternalRuleGroup] = group
@@ -127,6 +132,9 @@ func GenEndpointRuleId(group string, epRule *alerting.AlertingRule,
 
 	lbls := make(map[string]string)
 	for k, v := range labelsMap {
+		if k == LabelKeyRuleId {
+			continue
+		}
 		lbls[k] = v
 	}
 	lbls[LabelKeyInternalRuleGroup] = group