diff --git a/pkg/models/alerting/alerting.go b/pkg/models/alerting/alerting.go index 9f8f0ace9..1c191289a 100644 --- a/pkg/models/alerting/alerting.go +++ b/pkg/models/alerting/alerting.go @@ -24,8 +24,6 @@ import ( const ( rulerNamespace = constants.KubeSphereMonitoringNamespace - - customRuleResourceLabelKeyLevel = "custom-alerting-rule-level" ) // Operator contains all operations to alerting rules. The operations may involve manipulations of prometheusrule @@ -254,7 +252,7 @@ func (o *operator) listCustomAlertingRules(ctx context.Context, ruleNamespace *c } resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, - labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})) + labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})) if err != nil { return nil, err } @@ -287,7 +285,7 @@ func (o *operator) getCustomAlertingRule(ctx context.Context, ruleNamespace *cor } resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, - labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}), ruleName) + labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}), ruleName) if err != nil { return nil, err } @@ -426,14 +424,14 @@ func (o *operator) CreateCustomAlertingRule(ctx context.Context, namespace strin } rule.Query = expr } - ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level) + ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level) ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace) if err != nil { return err } - extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}) + extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}) resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name) if err != nil { return err @@ -491,14 +489,14 @@ func (o *operator) UpdateCustomAlertingRule(ctx context.Context, namespace, name } rule.Query = expr } - ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level) + ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level) ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace) if err != nil { return err } - extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}) + extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}) resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name) if err != nil { return err @@ -550,7 +548,7 @@ func (o *operator) DeleteCustomAlertingRule(ctx context.Context, namespace, name return err } - extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}) + extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}) resourceRules, err := o.resourceRuleCache.GetRuleByIdOrName(ruler, ruleNamespace, extraRuleResourceSelector, name) if err != nil { return err @@ -602,13 +600,13 @@ func (o *operator) CreateOrUpdateCustomAlertingRules(ctx context.Context, namesp } else { level = v2alpha1.RuleLevelNamespace } - ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level) + ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level) ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace) if err != nil { return nil, err } - extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}) + extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}) resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector) if err != nil { @@ -766,7 +764,7 @@ func (o *operator) DeleteCustomAlertingRules(ctx context.Context, namespace stri return nil, err } - extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}) + extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}) resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector) if err != nil { return nil, err diff --git a/pkg/models/alerting/rules/ruler.go b/pkg/models/alerting/rules/ruler.go index 12766e42b..49094a497 100644 --- a/pkg/models/alerting/rules/ruler.go +++ b/pkg/models/alerting/rules/ruler.go @@ -26,6 +26,8 @@ import ( const ( customAlertingRuleResourcePrefix = "custom-alerting-rule-" + CustomRuleResourceLabelKeyLevel = "custom-alerting-rule-level" + customRuleGroupDefaultPrefix = "alerting.custom.defaults." customRuleGroupSize = 20 ) @@ -144,6 +146,14 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error } } + var referNs = r.Namespace + if len(r.Labels) > 0 { + if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok && + v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster { + referNs = "" + } + } + addRules := func(g *promresourcesv1.RuleGroup) bool { var add bool var num = customRuleGroupSize - len(g.Rules) @@ -157,7 +167,7 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error // Because Prometheus may migrate information such as alerts from the old rule into the new rule // when updating a rule within a group. if _, ok := gNames[g.Name]; !ok { - g.Rules = append(g.Rules, rule.Rule) + appendRules(g, referNs, rule.Rule) num-- delete(ruleMap, name) add = true @@ -238,6 +248,14 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) { spec = new(promresourcesv1.PrometheusRuleSpec) } + var referNs = r.Namespace + if len(r.Labels) > 0 { + if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok && + v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster { + referNs = "" + } + } + // For the rules that have specific group names, add them to the matched groups. // For the rules that do not specify group names, add them to the automatically generated groups until the limit is reached. for i, g := range spec.Groups { @@ -253,7 +271,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) { if !groupedRulesDrained { if _, ok := groupedRules[gName]; ok { - spec.Groups[i].Rules = append(spec.Groups[i].Rules, groupedRules[gName]...) + appendRules(&spec.Groups[i], referNs, groupedRules[gName]...) delete(groupedRules, gName) commit = true } @@ -275,7 +293,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) { if stop = cursor + num; stop > len(unGroupedRules) { stop = len(unGroupedRules) } - spec.Groups[i].Rules = append(spec.Groups[i].Rules, unGroupedRules[cursor:stop]...) + appendRules(&spec.Groups[i], referNs, unGroupedRules[cursor:stop]...) cursor = stop commit = true } @@ -288,7 +306,9 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) { if len(rules) == 0 { continue } - spec.Groups = append(spec.Groups, promresourcesv1.RuleGroup{Name: gName, Rules: rules}) + g := promresourcesv1.RuleGroup{Name: gName} + appendRules(&g, referNs, rules...) + spec.Groups = append(spec.Groups, g) commit = true } for groupMax++; cursor < len(rules); groupMax++ { @@ -297,7 +317,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) { if stop = cursor + customRuleGroupSize; stop > len(unGroupedRules) { stop = len(unGroupedRules) } - g.Rules = append(g.Rules, unGroupedRules[cursor:stop]...) + appendRules(&g, referNs, unGroupedRules[cursor:stop]...) spec.Groups = append(spec.Groups, g) cursor = stop commit = true @@ -329,6 +349,25 @@ func (r *ruleResource) commit(ctx context.Context, prometheusResourceClient prom return nil } +func appendRules(group *promresourcesv1.RuleGroup, referNs string, rules ...promresourcesv1.Rule) { + if len(rules) == 0 { + return + } + for i := range rules { + r := &rules[i] + id := GenResourceRuleIdIgnoreFormat(group.Name, r) + if r.Labels == nil { + r.Labels = make(map[string]string) + } + var v = id + if referNs != "" { + v = referNs + "-" + id + } + r.Labels[LabelKeyRuleId] = v + } + group.Rules = append(group.Rules, rules...) +} + type PrometheusRuler struct { resource *promresourcesv1.Prometheus informer prominformersv1.PrometheusRuleInformer diff --git a/pkg/models/alerting/rules/utils.go b/pkg/models/alerting/rules/utils.go index c385ef387..cc5359808 100644 --- a/pkg/models/alerting/rules/utils.go +++ b/pkg/models/alerting/rules/utils.go @@ -30,6 +30,8 @@ const ( LabelKeyThanosRulerReplica = "thanos_ruler_replica" LabelKeyPrometheusReplica = "prometheus_replica" + LabelKeyRuleId = "rule_id" + LabelKeyAlertType = "alerttype" LabelValueAlertType = "metric" ) @@ -89,6 +91,9 @@ func GenResourceRuleIdIgnoreFormat(group string, rule *promresourcesv1.Rule) str lbls := make(map[string]string) for k, v := range rule.Labels { + if k == LabelKeyRuleId { + continue + } lbls[k] = v } lbls[LabelKeyInternalRuleGroup] = group @@ -127,6 +132,9 @@ func GenEndpointRuleId(group string, epRule *alerting.AlertingRule, lbls := make(map[string]string) for k, v := range labelsMap { + if k == LabelKeyRuleId { + continue + } lbls[k] = v } lbls[LabelKeyInternalRuleGroup] = group