admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3706 from junotx/ca

Browse Source 
fix notification triggering again after alerting rule changes
This commit is contained in:
KubeSphere CI Bot
2021-04-09 10:27:59 +08:00
committed by GitHub
parent 0688ca4472 5346c6b1ae
commit bb440927d2
3 changed files with 62 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/models/alerting/alerting.go
View File

@@ -24,8 +24,6 @@ import (
const (
rulerNamespace = constants.KubeSphereMonitoringNamespace
customRuleResourceLabelKeyLevel = "custom-alerting-rule-level"
)
// Operator contains all operations to alerting rules. The operations may involve manipulations of prometheusrule
@@ -254,7 +252,7 @@ func (o *operator) listCustomAlertingRules(ctx context.Context, ruleNamespace *c
}
resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace,
labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}))
labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}))
if err != nil {
return nil, err
}
@@ -287,7 +285,7 @@ func (o *operator) getCustomAlertingRule(ctx context.Context, ruleNamespace *cor
}
resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace,
labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)}), ruleName)
labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)}), ruleName)
if err != nil {
return nil, err
}
@@ -426,14 +424,14 @@ func (o *operator) CreateCustomAlertingRule(ctx context.Context, namespace strin
}
rule.Query = expr
}
ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
if err != nil {
return err
}
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
if err != nil {
return err
@@ -491,14 +489,14 @@ func (o *operator) UpdateCustomAlertingRule(ctx context.Context, namespace, name
}
rule.Query = expr
}
ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
if err != nil {
return err
}
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
resourceRule, err := o.resourceRuleCache.GetRule(ruler, ruleNamespace, extraRuleResourceSelector, rule.Name)
if err != nil {
return err
@@ -550,7 +548,7 @@ func (o *operator) DeleteCustomAlertingRule(ctx context.Context, namespace, name
return err
}
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
resourceRules, err := o.resourceRuleCache.GetRuleByIdOrName(ruler, ruleNamespace, extraRuleResourceSelector, name)
if err != nil {
return err
@@ -602,13 +600,13 @@ func (o *operator) CreateOrUpdateCustomAlertingRules(ctx context.Context, namesp
} else {
level = v2alpha1.RuleLevelNamespace
}
ruleResourceLabels[customRuleResourceLabelKeyLevel] = string(level)
ruleResourceLabels[rules.CustomRuleResourceLabelKeyLevel] = string(level)
ruleNamespace, err := o.namespaceInformer.Lister().Get(namespace)
if err != nil {
return nil, err
}
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector)
if err != nil {
@@ -766,7 +764,7 @@ func (o *operator) DeleteCustomAlertingRules(ctx context.Context, namespace stri
return nil, err
}
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{customRuleResourceLabelKeyLevel: string(level)})
extraRuleResourceSelector := labels.SelectorFromSet(labels.Set{rules.CustomRuleResourceLabelKeyLevel: string(level)})
resourceRulesMap, err := o.resourceRuleCache.ListRules(ruler, ruleNamespace, extraRuleResourceSelector)
if err != nil {
return nil, err

49
pkg/models/alerting/rules/ruler.go
View File

@@ -26,6 +26,8 @@ import (
const (
customAlertingRuleResourcePrefix = "custom-alerting-rule-"
CustomRuleResourceLabelKeyLevel = "custom-alerting-rule-level"
customRuleGroupDefaultPrefix = "alerting.custom.defaults."
customRuleGroupSize = 20
)
@@ -144,6 +146,14 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error
}
}
var referNs = r.Namespace
if len(r.Labels) > 0 {
if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok &&
v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster {
referNs = ""
}
}
addRules := func(g *promresourcesv1.RuleGroup) bool {
var add bool
var num = customRuleGroupSize - len(g.Rules)
@@ -157,7 +167,7 @@ func (r *ruleResource) updateAlertingRules(rules ...*RuleWithGroup) (bool, error
// Because Prometheus may migrate information such as alerts from the old rule into the new rule
// when updating a rule within a group.
if _, ok := gNames[g.Name]; !ok {
g.Rules = append(g.Rules, rule.Rule)
appendRules(g, referNs, rule.Rule)
num--
delete(ruleMap, name)
add = true
@@ -238,6 +248,14 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
spec = new(promresourcesv1.PrometheusRuleSpec)
}
var referNs = r.Namespace
if len(r.Labels) > 0 {
if l, ok := r.Labels[CustomRuleResourceLabelKeyLevel]; ok &&
v2alpha1.RuleLevel(l) == v2alpha1.RuleLevelCluster {
referNs = ""
}
}
// For the rules that have specific group names, add them to the matched groups.
// For the rules that do not specify group names, add them to the automatically generated groups until the limit is reached.
for i, g := range spec.Groups {
@@ -253,7 +271,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
if !groupedRulesDrained {
if _, ok := groupedRules[gName]; ok {
spec.Groups[i].Rules = append(spec.Groups[i].Rules, groupedRules[gName]...)
appendRules(&spec.Groups[i], referNs, groupedRules[gName]...)
delete(groupedRules, gName)
commit = true
}
@@ -275,7 +293,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
if stop = cursor + num; stop > len(unGroupedRules) {
stop = len(unGroupedRules)
}
spec.Groups[i].Rules = append(spec.Groups[i].Rules, unGroupedRules[cursor:stop]...)
appendRules(&spec.Groups[i], referNs, unGroupedRules[cursor:stop]...)
cursor = stop
commit = true
}
@@ -288,7 +306,9 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
if len(rules) == 0 {
continue
}
spec.Groups = append(spec.Groups, promresourcesv1.RuleGroup{Name: gName, Rules: rules})
g := promresourcesv1.RuleGroup{Name: gName}
appendRules(&g, referNs, rules...)
spec.Groups = append(spec.Groups, g)
commit = true
}
for groupMax++; cursor < len(rules); groupMax++ {
@@ -297,7 +317,7 @@ func (r *ruleResource) addAlertingRules(rules ...*RuleWithGroup) (bool, error) {
if stop = cursor + customRuleGroupSize; stop > len(unGroupedRules) {
stop = len(unGroupedRules)
}
g.Rules = append(g.Rules, unGroupedRules[cursor:stop]...)
appendRules(&g, referNs, unGroupedRules[cursor:stop]...)
spec.Groups = append(spec.Groups, g)
cursor = stop
commit = true
@@ -329,6 +349,25 @@ func (r *ruleResource) commit(ctx context.Context, prometheusResourceClient prom
return nil
}
func appendRules(group *promresourcesv1.RuleGroup, referNs string, rules ...promresourcesv1.Rule) {
if len(rules) == 0 {
return
}
for i := range rules {
r := &rules[i]
id := GenResourceRuleIdIgnoreFormat(group.Name, r)
if r.Labels == nil {
r.Labels = make(map[string]string)
}
var v = id
if referNs != "" {
v = referNs + "-" + id
}
r.Labels[LabelKeyRuleId] = v
}
group.Rules = append(group.Rules, rules...)
}
type PrometheusRuler struct {
resource *promresourcesv1.Prometheus
informer prominformersv1.PrometheusRuleInformer

8
pkg/models/alerting/rules/utils.go
View File

@@ -30,6 +30,8 @@ const (
LabelKeyThanosRulerReplica = "thanos_ruler_replica"
LabelKeyPrometheusReplica = "prometheus_replica"
LabelKeyRuleId = "rule_id"
LabelKeyAlertType = "alerttype"
LabelValueAlertType = "metric"
)
@@ -89,6 +91,9 @@ func GenResourceRuleIdIgnoreFormat(group string, rule *promresourcesv1.Rule) str
lbls := make(map[string]string)
for k, v := range rule.Labels {
if k == LabelKeyRuleId {
continue
}
lbls[k] = v
}
lbls[LabelKeyInternalRuleGroup] = group
@@ -127,6 +132,9 @@ func GenEndpointRuleId(group string, epRule *alerting.AlertingRule,
lbls := make(map[string]string)
for k, v := range labelsMap {
if k == LabelKeyRuleId {
continue
}
lbls[k] = v
}
lbls[LabelKeyInternalRuleGroup] = group