aa0da0c80d
add rule_type label to indicate rule type (#472) Signed-off-by: junot <junotxiang@kubesphere.io>
429 lines
13 KiB
Go
429 lines
13 KiB
Go
/*
|
|
Copyright 2019 The KubeSphere Authors.
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package alerting
|
|
|
|
import (
|
|
"context"
|
|
"reflect"
|
|
"sort"
|
|
"strconv"
|
|
|
|
"github.com/go-logr/logr"
|
|
"github.com/pkg/errors"
|
|
"github.com/prometheus-community/prom-label-proxy/injectproxy"
|
|
promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
|
|
promlabels "github.com/prometheus/prometheus/model/labels"
|
|
"github.com/prometheus/prometheus/promql/parser"
|
|
"gopkg.in/yaml.v2"
|
|
corev1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
alertingv2beta1 "kubesphere.io/api/alerting/v2beta1"
|
|
|
|
"kubesphere.io/kubesphere/pkg/constants"
|
|
)
|
|
|
|
const (
|
|
RuleLevelNamesapce RuleLevel = "namespace"
|
|
RuleLevelCluster RuleLevel = "cluster"
|
|
RuleLevelGlobal RuleLevel = "global"
|
|
|
|
RuleTypeTemplate RuleType = "template" // for template rule configured by exprBuilder to build expression
|
|
RuleTypeCustom RuleType = "custom" // for custom rule configured by direct expression
|
|
|
|
// for rule.labels
|
|
RuleLabelKeyRuleLevel = "rule_level"
|
|
RuleLabelKeyRuleGroup = "rule_group"
|
|
RuleLabelKeyRuleType = "rule_type"
|
|
RuleLabelKeyCluster = "cluster"
|
|
RuleLabelKeyNamespace = "namespace"
|
|
RuleLabelKeySeverity = "severity"
|
|
RuleLabelKeyAlertType = "alerttype"
|
|
RuleLabelValueAlertTypeMetric = "metric"
|
|
|
|
// label keys in RuleGroup/ClusterRuleGroup/GlobalRuleGroup.metadata.labels
|
|
SourceGroupResourceLabelKeyEnable = "alerting.kubesphere.io/enable"
|
|
SourceGroupResourceLabelValueEnableTrue = "true"
|
|
SourceGroupResourceLabelValueEnableFalse = "false"
|
|
|
|
// for PrometheusRule.metadata.labels
|
|
PrometheusRuleResourceLabelKeyOwnerNamespace = "alerting.kubesphere.io/owner_namespace"
|
|
PrometheusRuleResourceLabelKeyOwnerCluster = "alerting.kubesphere.io/owner_cluster"
|
|
PrometheusRuleResourceLabelKeyRuleLevel = "alerting.kubesphere.io/rule_level"
|
|
PrometheusRuleResourceLabelKeyBuiltin = "alerting.kubesphere.io/builtin"
|
|
PrometheusRuleResourceLabelValueBuiltinTrue = "true"
|
|
PrometheusRuleResourceLabelValueBuiltinFalse = "false"
|
|
|
|
// name prefix for PrometheusRule
|
|
PrometheusRulePrefix = "alertrules-"
|
|
PrometheusRulePrefixNamespaceLevel = PrometheusRulePrefix + "ns-"
|
|
PrometheusRulePrefixClusterLevel = PrometheusRulePrefix + "cl-"
|
|
PrometheusRulePrefixGlobalLevel = PrometheusRulePrefix + "gl-"
|
|
|
|
PrometheusRuleNamespace = constants.KubeSphereMonitoringNamespace
|
|
)
|
|
|
|
type RuleLevel string
|
|
|
|
type RuleType string
|
|
|
|
var maxConfigMapDataSize = int(float64(corev1.MaxSecretSize) * 0.5)
|
|
|
|
type enforceRuleFunc func(rule *promresourcesv1.Rule) error
|
|
|
|
type EnforceExprFunc func(expr string) (string, error)
|
|
|
|
var emptyEnforceExprFunc = func(expr string) (string, error) {
|
|
return expr, nil
|
|
}
|
|
|
|
func CreateEnforceExprFunc(enforceRuleMatchers []*promlabels.Matcher) EnforceExprFunc {
|
|
if len(enforceRuleMatchers) > 0 {
|
|
enforcer := injectproxy.NewEnforcer(false, enforceRuleMatchers...)
|
|
return func(expr string) (string, error) {
|
|
parsedExpr, err := parser.ParseExpr(expr)
|
|
if err != nil {
|
|
return expr, err
|
|
}
|
|
if err := enforcer.EnforceNode(parsedExpr); err != nil {
|
|
return expr, err
|
|
}
|
|
return parsedExpr.String(), nil
|
|
}
|
|
}
|
|
return emptyEnforceExprFunc
|
|
}
|
|
|
|
func createEnforceRuleFuncs(enforceRuleMatchers []*promlabels.Matcher, enforceRuleLabels map[string]string) []enforceRuleFunc {
|
|
var enforceFuncs []enforceRuleFunc
|
|
// enforce func for rule.expr
|
|
if len(enforceRuleMatchers) > 0 {
|
|
enforceExprFunc := CreateEnforceExprFunc(enforceRuleMatchers)
|
|
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
|
|
expr, err := enforceExprFunc(rule.Expr.String())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rule.Expr = intstr.FromString(expr)
|
|
return nil
|
|
})
|
|
}
|
|
// enforce func for rule.labels
|
|
if len(enforceRuleLabels) > 0 {
|
|
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
|
|
if rule.Labels == nil {
|
|
rule.Labels = make(map[string]string)
|
|
}
|
|
for n, v := range enforceRuleLabels {
|
|
rule.Labels[n] = v
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
return enforceFuncs
|
|
}
|
|
|
|
func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
|
|
commonEnforceFuncs ...enforceRuleFunc) ([]*promresourcesv1.RuleGroup, error) {
|
|
var rulegroups []*promresourcesv1.RuleGroup
|
|
|
|
convertRule := func(rule *alertingv2beta1.Rule, groupName string, enforceFuncs ...enforceRuleFunc) (*promresourcesv1.Rule, error) {
|
|
if rule.Disable { // ignoring disabled rule
|
|
return nil, nil
|
|
}
|
|
|
|
rule = rule.DeepCopy()
|
|
|
|
if rule.Labels == nil {
|
|
rule.Labels = make(map[string]string)
|
|
}
|
|
|
|
if rule.Severity != "" {
|
|
rule.Labels[RuleLabelKeySeverity] = string(rule.Severity)
|
|
}
|
|
|
|
prule := promresourcesv1.Rule{
|
|
Alert: rule.Alert,
|
|
For: promresourcesv1.Duration(rule.For),
|
|
Expr: rule.Expr,
|
|
Labels: rule.Labels,
|
|
Annotations: rule.Annotations,
|
|
}
|
|
|
|
enforceFuncs = append(enforceFuncs, commonEnforceFuncs...)
|
|
// enforce rule group label and alert type label
|
|
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
|
|
if rule.Labels == nil {
|
|
rule.Labels = make(map[string]string)
|
|
}
|
|
rule.Labels[RuleLabelKeyRuleGroup] = groupName
|
|
rule.Labels[RuleLabelKeyAlertType] = RuleLabelValueAlertTypeMetric
|
|
return nil
|
|
})
|
|
|
|
for _, f := range enforceFuncs {
|
|
if f == nil {
|
|
continue
|
|
}
|
|
err := f(&prule)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "alert: %s", rule.Alert)
|
|
}
|
|
}
|
|
|
|
return &prule, nil
|
|
}
|
|
|
|
switch list := groupList.(type) {
|
|
case *alertingv2beta1.RuleGroupList:
|
|
for _, group := range list.Items {
|
|
var prules []promresourcesv1.Rule
|
|
for _, rule := range group.Spec.Rules {
|
|
prule, err := convertRule(&rule.Rule, group.Name)
|
|
if err != nil {
|
|
log.WithValues("rulegroup", group.Namespace+"/"+group.Name).Error(err, "failed to convert")
|
|
continue
|
|
}
|
|
if prule != nil {
|
|
if rule.ExprBuilder != nil && rule.ExprBuilder.Workload != nil {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeTemplate)
|
|
} else {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeCustom)
|
|
}
|
|
prules = append(prules, *prule)
|
|
}
|
|
}
|
|
if len(prules) == 0 {
|
|
continue
|
|
}
|
|
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
|
Name: group.Name,
|
|
Interval: promresourcesv1.Duration(group.Spec.Interval),
|
|
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
|
Rules: prules,
|
|
})
|
|
}
|
|
case *alertingv2beta1.ClusterRuleGroupList:
|
|
for _, group := range list.Items {
|
|
var prules []promresourcesv1.Rule
|
|
for _, rule := range group.Spec.Rules {
|
|
prule, err := convertRule(&rule.Rule, group.Name)
|
|
if err != nil {
|
|
log.WithValues("clusterrulegroup", group.Name).Error(err, "failed to convert")
|
|
continue
|
|
}
|
|
if prule != nil {
|
|
if rule.ExprBuilder != nil && rule.ExprBuilder.Node != nil {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeTemplate)
|
|
} else {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeCustom)
|
|
}
|
|
prules = append(prules, *prule)
|
|
}
|
|
}
|
|
if len(prules) == 0 {
|
|
continue
|
|
}
|
|
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
|
Name: group.Name,
|
|
Interval: promresourcesv1.Duration(group.Spec.Interval),
|
|
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
|
Rules: prules,
|
|
})
|
|
}
|
|
case *alertingv2beta1.GlobalRuleGroupList:
|
|
for _, group := range list.Items {
|
|
var prules []promresourcesv1.Rule
|
|
for _, rule := range group.Spec.Rules {
|
|
|
|
prule, err := convertRule(&rule.Rule, group.Name,
|
|
createEnforceRuleFuncs(ParseGlobalRuleEnforceMatchers(&rule), nil)...)
|
|
if err != nil {
|
|
log.WithValues("globalrulegroup", group.Name).Error(err, "failed to convert")
|
|
continue
|
|
}
|
|
if prule != nil {
|
|
if rule.ExprBuilder != nil && (rule.ExprBuilder.Node != nil || rule.ExprBuilder.Workload != nil) {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeTemplate)
|
|
} else {
|
|
prule.Labels[RuleLabelKeyRuleType] = string(RuleTypeCustom)
|
|
}
|
|
prules = append(prules, *prule)
|
|
}
|
|
}
|
|
if len(prules) == 0 {
|
|
continue
|
|
}
|
|
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
|
Name: group.Name,
|
|
Interval: promresourcesv1.Duration(group.Spec.Interval),
|
|
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
|
Rules: prules,
|
|
})
|
|
}
|
|
}
|
|
|
|
return rulegroups, nil
|
|
}
|
|
|
|
func ParseGlobalRuleEnforceMatchers(rule *alertingv2beta1.GlobalRule) []*promlabels.Matcher {
|
|
var enforceRuleMatchers []*promlabels.Matcher
|
|
if rule.ClusterSelector != nil {
|
|
matcher := rule.ClusterSelector.ParseToMatcher(RuleLabelKeyCluster)
|
|
if matcher != nil {
|
|
enforceRuleMatchers = append(enforceRuleMatchers, matcher)
|
|
}
|
|
}
|
|
if rule.NamespaceSelector != nil {
|
|
matcher := rule.NamespaceSelector.ParseToMatcher(RuleLabelKeyNamespace)
|
|
if matcher != nil {
|
|
enforceRuleMatchers = append(enforceRuleMatchers, matcher)
|
|
}
|
|
}
|
|
return enforceRuleMatchers
|
|
}
|
|
|
|
func makePrometheusRuleResources(rulegroups []*promresourcesv1.RuleGroup, namespace, namePrefix string,
|
|
labels map[string]string, ownerReferences []metav1.OwnerReference) ([]*promresourcesv1.PrometheusRule, error) {
|
|
|
|
promruleSpecs, err := makePrometheusRuleSpecs(rulegroups)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var ps = make([]*promresourcesv1.PrometheusRule, len(promruleSpecs))
|
|
for i := range promruleSpecs {
|
|
ps[i] = &promresourcesv1.PrometheusRule{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: namespace,
|
|
Name: namePrefix + strconv.Itoa(i),
|
|
Labels: labels,
|
|
OwnerReferences: ownerReferences,
|
|
},
|
|
Spec: *promruleSpecs[i],
|
|
}
|
|
}
|
|
|
|
return ps, nil
|
|
}
|
|
|
|
type rulegroupsWrapper struct {
|
|
rulegroups []*promresourcesv1.RuleGroup
|
|
by func(g1, g2 *promresourcesv1.RuleGroup) bool
|
|
}
|
|
|
|
func (w rulegroupsWrapper) Len() int {
|
|
return len(w.rulegroups)
|
|
}
|
|
|
|
func (w rulegroupsWrapper) Swap(i, j int) {
|
|
w.rulegroups[i], w.rulegroups[j] = w.rulegroups[j], w.rulegroups[i]
|
|
}
|
|
|
|
func (w rulegroupsWrapper) Less(i, j int) bool {
|
|
return w.by(w.rulegroups[i], w.rulegroups[j])
|
|
}
|
|
|
|
func makePrometheusRuleSpecs(rulegroups []*promresourcesv1.RuleGroup) ([]*promresourcesv1.PrometheusRuleSpec, error) {
|
|
sort.Sort(rulegroupsWrapper{
|
|
rulegroups: rulegroups,
|
|
by: func(g1, g2 *promresourcesv1.RuleGroup) bool {
|
|
return g1.Name < g2.Name
|
|
},
|
|
})
|
|
|
|
var (
|
|
pSpecs []*promresourcesv1.PrometheusRuleSpec
|
|
pSpec = &promresourcesv1.PrometheusRuleSpec{}
|
|
size int
|
|
)
|
|
|
|
for i := range rulegroups {
|
|
rulegroup := rulegroups[i]
|
|
|
|
content, err := yaml.Marshal(rulegroup)
|
|
if err != nil {
|
|
return nil, errors.Wrap(err, "failed to marshal content")
|
|
}
|
|
contentLen := len(string(content))
|
|
size += contentLen
|
|
if size > maxConfigMapDataSize*80/100 { // leave space for enforcing possiable label matchers into expr
|
|
pSpecs = append(pSpecs, pSpec)
|
|
// reinit
|
|
size = contentLen
|
|
pSpec = &promresourcesv1.PrometheusRuleSpec{}
|
|
}
|
|
|
|
pSpec.Groups = append(pSpec.Groups, *rulegroup)
|
|
}
|
|
if len(pSpec.Groups) > 0 {
|
|
pSpecs = append(pSpecs, pSpec)
|
|
}
|
|
|
|
return pSpecs, nil
|
|
}
|
|
|
|
func bulkUpdatePrometheusRuleResources(client client.Client, ctx context.Context, current, desired []*promresourcesv1.PrometheusRule) error {
|
|
|
|
var (
|
|
currentMap = make(map[string]*promresourcesv1.PrometheusRule)
|
|
desiredMap = make(map[string]*promresourcesv1.PrometheusRule)
|
|
err error
|
|
)
|
|
for i := range current {
|
|
promrule := current[i]
|
|
currentMap[promrule.Namespace+"/"+promrule.Name] = promrule
|
|
}
|
|
for i := range desired {
|
|
promrule := desired[i]
|
|
desiredMap[promrule.Namespace+"/"+promrule.Name] = promrule
|
|
}
|
|
|
|
// update if exists in current PrometheusRules, or create
|
|
for name, desired := range desiredMap {
|
|
if current, ok := currentMap[name]; ok {
|
|
if !reflect.DeepEqual(current.Spec, desired.Spec) ||
|
|
!reflect.DeepEqual(current.Labels, desired.Labels) ||
|
|
!reflect.DeepEqual(current.OwnerReferences, desired.OwnerReferences) {
|
|
desired.SetResourceVersion(current.ResourceVersion)
|
|
err = client.Update(ctx, desired)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
} else {
|
|
err = client.Create(ctx, desired)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
// delete if not in desired PrometheusRules
|
|
for name, current := range currentMap {
|
|
if _, ok := desiredMap[name]; !ok {
|
|
err = client.Delete(ctx, current)
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
continue
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|