reconcile rulegroups to prometheusrules (#5081)
reconcile rulegroups Signed-off-by: junot <junotxiang@kubesphere.io>
This commit is contained in:
junot
345
pkg/controller/alerting/util.go
Normal file
345
pkg/controller/alerting/util.go
Normal file
@@ -0,0 +1,345 @@
|
||||
/*
|
||||
Copyright 2019 The KubeSphere Authors.
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
|
||||
"github.com/go-logr/logr"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus-community/prom-label-proxy/injectproxy"
|
||||
promresourcesv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
|
||||
promlabels "github.com/prometheus/prometheus/pkg/labels"
|
||||
"github.com/prometheus/prometheus/promql/parser"
|
||||
"gopkg.in/yaml.v2"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
|
||||
alertingv2beta1 "kubesphere.io/api/alerting/v2beta1"
|
||||
|
||||
"kubesphere.io/kubesphere/pkg/constants"
|
||||
)
|
||||
|
||||
const (
|
||||
RuleLevelNamesapce RuleLevel = "namespace"
|
||||
RuleLevelCluster RuleLevel = "cluster"
|
||||
RuleLevelGlobal RuleLevel = "global"
|
||||
|
||||
// label keys in rule.labels
|
||||
RuleLabelKeyRuleLevel = "rule_level"
|
||||
RuleLabelKeyCluster = "cluster"
|
||||
RuleLabelKeyNamespace = "namespace"
|
||||
RuleLabelKeySeverity = "severity"
|
||||
|
||||
// label keys in RuleGroup/ClusterRuleGroup/GlobalRuleGroup.metadata.labels
|
||||
SourceGroupResourceLabelKeyEnable = "alerting.kubesphere.io/enable"
|
||||
SourceGroupResourceLabelValueEnableTrue = "true"
|
||||
SourceGroupResourceLabelValueEnableFalse = "false"
|
||||
|
||||
// label keys in PrometheusRule.metadata.labels
|
||||
PrometheusRuleResourceLabelKeyOwnerNamespace = "alerting.kubesphere.io/owner_namespace"
|
||||
PrometheusRuleResourceLabelKeyOwnerCluster = "alerting.kubesphere.io/owner_cluster"
|
||||
PrometheusRuleResourceLabelKeyRuleLevel = "alerting.kubesphere.io/rule_level"
|
||||
|
||||
// name prefix for PrometheusRule
|
||||
PrometheusRulePrefix = "alertrules-"
|
||||
PrometheusRulePrefixNamespaceLevel = PrometheusRulePrefix + "ns-"
|
||||
PrometheusRulePrefixClusterLevel = PrometheusRulePrefix + "cl-"
|
||||
PrometheusRulePrefixGlobalLevel = PrometheusRulePrefix + "gl-"
|
||||
|
||||
PrometheusRuleNamespace = constants.KubeSphereMonitoringNamespace
|
||||
)
|
||||
|
||||
type RuleLevel string
|
||||
|
||||
var maxConfigMapDataSize = int(float64(corev1.MaxSecretSize) * 0.5)
|
||||
|
||||
type enforceRuleFunc func(rule *promresourcesv1.Rule) error
|
||||
|
||||
func createEnforceRuleFuncs(enforceRuleMatchers []*promlabels.Matcher, enforceRuleLabels map[string]string) []enforceRuleFunc {
|
||||
var enforceFuncs []enforceRuleFunc
|
||||
// enforce func for rule.expr
|
||||
if len(enforceRuleMatchers) > 0 {
|
||||
enforcer := injectproxy.NewEnforcer(enforceRuleMatchers...)
|
||||
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
|
||||
if enforcer != nil {
|
||||
expr := rule.Expr.String()
|
||||
parsedExpr, err := parser.ParseExpr(expr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = enforcer.EnforceNode(parsedExpr); err != nil {
|
||||
return err
|
||||
}
|
||||
rule.Expr = intstr.FromString(parsedExpr.String())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
// enforce func for rule.labels
|
||||
if len(enforceRuleLabels) > 0 {
|
||||
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
|
||||
if rule.Labels == nil {
|
||||
rule.Labels = make(map[string]string)
|
||||
}
|
||||
for n, v := range enforceRuleLabels {
|
||||
rule.Labels[n] = v
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
return enforceFuncs
|
||||
}
|
||||
|
||||
func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
|
||||
enforceFuncs ...enforceRuleFunc) ([]*promresourcesv1.RuleGroup, error) {
|
||||
var rulegroups []*promresourcesv1.RuleGroup
|
||||
|
||||
convertRule := func(rule *alertingv2beta1.Rule) (*promresourcesv1.Rule, error) {
|
||||
if rule.Disable { // ignoring disabled rule
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rule = rule.DeepCopy()
|
||||
|
||||
if rule.Labels == nil {
|
||||
rule.Labels = make(map[string]string)
|
||||
}
|
||||
|
||||
if rule.Severity != "" {
|
||||
rule.Labels[RuleLabelKeySeverity] = string(rule.Severity)
|
||||
}
|
||||
|
||||
prule := promresourcesv1.Rule{
|
||||
Alert: rule.Alert,
|
||||
For: string(rule.For),
|
||||
Expr: rule.Expr,
|
||||
Labels: rule.Labels,
|
||||
Annotations: rule.Annotations,
|
||||
}
|
||||
|
||||
for _, f := range enforceFuncs {
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
err := f(&prule)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(err, "alert: %s", rule.Alert)
|
||||
}
|
||||
}
|
||||
|
||||
return &prule, nil
|
||||
}
|
||||
|
||||
switch list := groupList.(type) {
|
||||
case *alertingv2beta1.RuleGroupList:
|
||||
for _, group := range list.Items {
|
||||
var prules []promresourcesv1.Rule
|
||||
for _, rule := range group.Spec.Rules {
|
||||
prule, err := convertRule(&rule.Rule)
|
||||
if err != nil {
|
||||
log.WithValues("rulegroup", group.Namespace+"/"+group.Name).Error(err, "failed to convert")
|
||||
continue
|
||||
}
|
||||
if prule != nil {
|
||||
prules = append(prules, *prule)
|
||||
}
|
||||
}
|
||||
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
||||
Name: group.Name,
|
||||
Interval: group.Spec.Interval,
|
||||
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
||||
Rules: prules,
|
||||
})
|
||||
}
|
||||
case *alertingv2beta1.ClusterRuleGroupList:
|
||||
for _, group := range list.Items {
|
||||
var prules []promresourcesv1.Rule
|
||||
for _, rule := range group.Spec.Rules {
|
||||
prule, err := convertRule(&rule.Rule)
|
||||
if err != nil {
|
||||
log.WithValues("clusterrulegroup", group.Name).Error(err, "failed to convert")
|
||||
continue
|
||||
}
|
||||
if prule != nil {
|
||||
prules = append(prules, *prule)
|
||||
}
|
||||
}
|
||||
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
||||
Name: group.Name,
|
||||
Interval: group.Spec.Interval,
|
||||
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
||||
Rules: prules,
|
||||
})
|
||||
}
|
||||
case *alertingv2beta1.GlobalRuleGroupList:
|
||||
for _, group := range list.Items {
|
||||
var prules []promresourcesv1.Rule
|
||||
for _, rule := range group.Spec.Rules {
|
||||
prule, err := convertRule(&rule.Rule)
|
||||
if err != nil {
|
||||
log.WithValues("globalrulegroup", group.Name).Error(err, "failed to convert")
|
||||
continue
|
||||
}
|
||||
if prule != nil {
|
||||
prules = append(prules, *prule)
|
||||
}
|
||||
}
|
||||
rulegroups = append(rulegroups, &promresourcesv1.RuleGroup{
|
||||
Name: group.Name,
|
||||
Interval: group.Spec.Interval,
|
||||
PartialResponseStrategy: group.Spec.PartialResponseStrategy,
|
||||
Rules: prules,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return rulegroups, nil
|
||||
}
|
||||
|
||||
func makePrometheusRuleResources(rulegroups []*promresourcesv1.RuleGroup, namespace, namePrefix string,
|
||||
labels map[string]string, ownerReferences []metav1.OwnerReference) ([]*promresourcesv1.PrometheusRule, error) {
|
||||
|
||||
promruleSpecs, err := makePrometheusRuleSpecs(rulegroups)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var ps = make([]*promresourcesv1.PrometheusRule, len(promruleSpecs))
|
||||
for i := range promruleSpecs {
|
||||
ps[i] = &promresourcesv1.PrometheusRule{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Namespace: namespace,
|
||||
Name: namePrefix + strconv.Itoa(i),
|
||||
Labels: labels,
|
||||
OwnerReferences: ownerReferences,
|
||||
},
|
||||
Spec: *promruleSpecs[i],
|
||||
}
|
||||
}
|
||||
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
type rulegroupsWrapper struct {
|
||||
rulegroups []*promresourcesv1.RuleGroup
|
||||
by func(g1, g2 *promresourcesv1.RuleGroup) bool
|
||||
}
|
||||
|
||||
func (w rulegroupsWrapper) Len() int {
|
||||
return len(w.rulegroups)
|
||||
}
|
||||
|
||||
func (w rulegroupsWrapper) Swap(i, j int) {
|
||||
w.rulegroups[i], w.rulegroups[j] = w.rulegroups[j], w.rulegroups[i]
|
||||
}
|
||||
|
||||
func (w rulegroupsWrapper) Less(i, j int) bool {
|
||||
return w.by(w.rulegroups[i], w.rulegroups[j])
|
||||
}
|
||||
|
||||
func makePrometheusRuleSpecs(rulegroups []*promresourcesv1.RuleGroup) ([]*promresourcesv1.PrometheusRuleSpec, error) {
|
||||
sort.Sort(rulegroupsWrapper{
|
||||
rulegroups: rulegroups,
|
||||
by: func(g1, g2 *promresourcesv1.RuleGroup) bool {
|
||||
return g1.Name < g2.Name
|
||||
},
|
||||
})
|
||||
|
||||
var (
|
||||
pSpecs []*promresourcesv1.PrometheusRuleSpec
|
||||
pSpec promresourcesv1.PrometheusRuleSpec
|
||||
size int
|
||||
)
|
||||
|
||||
for i := range rulegroups {
|
||||
rulegroup := rulegroups[i]
|
||||
|
||||
content, err := yaml.Marshal(rulegroup)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to marshal content")
|
||||
}
|
||||
contentLen := len(string(content))
|
||||
size += contentLen
|
||||
if size > maxConfigMapDataSize*80/100 { // leave space for enforcing possiable label matchers into expr
|
||||
pSpecs = append(pSpecs, &pSpec)
|
||||
// reinit
|
||||
size = contentLen
|
||||
pSpec = promresourcesv1.PrometheusRuleSpec{}
|
||||
}
|
||||
|
||||
pSpec.Groups = append(pSpec.Groups, *rulegroup)
|
||||
}
|
||||
if len(pSpec.Groups) > 0 {
|
||||
pSpecs = append(pSpecs, &pSpec)
|
||||
}
|
||||
|
||||
return pSpecs, nil
|
||||
}
|
||||
|
||||
func bulkUpdatePrometheusRuleResources(client client.Client, ctx context.Context, current, desired []*promresourcesv1.PrometheusRule) error {
|
||||
|
||||
var (
|
||||
currentMap = make(map[string]*promresourcesv1.PrometheusRule)
|
||||
desiredMap = make(map[string]*promresourcesv1.PrometheusRule)
|
||||
err error
|
||||
)
|
||||
for i := range current {
|
||||
promrule := current[i]
|
||||
currentMap[promrule.Namespace+"/"+promrule.Name] = promrule
|
||||
}
|
||||
for i := range desired {
|
||||
promrule := desired[i]
|
||||
desiredMap[promrule.Namespace+"/"+promrule.Name] = promrule
|
||||
}
|
||||
|
||||
// update if exists in current PrometheusRules, or create
|
||||
for name, desired := range desiredMap {
|
||||
if current, ok := currentMap[name]; ok {
|
||||
if !reflect.DeepEqual(current.Spec, desired.Spec) ||
|
||||
!reflect.DeepEqual(current.Labels, desired.Labels) ||
|
||||
!reflect.DeepEqual(current.OwnerReferences, desired.OwnerReferences) {
|
||||
desired.SetResourceVersion(current.ResourceVersion)
|
||||
err = client.Update(ctx, desired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err = client.Create(ctx, desired)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// delete if not in desired PrometheusRules
|
||||
for name, current := range currentMap {
|
||||
if _, ok := desiredMap[name]; !ok {
|
||||
err = client.Delete(ctx, current)
|
||||
if err != nil {
|
||||
if apierrors.IsNotFound(err) {
|
||||
continue
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user