admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhance globalrulegroups (#5134)

Browse Source 
Signed-off-by: junot <junotxiang@kubesphere.io>

Signed-off-by: junot <junotxiang@kubesphere.io>
This commit is contained in:
junot
2022-08-12 20:30:33 +08:00
committed by GitHub
parent f741bc7943
commit b97a49b925
8 changed files with 555 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/api/alerting/v2beta1/types.go
View File

@@ -25,11 +25,13 @@ import (
const (
// for rulegroup/alert
FieldState = "state"
FieldState = "state"
FieldBuiltin = "builtin"
// for rulegroup
FieldRuleGroupEvaluationTime = "evaluationTime"
FieldRuleGroupLastEvaluation = "lastEvalution"
// for alert
FieldAlertLabelFilters = "label_filters"
FieldAlertActiveAt = "activeAt"
@@ -69,6 +71,7 @@ type RuleGroupStatus struct {
}
type RuleStatus struct {
Expr string `json:"expr,omitempty" description:"expression evaluated, for global rules only"`
State string `json:"state,omitempty" description:"state of a rule, one of firing, pending or inactive depending on its alerts"`
Health string `json:"health,omitempty" description:"health state of a rule, one of ok, err, unknown depending on the last execution result"`
LastError string `json:"lastError,omitempty" description:"error of the last evaluation"`

70
pkg/controller/alerting/util.go
View File

@@ -53,10 +53,13 @@ const (
SourceGroupResourceLabelValueEnableTrue = "true"
SourceGroupResourceLabelValueEnableFalse = "false"
// label keys in PrometheusRule.metadata.labels
// for PrometheusRule.metadata.labels
PrometheusRuleResourceLabelKeyOwnerNamespace = "alerting.kubesphere.io/owner_namespace"
PrometheusRuleResourceLabelKeyOwnerCluster = "alerting.kubesphere.io/owner_cluster"
PrometheusRuleResourceLabelKeyRuleLevel = "alerting.kubesphere.io/rule_level"
PrometheusRuleResourceLabelKeyBuiltin = "alerting.kubesphere.io/builtin"
PrometheusRuleResourceLabelValueBuiltinTrue = "true"
PrometheusRuleResourceLabelValueBuiltinFalse = "false"
// name prefix for PrometheusRule
PrometheusRulePrefix = "alertrules-"
@@ -73,23 +76,40 @@ var maxConfigMapDataSize = int(float64(corev1.MaxSecretSize) * 0.5)
type enforceRuleFunc func(rule *promresourcesv1.Rule) error
type EnforceExprFunc func(expr string) (string, error)
var emptyEnforceExprFunc = func(expr string) (string, error) {
return expr, nil
}
func CreateEnforceExprFunc(enforceRuleMatchers []*promlabels.Matcher) EnforceExprFunc {
if len(enforceRuleMatchers) > 0 {
enforcer := injectproxy.NewEnforcer(enforceRuleMatchers...)
return func(expr string) (string, error) {
parsedExpr, err := parser.ParseExpr(expr)
if err != nil {
return expr, err
}
if err := enforcer.EnforceNode(parsedExpr); err != nil {
return expr, err
}
return parsedExpr.String(), nil
}
}
return emptyEnforceExprFunc
}
func createEnforceRuleFuncs(enforceRuleMatchers []*promlabels.Matcher, enforceRuleLabels map[string]string) []enforceRuleFunc {
var enforceFuncs []enforceRuleFunc
// enforce func for rule.expr
if len(enforceRuleMatchers) > 0 {
enforcer := injectproxy.NewEnforcer(enforceRuleMatchers...)
enforceExprFunc := CreateEnforceExprFunc(enforceRuleMatchers)
enforceFuncs = append(enforceFuncs, func(rule *promresourcesv1.Rule) error {
if enforcer != nil {
expr := rule.Expr.String()
parsedExpr, err := parser.ParseExpr(expr)
if err != nil {
return err
}
if err = enforcer.EnforceNode(parsedExpr); err != nil {
return err
}
rule.Expr = intstr.FromString(parsedExpr.String())
expr, err := enforceExprFunc(rule.Expr.String())
if err != nil {
return err
}
rule.Expr = intstr.FromString(expr)
return nil
})
}
@@ -109,10 +129,10 @@ func createEnforceRuleFuncs(enforceRuleMatchers []*promlabels.Matcher, enforceRu
}
func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
enforceFuncs ...enforceRuleFunc) ([]*promresourcesv1.RuleGroup, error) {
commonEnforceFuncs ...enforceRuleFunc) ([]*promresourcesv1.RuleGroup, error) {
var rulegroups []*promresourcesv1.RuleGroup
convertRule := func(rule *alertingv2beta1.Rule) (*promresourcesv1.Rule, error) {
convertRule := func(rule *alertingv2beta1.Rule, enforceFuncs ...enforceRuleFunc) (*promresourcesv1.Rule, error) {
if rule.Disable { // ignoring disabled rule
return nil, nil
}
@@ -135,6 +155,8 @@ func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
Annotations: rule.Annotations,
}
enforceFuncs = append(enforceFuncs, commonEnforceFuncs...)
for _, f := range enforceFuncs {
if f == nil {
continue
@@ -193,7 +215,8 @@ func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
for _, group := range list.Items {
var prules []promresourcesv1.Rule
for _, rule := range group.Spec.Rules {
prule, err := convertRule(&rule.Rule)
prule, err := convertRule(&rule.Rule, createEnforceRuleFuncs(ParseGlobalRuleEnforceMatchers(&rule), nil)...)
if err != nil {
log.WithValues("globalrulegroup", group.Name).Error(err, "failed to convert")
continue
@@ -214,6 +237,23 @@ func makePrometheusRuleGroups(log logr.Logger, groupList client.ObjectList,
return rulegroups, nil
}
func ParseGlobalRuleEnforceMatchers(rule *alertingv2beta1.GlobalRule) []*promlabels.Matcher {
var enforceRuleMatchers []*promlabels.Matcher
if rule.ClusterSelector != nil {
matcher := rule.ClusterSelector.ParseToMatcher(RuleLabelKeyCluster)
if matcher != nil {
enforceRuleMatchers = append(enforceRuleMatchers, matcher)
}
}
if rule.NamespaceSelector != nil {
matcher := rule.NamespaceSelector.ParseToMatcher(RuleLabelKeyNamespace)
if matcher != nil {
enforceRuleMatchers = append(enforceRuleMatchers, matcher)
}
}
return enforceRuleMatchers
}
func makePrometheusRuleResources(rulegroups []*promresourcesv1.RuleGroup, namespace, namePrefix string,
labels map[string]string, ownerReferences []metav1.OwnerReference) ([]*promresourcesv1.PrometheusRule, error) {

2
pkg/kapis/alerting/v2beta1/register.go
View File

@@ -108,6 +108,7 @@ func AddToContainer(container *restful.Container, ksclient kubesphere.Interface,
Param(ws.QueryParameter(query.ParameterAscending, "sort parameters, e.g. reverse=true").Required(false).DefaultValue("ascending=false")).
Param(ws.QueryParameter(query.ParameterOrderBy, "sort parameters, e.g. orderBy=createTime")).
Param(ws.QueryParameter(kapialertingv2beta1.FieldState, "state of a rulegroup, one of `firing`, `pending`, `inactive` depending on its rules")).
Param(ws.QueryParameter(kapialertingv2beta1.FieldBuiltin, "filter rule groups, `true` for built-in rule groups and `false` for custom rule groups")).
Returns(http.StatusOK, kapi.StatusOK, kapi.ListResult{}).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.AlertingTag}))
@@ -126,6 +127,7 @@ func AddToContainer(container *restful.Container, ksclient kubesphere.Interface,
Param(ws.QueryParameter(query.ParameterOrderBy, "sort parameters, one of `activeAt`. e.g. orderBy=activeAt")).
Param(ws.QueryParameter(kapialertingv2beta1.FieldState, "state, one of `firing`, `pending`")).
Param(ws.QueryParameter(kapialertingv2beta1.FieldAlertLabelFilters, "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a").DataFormat("key=%s,key~%s")).
Param(ws.QueryParameter(kapialertingv2beta1.FieldBuiltin, "filter alerts, `true` for alerts from built-in rule groups and `false` for alerts from custom rule groups")).
Returns(http.StatusOK, kapi.StatusOK, kapi.ListResult{}).
Metadata(restfulspec.KeyOpenAPITags, []string{constants.AlertingTag}))

120
pkg/models/alerting/rulegroup.go
View File

@@ -22,6 +22,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
"kubesphere.io/kubesphere/pkg/api"
kapialertingv2beta1 "kubesphere.io/kubesphere/pkg/api/alerting/v2beta1"
@@ -106,6 +107,14 @@ func (o *ruleGroupOperator) listRuleGroups(ctx context.Context, namespace string
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) { // assure that they are the same rulegroups
copyRuleGroupStatus(statusg, &g.Status)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for range g.Spec.Rules {
g.Status.RulesStatus = append(g.Status.RulesStatus, kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
})
}
}
groups[i] = g
}
@@ -272,12 +281,23 @@ func (o *ruleGroupOperator) GetRuleGroup(ctx context.Context, namespace, name st
return nil, err
}
var setStatus bool
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
setStatus = true
break
}
}
if !setStatus {
// for rules not loaded by rule reloader (eg.thanos) yet
for range ret.Spec.Rules {
ret.Status.RulesStatus = append(ret.Status.RulesStatus, kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
})
}
}
return ret, nil
}
@@ -324,6 +344,14 @@ func (o *ruleGroupOperator) listClusterRuleGroups(ctx context.Context, selector
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) {
copyRuleGroupStatus(statusg, &g.Status)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for range g.Spec.Rules {
g.Status.RulesStatus = append(g.Status.RulesStatus, kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
})
}
}
groups[i] = g
}
@@ -410,12 +438,23 @@ func (o *ruleGroupOperator) GetClusterRuleGroup(ctx context.Context, name string
return nil, err
}
var setStatus bool
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
setStatus = true
break
}
}
if !setStatus {
// for rules not loaded by rule reloader (eg.thanos) yet
for range ret.Spec.Rules {
ret.Status.RulesStatus = append(ret.Status.RulesStatus, kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
})
}
}
return ret, nil
}
@@ -463,6 +502,21 @@ func (o *ruleGroupOperator) listGlobalRuleGroups(ctx context.Context, selector l
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) {
copyRuleGroupStatus(statusg, &g.Status)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for _, rule := range g.Spec.Rules {
ruleStatus := kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
}
enforceExprFunc := controller.CreateEnforceExprFunc(controller.ParseGlobalRuleEnforceMatchers(&rule))
expr, err := enforceExprFunc(rule.Expr.String())
if err != nil {
return nil, err
}
ruleStatus.Expr = expr
g.Status.RulesStatus = append(g.Status.RulesStatus, ruleStatus)
}
}
groups[i] = g
}
@@ -472,7 +526,22 @@ func (o *ruleGroupOperator) listGlobalRuleGroups(ctx context.Context, selector l
func (o *ruleGroupOperator) ListGlobalRuleGroups(ctx context.Context,
queryParam *query.Query) (*api.ListResult, error) {
groups, err := o.listGlobalRuleGroups(ctx, queryParam.Selector())
selector := queryParam.Selector()
if val, ok := queryParam.Filters[kapialertingv2beta1.FieldBuiltin]; ok {
// add match requirement to the selector to select only builtin or custom rulegroups
var operator selection.Operator
if val == controller.PrometheusRuleResourceLabelValueBuiltinTrue {
operator = selection.Equals
} else {
operator = selection.NotEquals
}
requirement, _ := labels.NewRequirement(
controller.PrometheusRuleResourceLabelKeyBuiltin,
operator,
[]string{controller.PrometheusRuleResourceLabelValueBuiltinTrue})
selector = selector.Add(*requirement)
}
groups, err := o.listGlobalRuleGroups(ctx, selector)
if err != nil {
return nil, err
}
@@ -486,6 +555,9 @@ func (o *ruleGroupOperator) ListGlobalRuleGroups(ctx context.Context,
return resources.DefaultObjectMetaCompare(
left.(*kapialertingv2beta1.GlobalRuleGroup).ObjectMeta, right.(*kapialertingv2beta1.GlobalRuleGroup).ObjectMeta, field)
}, func(obj runtime.Object, filter query.Filter) bool {
if filter.Field == kapialertingv2beta1.FieldBuiltin { // ignoring this filter because it is filtered at the front
return true
}
hit, selected := o.filterRuleGroupStatus(&obj.(*kapialertingv2beta1.GlobalRuleGroup).Status, filter)
if hit {
return selected
@@ -497,7 +569,22 @@ func (o *ruleGroupOperator) ListGlobalRuleGroups(ctx context.Context,
func (o *ruleGroupOperator) ListGlobalAlerts(ctx context.Context,
queryParam *query.Query) (*api.ListResult, error) {
groups, err := o.listGlobalRuleGroups(ctx, labels.Everything())
selector := labels.Everything()
if val, ok := queryParam.Filters[kapialertingv2beta1.FieldBuiltin]; ok {
// add match requirement to the selector to select only builtin or custom rulegroups
var operator selection.Operator
if val == controller.PrometheusRuleResourceLabelValueBuiltinTrue {
operator = selection.Equals
} else {
operator = selection.NotEquals
}
requirement, _ := labels.NewRequirement(
controller.PrometheusRuleResourceLabelKeyBuiltin,
operator,
[]string{controller.PrometheusRuleResourceLabelValueBuiltinTrue})
selector = selector.Add(*requirement)
}
groups, err := o.listGlobalRuleGroups(ctx, selector)
if err != nil {
return nil, err
}
@@ -549,12 +636,30 @@ func (o *ruleGroupOperator) GetGlobalRuleGroup(ctx context.Context, name string)
return nil, err
}
var setStatus bool
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
setStatus = true
break
}
}
if !setStatus {
// for rules not loaded by rule reloader (eg.thanos) yet
for _, rule := range ret.Spec.Rules {
ruleStatus := kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
}
enforceExprFunc := controller.CreateEnforceExprFunc(controller.ParseGlobalRuleEnforceMatchers(&rule))
expr, err := enforceExprFunc(rule.Expr.String())
if err != nil {
return nil, err
}
ruleStatus.Expr = expr
ret.Status.RulesStatus = append(ret.Status.RulesStatus, ruleStatus)
}
}
return ret, nil
}
@@ -582,14 +687,21 @@ func copyRuleGroupStatus(source *alerting.RuleGroup, target *kapialertingv2beta1
Value: alert.Value,
})
}
target.RulesStatus = append(target.RulesStatus, kapialertingv2beta1.RuleStatus{
ruleStatus := kapialertingv2beta1.RuleStatus{
State: rule.State,
Health: rule.Health,
LastError: rule.LastError,
EvaluationTime: rule.EvaluationTime,
LastEvaluation: rule.LastEvaluation,
Alerts: alerts,
})
}
if len(rule.Labels) > 0 {
if level, ok := rule.Labels[controller.RuleLabelKeyRuleLevel]; ok &&
level == string(controller.RuleLevelGlobal) { // provided only for global rules
ruleStatus.Expr = rule.Query
}
}
target.RulesStatus = append(target.RulesStatus, ruleStatus)
}
target.State = groupState.String()
}