admin/kubesphere
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix rule status set (#5219)

Browse Source 
Signed-off-by: junot <junotxiang@kubesphere.io>

Signed-off-by: junot <junotxiang@kubesphere.io>
This commit is contained in:
junot
2022-09-16 18:43:48 +08:00
committed by GitHub
parent 5ea9d3deac
commit 789a0ab1e4
8 changed files with 205 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/apiserver/apiserver.go
View File

@@ -265,7 +265,7 @@ func (s *APIServer) installKubeSphereAPIs(stopCh <-chan struct{}) {
urlruntime.Must(alertingv1.AddToContainer(s.container, s.Config.AlertingOptions.Endpoint))
urlruntime.Must(alertingv2alpha1.AddToContainer(s.container, s.InformerFactory,
s.KubernetesClient.Prometheus(), s.AlertingClient, s.Config.AlertingOptions))
urlruntime.Must(alertingv2beta1.AddToContainer(s.container, s.KubernetesClient.KubeSphere(), s.AlertingClient))
urlruntime.Must(alertingv2beta1.AddToContainer(s.container, s.InformerFactory, s.AlertingClient))
urlruntime.Must(version.AddToContainer(s.container, s.KubernetesClient.Kubernetes().Discovery()))
urlruntime.Must(kubeedgev1alpha1.AddToContainer(s.container, s.Config.KubeEdgeOptions.Endpoint))
urlruntime.Must(edgeruntimev1alpha1.AddToContainer(s.container, s.Config.EdgeRuntimeOptions.Endpoint))

23
pkg/controller/alerting/clusterrulegroup_controller.go
View File

@@ -22,6 +22,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -60,6 +61,28 @@ func (r *ClusterRuleGroupReconciler) Reconcile(ctx context.Context, req reconcil
return reconcile.Result{}, err
}
// add rule_id label that may have been missed
var updated bool
for i := range clusterrulegroupList.Items {
g := clusterrulegroupList.Items[i]
for j := range g.Spec.Rules {
if g.Spec.Rules[j].Labels == nil {
g.Spec.Rules[j].Labels = make(map[string]string)
}
if _, ok := g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId]; !ok {
g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId] = string(uuid.NewUUID())
err = r.Client.Update(ctx, &g)
if err != nil {
return reconcile.Result{}, err
}
updated = true
}
}
}
if updated {
return reconcile.Result{}, nil
}
// labels added to rule.labels
enforceRuleLabels := map[string]string{
RuleLabelKeyRuleLevel: string(ruleLevel),

23
pkg/controller/alerting/globalrulegroup_controller.go
View File

@@ -21,6 +21,7 @@ import (
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -59,6 +60,28 @@ func (r *GlobalRuleGroupReconciler) Reconcile(ctx context.Context, req reconcile
return reconcile.Result{}, err
}
// add rule_id label that may have been missed
var updated bool
for i := range globalrulegroupList.Items {
g := globalrulegroupList.Items[i]
for j := range g.Spec.Rules {
if g.Spec.Rules[j].Labels == nil {
g.Spec.Rules[j].Labels = make(map[string]string)
}
if _, ok := g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId]; !ok {
g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId] = string(uuid.NewUUID())
err = r.Client.Update(ctx, &g)
if err != nil {
return reconcile.Result{}, err
}
updated = true
}
}
}
if updated {
return reconcile.Result{}, nil
}
// labels added to rule.labels
enforceRuleLabels := map[string]string{
RuleLabelKeyRuleLevel: string(ruleLevel),

23
pkg/controller/alerting/rulegroup_controller.go
View File

@@ -25,6 +25,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/uuid"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -65,6 +66,28 @@ func (r *RuleGroupReconciler) Reconcile(ctx context.Context, req reconcile.Reque
return reconcile.Result{}, err
}
// add rule_id label that may have been missed
var updated bool
for i := range rulegroupList.Items {
g := rulegroupList.Items[i]
for j := range g.Spec.Rules {
if g.Spec.Rules[j].Labels == nil {
g.Spec.Rules[j].Labels = make(map[string]string)
}
if _, ok := g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId]; !ok {
g.Spec.Rules[j].Labels[alertingv2beta1.RuleLabelKeyRuleId] = string(uuid.NewUUID())
err = r.Client.Update(ctx, &g)
if err != nil {
return reconcile.Result{}, err
}
updated = true
}
}
}
if updated {
return reconcile.Result{}, nil
}
// labels added to rule.labels
enforceRuleLabels := map[string]string{
RuleLabelKeyRuleLevel: string(ruleLevel),

6
pkg/kapis/alerting/v2beta1/handler.go
View File

@@ -22,7 +22,7 @@ import (
kapi "kubesphere.io/kubesphere/pkg/api"
"kubesphere.io/kubesphere/pkg/apiserver/query"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
"kubesphere.io/kubesphere/pkg/informers"
alertingmodels "kubesphere.io/kubesphere/pkg/models/alerting"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
)
@@ -31,9 +31,9 @@ type handler struct {
operator alertingmodels.RuleGroupOperator
}
func newHandler(ksclient kubesphere.Interface, ruleClient alerting.RuleClient) *handler {
func newHandler(informers informers.InformerFactory, ruleClient alerting.RuleClient) *handler {
return &handler{
operator: alertingmodels.NewRuleGroupOperator(ksclient, ruleClient),
operator: alertingmodels.NewRuleGroupOperator(informers, ruleClient),
}
}

6
pkg/kapis/alerting/v2beta1/register.go
View File

@@ -28,16 +28,16 @@ import (
kapialertingv2beta1 "kubesphere.io/kubesphere/pkg/api/alerting/v2beta1"
"kubesphere.io/kubesphere/pkg/apiserver/query"
"kubesphere.io/kubesphere/pkg/apiserver/runtime"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
"kubesphere.io/kubesphere/pkg/constants"
"kubesphere.io/kubesphere/pkg/informers"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
)
func AddToContainer(container *restful.Container, ksclient kubesphere.Interface, ruleClient alerting.RuleClient) error {
func AddToContainer(container *restful.Container, informers informers.InformerFactory, ruleClient alerting.RuleClient) error {
ws := runtime.NewWebService(alertingv2beta1.SchemeGroupVersion)
handler := newHandler(ksclient, ruleClient)
handler := newHandler(informers, ruleClient)
ws.Route(ws.GET("/namespaces/{namespace}/rulegroups").
To(handler.handleListRuleGroups).

152
pkg/models/alerting/rulegroup.go
View File

@@ -20,16 +20,18 @@ import (
promlabels "github.com/prometheus/prometheus/pkg/labels"
promrules "github.com/prometheus/prometheus/rules"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/selection"
alertingv2beta1 "kubesphere.io/api/alerting/v2beta1"
"kubesphere.io/kubesphere/pkg/api"
kapialertingv2beta1 "kubesphere.io/kubesphere/pkg/api/alerting/v2beta1"
"kubesphere.io/kubesphere/pkg/apiserver/query"
kubesphere "kubesphere.io/kubesphere/pkg/client/clientset/versioned"
alertinglisters "kubesphere.io/kubesphere/pkg/client/listers/alerting/v2beta1"
controller "kubesphere.io/kubesphere/pkg/controller/alerting"
"kubesphere.io/kubesphere/pkg/informers"
resources "kubesphere.io/kubesphere/pkg/models/resources/v1alpha3"
"kubesphere.io/kubesphere/pkg/simple/client/alerting"
)
@@ -48,23 +50,24 @@ type RuleGroupOperator interface {
ListClusterAlerts(ctx context.Context, queryParam *query.Query) (*api.ListResult, error)
}
func NewRuleGroupOperator(ksclient kubesphere.Interface, ruleClient alerting.RuleClient) RuleGroupOperator {
func NewRuleGroupOperator(informers informers.InformerFactory, ruleClient alerting.RuleClient) RuleGroupOperator {
return &ruleGroupOperator{
ksclient: ksclient,
ruleClient: ruleClient,
ruleClient: ruleClient,
ruleGroupLister: informers.KubeSphereSharedInformerFactory().Alerting().V2beta1().RuleGroups().Lister(),
clusterRuleGroupLister: informers.KubeSphereSharedInformerFactory().Alerting().V2beta1().ClusterRuleGroups().Lister(),
globalRuleGroupLister: informers.KubeSphereSharedInformerFactory().Alerting().V2beta1().GlobalRuleGroups().Lister(),
}
}
type ruleGroupOperator struct {
ruleClient alerting.RuleClient
ksclient kubesphere.Interface
ruleClient alerting.RuleClient
ruleGroupLister alertinglisters.RuleGroupLister
clusterRuleGroupLister alertinglisters.ClusterRuleGroupLister
globalRuleGroupLister alertinglisters.GlobalRuleGroupLister
}
func (o *ruleGroupOperator) listRuleGroups(ctx context.Context, namespace string, selector labels.Selector) ([]runtime.Object, error) {
resourceRuleGroups, err := o.ksclient.AlertingV2beta1().RuleGroups(namespace).List(ctx,
metav1.ListOptions{
LabelSelector: selector.String(),
})
resourceRuleGroups, err := o.ruleGroupLister.RuleGroups(namespace).List(selector)
if err != nil {
return nil, err
}
@@ -97,17 +100,25 @@ func (o *ruleGroupOperator) listRuleGroups(ctx context.Context, namespace string
}
// copy status info of statusRuleGroups to matched rulegroups
var groups = make([]runtime.Object, len(resourceRuleGroups.Items))
for i := range resourceRuleGroups.Items {
var groups = make([]runtime.Object, len(resourceRuleGroups))
for i := range resourceRuleGroups {
g := &kapialertingv2beta1.RuleGroup{
RuleGroup: resourceRuleGroups.Items[i],
RuleGroup: *resourceRuleGroups[i],
Status: kapialertingv2beta1.RuleGroupStatus{
State: promrules.StateInactive.String(),
},
}
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) { // assure that they are the same rulegroups
copyRuleGroupStatus(statusg, &g.Status)
specRules := g.Spec.Rules
if ok && len(statusg.Rules) == len(specRules) { // assure that they are the same rulegroups
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(statusg, &g.Status, ruleIds)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for range g.Spec.Rules {
@@ -281,7 +292,7 @@ func (d *ruleGroupOperator) createFilterAlertFunc(queryParam *query.Query) func(
}
func (o *ruleGroupOperator) GetRuleGroup(ctx context.Context, namespace, name string) (*kapialertingv2beta1.RuleGroup, error) {
resourceRuleGroup, err := o.ksclient.AlertingV2beta1().RuleGroups(namespace).Get(ctx, name, metav1.GetOptions{})
resourceRuleGroup, err := o.ruleGroupLister.RuleGroups(namespace).Get(name)
if err != nil {
return nil, err
}
@@ -308,9 +319,17 @@ func (o *ruleGroupOperator) GetRuleGroup(ctx context.Context, namespace, name st
}
var setStatus bool
specRules := resourceRuleGroup.Spec.Rules
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(specRules) {
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(g, &ret.Status, ruleIds)
setStatus = true
break
}
@@ -333,9 +352,7 @@ func (o *ruleGroupOperator) GetRuleGroup(ctx context.Context, namespace, name st
}
func (o *ruleGroupOperator) listClusterRuleGroups(ctx context.Context, selector labels.Selector) ([]runtime.Object, error) {
resourceRuleGroups, err := o.ksclient.AlertingV2beta1().ClusterRuleGroups().List(ctx, metav1.ListOptions{
LabelSelector: selector.String(),
})
resourceRuleGroups, err := o.clusterRuleGroupLister.List(selector)
if err != nil {
return nil, err
}
@@ -363,17 +380,25 @@ func (o *ruleGroupOperator) listClusterRuleGroups(ctx context.Context, selector
}
}
// copy status info of statusRuleGroups to matched rulegroups
var groups = make([]runtime.Object, len(resourceRuleGroups.Items))
for i := range resourceRuleGroups.Items {
var groups = make([]runtime.Object, len(resourceRuleGroups))
for i := range resourceRuleGroups {
g := &kapialertingv2beta1.ClusterRuleGroup{
ClusterRuleGroup: resourceRuleGroups.Items[i],
ClusterRuleGroup: *resourceRuleGroups[i],
Status: kapialertingv2beta1.RuleGroupStatus{
State: promrules.StateInactive.String(),
},
}
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) {
copyRuleGroupStatus(statusg, &g.Status)
specRules := g.Spec.Rules
if ok && len(statusg.Rules) == len(specRules) {
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(statusg, &g.Status, ruleIds)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for range g.Spec.Rules {
@@ -456,7 +481,7 @@ func (o *ruleGroupOperator) ListClusterAlerts(ctx context.Context,
}
func (o *ruleGroupOperator) GetClusterRuleGroup(ctx context.Context, name string) (*kapialertingv2beta1.ClusterRuleGroup, error) {
resourceRuleGroup, err := o.ksclient.AlertingV2beta1().ClusterRuleGroups().Get(ctx, name, metav1.GetOptions{})
resourceRuleGroup, err := o.clusterRuleGroupLister.Get(name)
if err != nil {
return nil, err
}
@@ -479,9 +504,17 @@ func (o *ruleGroupOperator) GetClusterRuleGroup(ctx context.Context, name string
}
var setStatus bool
specRules := resourceRuleGroup.Spec.Rules
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(specRules) {
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(g, &ret.Status, ruleIds)
setStatus = true
break
}
@@ -504,10 +537,7 @@ func (o *ruleGroupOperator) GetClusterRuleGroup(ctx context.Context, name string
}
func (o *ruleGroupOperator) listGlobalRuleGroups(ctx context.Context, selector labels.Selector) ([]runtime.Object, error) {
resourceRuleGroups, err := o.ksclient.AlertingV2beta1().GlobalRuleGroups().List(ctx,
metav1.ListOptions{
LabelSelector: selector.String(),
})
resourceRuleGroups, err := o.globalRuleGroupLister.List(selector)
if err != nil {
return nil, err
}
@@ -535,17 +565,25 @@ func (o *ruleGroupOperator) listGlobalRuleGroups(ctx context.Context, selector l
}
}
// copy status info of statusRuleGroups to matched rulegroups
var groups = make([]runtime.Object, len(resourceRuleGroups.Items))
for i := range resourceRuleGroups.Items {
var groups = make([]runtime.Object, len(resourceRuleGroups))
for i := range resourceRuleGroups {
g := &kapialertingv2beta1.GlobalRuleGroup{
GlobalRuleGroup: resourceRuleGroups.Items[i],
GlobalRuleGroup: *resourceRuleGroups[i],
Status: kapialertingv2beta1.RuleGroupStatus{
State: promrules.StateInactive.String(),
},
}
statusg, ok := statusRuleGroupMap[g.Name]
if ok && len(statusg.Rules) == len(g.Spec.Rules) {
copyRuleGroupStatus(statusg, &g.Status)
specRules := g.Spec.Rules
if ok && len(statusg.Rules) == len(specRules) {
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(statusg, &g.Status, ruleIds)
} else {
// for rules not loaded by rule reloader (eg.thanos) yet
for _, rule := range g.Spec.Rules {
@@ -668,7 +706,7 @@ func (o *ruleGroupOperator) ListGlobalAlerts(ctx context.Context,
}
func (o *ruleGroupOperator) GetGlobalRuleGroup(ctx context.Context, name string) (*kapialertingv2beta1.GlobalRuleGroup, error) {
resourceRuleGroup, err := o.ksclient.AlertingV2beta1().GlobalRuleGroups().Get(ctx, name, metav1.GetOptions{})
resourceRuleGroup, err := o.globalRuleGroupLister.Get(name)
if err != nil {
return nil, err
}
@@ -692,8 +730,16 @@ func (o *ruleGroupOperator) GetGlobalRuleGroup(ctx context.Context, name string)
var setStatus bool
for _, g := range statusRuleGroups {
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(resourceRuleGroup.Spec.Rules) {
copyRuleGroupStatus(g, &ret.Status)
specRules := resourceRuleGroup.Spec.Rules
if g.Name == resourceRuleGroup.Name && len(g.Rules) == len(specRules) {
var ruleIds = make([]string, len(specRules))
for i := range specRules {
if specRules[i].Labels != nil {
ruleIds[i] = specRules[i].Labels[alertingv2beta1.RuleLabelKeyRuleId]
}
}
copyRuleGroupStatus(g, &ret.Status, ruleIds)
setStatus = true
break
}
@@ -723,14 +769,32 @@ func (o *ruleGroupOperator) GetGlobalRuleGroup(ctx context.Context, name string)
}
// copyRuleGroupStatus copies group/rule status and alerts from source to target
func copyRuleGroupStatus(source *alerting.RuleGroup, target *kapialertingv2beta1.RuleGroupStatus) {
func copyRuleGroupStatus(source *alerting.RuleGroup, target *kapialertingv2beta1.RuleGroupStatus, ruleIds []string) {
target.LastEvaluation = source.LastEvaluation
if source.EvaluationTime > 0 {
target.EvaluationTime = &source.EvaluationTime
}
target.RulesStatus = make([]kapialertingv2beta1.RuleStatus, len(source.Rules))
groupState := promrules.StateInactive
sourceRuleMap := make(map[string]*alerting.AlertingRule, len(source.Rules))
for i := range source.Rules {
rule := source.Rules[i]
if len(rule.Labels) > 0 {
if ruleId, ok := rule.Labels[alertingv2beta1.RuleLabelKeyRuleId]; ok {
sourceRuleMap[ruleId] = rule
}
}
}
for i, ruleId := range ruleIds {
rule, ok := sourceRuleMap[ruleId]
if !ok {
target.RulesStatus[i] = kapialertingv2beta1.RuleStatus{
State: stateInactiveString,
Health: string(promrules.HealthUnknown),
}
continue
}
// the group state takes the max state of its rules
if ruleState := parseAlertState(rule.State); ruleState > groupState {
groupState = ruleState
@@ -764,7 +828,7 @@ func copyRuleGroupStatus(source *alerting.RuleGroup, target *kapialertingv2beta1
ruleStatus.Expr = rule.Query
}
}
target.RulesStatus = append(target.RulesStatus, ruleStatus)
target.RulesStatus[i] = ruleStatus
}
target.State = groupState.String()
}

21
staging/src/kubesphere.io/api/alerting/v2beta1/rulegroup_webhook.go
View File

@@ -25,6 +25,7 @@ import (
yaml "gopkg.in/yaml.v3"
runtime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/uuid"
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"
@@ -32,6 +33,8 @@ import (
var rulegrouplog = logf.Log.WithName("rulegroup")
const RuleLabelKeyRuleId = "rule_id"
func (r *RuleGroup) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(r).
@@ -50,10 +53,26 @@ func (r *RuleGroup) Default() {
rule.Expr = intstr.FromString(rule.ExprBuilder.Workload.Build())
}
}
setRuleId(&rule.Rule)
r.Spec.Rules[i] = rule
}
}
func setRuleId(rule *Rule) {
var setRuleId = true
if len(rule.Labels) > 0 {
if _, ok := rule.Labels[RuleLabelKeyRuleId]; ok {
setRuleId = false
}
}
if setRuleId {
if rule.Labels == nil {
rule.Labels = make(map[string]string)
}
rule.Labels[RuleLabelKeyRuleId] = string(uuid.NewUUID())
}
}
var _ webhook.Validator = &RuleGroup{}
func (r *RuleGroup) ValidateCreate() error {
@@ -171,6 +190,7 @@ func (r *ClusterRuleGroup) Default() {
rule.Expr = intstr.FromString(rule.ExprBuilder.Node.Build())
}
}
setRuleId(&rule.Rule)
r.Spec.Rules[i] = rule
}
}
@@ -224,6 +244,7 @@ func (r *GlobalRuleGroup) Default() {
rule.Expr = intstr.FromString(rule.ExprBuilder.Workload.Build())
}
}
setRuleId(&rule.Rule)
r.Spec.Rules[i] = rule
}
}