@@ -45,14 +45,10 @@ var (
|
|||||||
|
|
||||||
type RuleLevel string
|
type RuleLevel string
|
||||||
|
|
||||||
type AlertingRuleQualifier struct {
|
type AlertingRule struct {
|
||||||
Id string `json:"id,omitempty" description:"rule id is mainly for the builtin rules"`
|
Id string `json:"id,omitempty" description:"rule id is only used by built-in alerting rules"`
|
||||||
Name string `json:"name,omitempty" description:"rule name which should be uniq for custom rules in the specified namespace"`
|
Name string `json:"name,omitempty" description:"rule name should be unique in one namespace for custom alerting rules"`
|
||||||
Level RuleLevel `json:"level,omitempty" description:"rule level is only for the custom rules and its value is one of cluster, namespace"`
|
|
||||||
Custom bool `json:"custom" description:"whether to be a custom rule. The builtin rules are not custom and can only be viewed"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AlertingRuleProps struct {
|
|
||||||
Query string `json:"query,omitempty" description:"prometheus query expression, grammars of which may be referred to https://prometheus.io/docs/prometheus/latest/querying/basics/"`
|
Query string `json:"query,omitempty" description:"prometheus query expression, grammars of which may be referred to https://prometheus.io/docs/prometheus/latest/querying/basics/"`
|
||||||
Duration string `json:"duration,omitempty" description:"duration an alert transitions from Pending to Firing state, which must match ^([0-9]+)(y|w|d|h|m|s|ms)$"`
|
Duration string `json:"duration,omitempty" description:"duration an alert transitions from Pending to Firing state, which must match ^([0-9]+)(y|w|d|h|m|s|ms)$"`
|
||||||
Labels map[string]string `json:"labels,omitempty" description:"extra labels to attach to the resulting alert sample vectors (the key string has to match [a-zA-Z_][a-zA-Z0-9_]*). eg: a typical label called severity, whose value may be info, warning, error, critical, is usually used to indicate the severity of an alert"`
|
Labels map[string]string `json:"labels,omitempty" description:"extra labels to attach to the resulting alert sample vectors (the key string has to match [a-zA-Z_][a-zA-Z0-9_]*). eg: a typical label called severity, whose value may be info, warning, error, critical, is usually used to indicate the severity of an alert"`
|
||||||
@@ -60,11 +56,7 @@ type AlertingRuleProps struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type PostableAlertingRule struct {
|
type PostableAlertingRule struct {
|
||||||
Name string `json:"name,omitempty" description:"rule name which should be uniq for custom rules in the specified namespace"`
|
AlertingRule `json:",omitempty"`
|
||||||
Alias string `json:"alias,omitempty" description:"alias for the rule"`
|
|
||||||
Description string `json:"description,omitempty" description:"description for the rule"`
|
|
||||||
|
|
||||||
AlertingRuleProps `json:",omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (r *PostableAlertingRule) Validate() error {
|
func (r *PostableAlertingRule) Validate() error {
|
||||||
@@ -95,18 +87,15 @@ func (r *PostableAlertingRule) Validate() error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type GettableAlertingRule struct {
|
type GettableAlertingRule struct {
|
||||||
AlertingRuleQualifier `json:",omitempty"`
|
AlertingRule `json:",omitempty"`
|
||||||
AlertingRuleProps `json:",omitempty"`
|
|
||||||
|
|
||||||
Alias string `json:"alias,omitempty" description:"alias for the rule"`
|
|
||||||
Description string `json:"description,omitempty" description:"description for the rule"`
|
|
||||||
|
|
||||||
State string `json:"state,omitempty" description:"state of a rule based on its alerts, one of firing, pending, inactive"`
|
State string `json:"state,omitempty" description:"state of a rule based on its alerts, one of firing, pending, inactive"`
|
||||||
Health string `json:"health,omitempty" description:"health state of a rule based on the last execution, one of ok, err, unknown"`
|
Health string `json:"health,omitempty" description:"health state of a rule based on the last execution, one of ok, err, unknown"`
|
||||||
LastError string `json:"lastError,omitempty" description:"error for the last execution"`
|
LastError string `json:"lastError,omitempty" description:"error for the last execution"`
|
||||||
EvaluationDurationSeconds float64 `json:"evaluationTime,omitempty" description:"taken seconds for evaluation of query expression"`
|
EvaluationDurationSeconds float64 `json:"evaluationTime,omitempty" description:"taken seconds for evaluation of query expression"`
|
||||||
LastEvaluation *time.Time `json:"lastEvaluation,omitempty" description:"time for last evaluation of query expression"`
|
LastEvaluation *time.Time `json:"lastEvaluation,omitempty" description:"time for last evaluation of query expression"`
|
||||||
Alerts []*Alert `json:"alerts,omitempty" description:"alerts"`
|
|
||||||
|
Alerts []*Alert `json:"alerts,omitempty" description:"alerts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type GettableAlertingRuleList struct {
|
type GettableAlertingRuleList struct {
|
||||||
@@ -121,7 +110,8 @@ type Alert struct {
|
|||||||
State string `json:"state,omitempty" description:"state"`
|
State string `json:"state,omitempty" description:"state"`
|
||||||
Value string `json:"value,omitempty" description:"the value at the last evaluation of the query expression"`
|
Value string `json:"value,omitempty" description:"the value at the last evaluation of the query expression"`
|
||||||
|
|
||||||
Rule *AlertingRuleQualifier `json:"rule,omitempty" description:"rule triggering the alert"`
|
RuleId string `json:"ruleId,omitempty" description:"rule id triggering the alert"`
|
||||||
|
RuleName string `json:"ruleName,omitempty" description:"rule name triggering the alert"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type AlertList struct {
|
type AlertList struct {
|
||||||
|
|||||||
@@ -110,11 +110,11 @@ func (h *handler) handleGetCustomAlertingRule(req *restful.Request, resp *restfu
|
|||||||
resp.WriteEntity(rule)
|
resp.WriteEntity(rule)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) handleListCustomSpecifiedRuleAlerts(req *restful.Request, resp *restful.Response) {
|
func (h *handler) handleListCustomRuleAlerts(req *restful.Request, resp *restful.Response) {
|
||||||
namespace := req.PathParameter("namespace")
|
namespace := req.PathParameter("namespace")
|
||||||
ruleName := req.PathParameter("rule_name")
|
ruleName := req.PathParameter("rule_name")
|
||||||
|
|
||||||
alerts, err := h.operator.ListCustomSpecifiedRuleAlerts(req.Request.Context(), namespace, ruleName)
|
alerts, err := h.operator.ListCustomRuleAlerts(req.Request.Context(), namespace, ruleName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Error(err)
|
klog.Error(err)
|
||||||
switch {
|
switch {
|
||||||
@@ -266,10 +266,10 @@ func (h *handler) handleGetBuiltinAlertingRule(req *restful.Request, resp *restf
|
|||||||
resp.WriteEntity(rule)
|
resp.WriteEntity(rule)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (h *handler) handleListBuiltinSpecifiedRuleAlerts(req *restful.Request, resp *restful.Response) {
|
func (h *handler) handleListBuiltinRuleAlerts(req *restful.Request, resp *restful.Response) {
|
||||||
ruleId := req.PathParameter("rule_id")
|
ruleId := req.PathParameter("rule_id")
|
||||||
|
|
||||||
alerts, err := h.operator.ListBuiltinSpecifiedRuleAlerts(req.Request.Context(), ruleId)
|
alerts, err := h.operator.ListBuiltinRuleAlerts(req.Request.Context(), ruleId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Error(err)
|
klog.Error(err)
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -76,7 +76,7 @@ func AddToContainer(container *restful.Container, informers informers.InformerFa
|
|||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
|
|
||||||
ws.Route(ws.GET("/rules/{rule_name}/alerts").
|
ws.Route(ws.GET("/rules/{rule_name}/alerts").
|
||||||
To(handler.handleListCustomSpecifiedRuleAlerts).
|
To(handler.handleListCustomRuleAlerts).
|
||||||
Doc("list the alerts of the cluster-level custom alerting rule with the specified name").
|
Doc("list the alerts of the cluster-level custom alerting rule with the specified name").
|
||||||
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
@@ -132,7 +132,7 @@ func AddToContainer(container *restful.Container, informers informers.InformerFa
|
|||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
|
|
||||||
ws.Route(ws.GET("/namespaces/{namespace}/rules/{rule_name}/alerts").
|
ws.Route(ws.GET("/namespaces/{namespace}/rules/{rule_name}/alerts").
|
||||||
To(handler.handleListCustomSpecifiedRuleAlerts).
|
To(handler.handleListCustomRuleAlerts).
|
||||||
Doc("get the alerts of the custom alerting rule with the specified name in the specified namespace").
|
Doc("get the alerts of the custom alerting rule with the specified name in the specified namespace").
|
||||||
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
@@ -188,7 +188,7 @@ func AddToContainer(container *restful.Container, informers informers.InformerFa
|
|||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
|
|
||||||
ws.Route(ws.GET("/builtin/rules/{rule_id}/alerts").
|
ws.Route(ws.GET("/builtin/rules/{rule_id}/alerts").
|
||||||
To(handler.handleListBuiltinSpecifiedRuleAlerts).
|
To(handler.handleListBuiltinRuleAlerts).
|
||||||
Doc("list the alerts of the builtin(non-custom) alerting rule with the specified id").
|
Doc("list the alerts of the builtin(non-custom) alerting rule with the specified id").
|
||||||
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
Returns(http.StatusOK, ksapi.StatusOK, []customalertingv1alpha1.Alert{}).
|
||||||
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
Metadata(restfulspec.KeyOpenAPITags, []string{constants.CustomAlertingTag}))
|
||||||
|
|||||||
@@ -47,8 +47,8 @@ type Operator interface {
|
|||||||
queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
|
queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
|
||||||
// GetCustomAlertingRule gets the custom alerting rule with the given name.
|
// GetCustomAlertingRule gets the custom alerting rule with the given name.
|
||||||
GetCustomAlertingRule(ctx context.Context, namespace, ruleName string) (*v1alpha1.GettableAlertingRule, error)
|
GetCustomAlertingRule(ctx context.Context, namespace, ruleName string) (*v1alpha1.GettableAlertingRule, error)
|
||||||
// ListCustomSpecifiedRuleAlerts lists the alerts of the custom alerting rule with the given name.
|
// ListCustomRuleAlerts lists the alerts of the custom alerting rule with the given name.
|
||||||
ListCustomSpecifiedRuleAlerts(ctx context.Context, namespace, ruleName string) ([]*v1alpha1.Alert, error)
|
ListCustomRuleAlerts(ctx context.Context, namespace, ruleName string) ([]*v1alpha1.Alert, error)
|
||||||
// CreateCustomAlertingRule creates a custom alerting rule.
|
// CreateCustomAlertingRule creates a custom alerting rule.
|
||||||
CreateCustomAlertingRule(ctx context.Context, namespace string, rule *v1alpha1.PostableAlertingRule) error
|
CreateCustomAlertingRule(ctx context.Context, namespace string, rule *v1alpha1.PostableAlertingRule) error
|
||||||
// UpdateCustomAlertingRule updates the custom alerting rule with the given name.
|
// UpdateCustomAlertingRule updates the custom alerting rule with the given name.
|
||||||
@@ -64,8 +64,8 @@ type Operator interface {
|
|||||||
queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
|
queryParams *v1alpha1.AlertQueryParams) (*v1alpha1.AlertList, error)
|
||||||
// GetBuiltinAlertingRule gets the builtin(non-custom) alerting rule with the given id
|
// GetBuiltinAlertingRule gets the builtin(non-custom) alerting rule with the given id
|
||||||
GetBuiltinAlertingRule(ctx context.Context, ruleId string) (*v1alpha1.GettableAlertingRule, error)
|
GetBuiltinAlertingRule(ctx context.Context, ruleId string) (*v1alpha1.GettableAlertingRule, error)
|
||||||
// ListBuiltinSpecifiedRuleAlerts lists the alerts of the builtin(non-custom) alerting rule with the given id
|
// ListBuiltinRuleAlerts lists the alerts of the builtin(non-custom) alerting rule with the given id
|
||||||
ListBuiltinSpecifiedRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error)
|
ListBuiltinRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewOperator(informers informers.InformerFactory,
|
func NewOperator(informers informers.InformerFactory,
|
||||||
@@ -90,7 +90,7 @@ func NewOperator(informers informers.InformerFactory,
|
|||||||
if option != nil && len(option.ThanosRuleResourceLabels) != 0 {
|
if option != nil && len(option.ThanosRuleResourceLabels) != 0 {
|
||||||
lblStrings := strings.Split(option.ThanosRuleResourceLabels, ",")
|
lblStrings := strings.Split(option.ThanosRuleResourceLabels, ",")
|
||||||
for _, lblString := range lblStrings {
|
for _, lblString := range lblStrings {
|
||||||
lbl := strings.Split(lblString, "=")
|
lbl := strings.Split(strings.TrimSpace(lblString), "=")
|
||||||
if len(lbl) == 2 {
|
if len(lbl) == 2 {
|
||||||
o.thanosRuleResourceLabels[lbl[0]] = lbl[1]
|
o.thanosRuleResourceLabels[lbl[0]] = lbl[1]
|
||||||
}
|
}
|
||||||
@@ -183,7 +183,7 @@ func (o *operator) GetCustomAlertingRule(ctx context.Context, namespace, ruleNam
|
|||||||
return o.getCustomAlertingRule(ctx, ruleNamespace, ruleName, level)
|
return o.getCustomAlertingRule(ctx, ruleNamespace, ruleName, level)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *operator) ListCustomSpecifiedRuleAlerts(ctx context.Context, namespace, ruleName string) (
|
func (o *operator) ListCustomRuleAlerts(ctx context.Context, namespace, ruleName string) (
|
||||||
[]*v1alpha1.Alert, error) {
|
[]*v1alpha1.Alert, error) {
|
||||||
|
|
||||||
rule, err := o.GetCustomAlertingRule(ctx, namespace, ruleName)
|
rule, err := o.GetCustomAlertingRule(ctx, namespace, ruleName)
|
||||||
@@ -223,7 +223,7 @@ func (o *operator) GetBuiltinAlertingRule(ctx context.Context, ruleId string) (
|
|||||||
return o.getBuiltinAlertingRule(ctx, ruleId)
|
return o.getBuiltinAlertingRule(ctx, ruleId)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (o *operator) ListBuiltinSpecifiedRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error) {
|
func (o *operator) ListBuiltinRuleAlerts(ctx context.Context, ruleId string) ([]*v1alpha1.Alert, error) {
|
||||||
rule, err := o.getBuiltinAlertingRule(ctx, ruleId)
|
rule, err := o.getBuiltinAlertingRule(ctx, ruleId)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -290,7 +290,7 @@ func (o *operator) listCustomAlertingRules(ctx context.Context, ruleNamespace *c
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return rules.MixAlertingRules(ruleNamespace.Name, &rules.ResourceRuleChunk{
|
return rules.GetAlertingRulesStatus(ruleNamespace.Name, &rules.ResourceRuleChunk{
|
||||||
ResourceRulesMap: resourceRulesMap,
|
ResourceRulesMap: resourceRulesMap,
|
||||||
Custom: true,
|
Custom: true,
|
||||||
Level: level,
|
Level: level,
|
||||||
@@ -322,10 +322,10 @@ func (o *operator) getCustomAlertingRule(ctx context.Context, ruleNamespace *cor
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return rules.MixAlertingRule(ruleNamespace.Name, &rules.ResourceRuleSole{
|
return rules.GetAlertingRuleStatus(ruleNamespace.Name, &rules.ResourceRule{
|
||||||
ResourceRule: *resourceRule,
|
ResourceRuleItem: *resourceRule,
|
||||||
Custom: true,
|
Custom: true,
|
||||||
Level: level,
|
Level: level,
|
||||||
}, ruleGroups, ruler.ExternalLabels())
|
}, ruleGroups, ruler.ExternalLabels())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -361,7 +361,7 @@ func (o *operator) listBuiltinAlertingRules(ctx context.Context) (
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return rules.MixAlertingRules(ruleNamespace.Name, &rules.ResourceRuleChunk{
|
return rules.GetAlertingRulesStatus(ruleNamespace.Name, &rules.ResourceRuleChunk{
|
||||||
ResourceRulesMap: resourceRulesMap,
|
ResourceRulesMap: resourceRulesMap,
|
||||||
Custom: false,
|
Custom: false,
|
||||||
Level: v1alpha1.RuleLevelCluster,
|
Level: v1alpha1.RuleLevelCluster,
|
||||||
@@ -413,10 +413,10 @@ func (o *operator) getBuiltinAlertingRule(ctx context.Context, ruleId string) (*
|
|||||||
return nil, v1alpha1.ErrAlertingRuleNotFound
|
return nil, v1alpha1.ErrAlertingRuleNotFound
|
||||||
}
|
}
|
||||||
|
|
||||||
return rules.MixAlertingRule(ruleNamespace.Name, &rules.ResourceRuleSole{
|
return rules.GetAlertingRuleStatus(ruleNamespace.Name, &rules.ResourceRule{
|
||||||
ResourceRule: *resourceRule,
|
ResourceRuleItem: *resourceRule,
|
||||||
Custom: false,
|
Custom: false,
|
||||||
Level: v1alpha1.RuleLevelCluster,
|
Level: v1alpha1.RuleLevelCluster,
|
||||||
}, ruleGroups, ruler.ExternalLabels())
|
}, ruleGroups, ruler.ExternalLabels())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -562,7 +562,7 @@ func (o *operator) getPrometheusRuler() (rules.Ruler, error) {
|
|||||||
return nil, errors.Wrap(err, "error listing prometheuses")
|
return nil, errors.Wrap(err, "error listing prometheuses")
|
||||||
}
|
}
|
||||||
if len(prometheuses) > 1 {
|
if len(prometheuses) > 1 {
|
||||||
// it is not supported temporarily to have multiple prometheuses in the monitoring namespace
|
// It is not supported to have multiple Prometheus instances in the monitoring namespace for now
|
||||||
return nil, errors.Errorf(
|
return nil, errors.Errorf(
|
||||||
"there is more than one prometheus custom resource in %s", rulerNamespace)
|
"there is more than one prometheus custom resource in %s", rulerNamespace)
|
||||||
}
|
}
|
||||||
@@ -579,7 +579,7 @@ func (o *operator) getThanosRuler() (rules.Ruler, error) {
|
|||||||
return nil, errors.Wrap(err, "error listing thanosrulers: ")
|
return nil, errors.Wrap(err, "error listing thanosrulers: ")
|
||||||
}
|
}
|
||||||
if len(thanosrulers) > 1 {
|
if len(thanosrulers) > 1 {
|
||||||
// it is not supported temporarily to have multiple thanosrulers in the monitoring namespace
|
// It is not supported to have multiple thanosruler instances in the monitoring namespace for now
|
||||||
return nil, errors.Errorf(
|
return nil, errors.Errorf(
|
||||||
"there is more than one thanosruler custom resource in %s", rulerNamespace)
|
"there is more than one thanosruler custom resource in %s", rulerNamespace)
|
||||||
}
|
}
|
||||||
@@ -592,14 +592,11 @@ func (o *operator) getThanosRuler() (rules.Ruler, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func parseToPrometheusRule(rule *v1alpha1.PostableAlertingRule) *promresourcesv1.Rule {
|
func parseToPrometheusRule(rule *v1alpha1.PostableAlertingRule) *promresourcesv1.Rule {
|
||||||
lbls := rule.Labels
|
|
||||||
lbls[rules.LabelKeyInternalRuleAlias] = rule.Alias
|
|
||||||
lbls[rules.LabelKeyInternalRuleDescription] = rule.Description
|
|
||||||
return &promresourcesv1.Rule{
|
return &promresourcesv1.Rule{
|
||||||
Alert: rule.Name,
|
Alert: rule.Name,
|
||||||
Expr: intstr.FromString(rule.Query),
|
Expr: intstr.FromString(rule.Query),
|
||||||
For: rule.Duration,
|
For: rule.Duration,
|
||||||
Labels: lbls,
|
Labels: rule.Labels,
|
||||||
Annotations: rule.Annotations,
|
Annotations: rule.Annotations,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -104,7 +104,7 @@ func (c *RuleCache) getResourceRuleCaches(ruler Ruler, ruleNamespace *corev1.Nam
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
|
func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
|
||||||
extraRuleResourceSelector labels.Selector, idOrName string) (*ResourceRule, error) {
|
extraRuleResourceSelector labels.Selector, idOrName string) (*ResourceRuleItem, error) {
|
||||||
|
|
||||||
caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
|
caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -114,12 +114,12 @@ func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var rules []*ResourceRule
|
var rules []*ResourceRuleItem
|
||||||
switch ruler.(type) {
|
switch ruler.(type) {
|
||||||
case *PrometheusRuler:
|
case *PrometheusRuler:
|
||||||
for rn, rc := range caches {
|
for rn, rc := range caches {
|
||||||
if rule, ok := rc.IdRules[idOrName]; ok {
|
if rule, ok := rc.IdRules[idOrName]; ok {
|
||||||
rules = append(rules, &ResourceRule{
|
rules = append(rules, &ResourceRuleItem{
|
||||||
Group: rule.Group,
|
Group: rule.Group,
|
||||||
Id: rule.Id,
|
Id: rule.Id,
|
||||||
Rule: rule.Rule.DeepCopy(),
|
Rule: rule.Rule.DeepCopy(),
|
||||||
@@ -131,7 +131,7 @@ func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
|
|||||||
for rn, rc := range caches {
|
for rn, rc := range caches {
|
||||||
if nrules, ok := rc.NameRules[idOrName]; ok {
|
if nrules, ok := rc.NameRules[idOrName]; ok {
|
||||||
for _, nrule := range nrules {
|
for _, nrule := range nrules {
|
||||||
rules = append(rules, &ResourceRule{
|
rules = append(rules, &ResourceRuleItem{
|
||||||
Group: nrule.Group,
|
Group: nrule.Group,
|
||||||
Id: nrule.Id,
|
Id: nrule.Id,
|
||||||
Rule: nrule.Rule.DeepCopy(),
|
Rule: nrule.Rule.DeepCopy(),
|
||||||
@@ -156,7 +156,7 @@ func (c *RuleCache) GetRule(ruler Ruler, ruleNamespace *corev1.Namespace,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (c *RuleCache) ListRules(ruler Ruler, ruleNamespace *corev1.Namespace,
|
func (c *RuleCache) ListRules(ruler Ruler, ruleNamespace *corev1.Namespace,
|
||||||
extraRuleResourceSelector labels.Selector) (map[string]*ResourceRules, error) {
|
extraRuleResourceSelector labels.Selector) (map[string]*ResourceRuleCollection, error) {
|
||||||
|
|
||||||
caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
|
caches, err := c.getResourceRuleCaches(ruler, ruleNamespace, extraRuleResourceSelector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -166,17 +166,17 @@ func (c *RuleCache) ListRules(ruler Ruler, ruleNamespace *corev1.Namespace,
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
ret := make(map[string]*ResourceRules)
|
ret := make(map[string]*ResourceRuleCollection)
|
||||||
for rn, rc := range caches {
|
for rn, rc := range caches {
|
||||||
rrs := &ResourceRules{
|
rrs := &ResourceRuleCollection{
|
||||||
GroupSet: make(map[string]struct{}),
|
GroupSet: make(map[string]struct{}),
|
||||||
IdRules: make(map[string]*ResourceRule),
|
IdRules: make(map[string]*ResourceRuleItem),
|
||||||
NameRules: make(map[string][]*ResourceRule),
|
NameRules: make(map[string][]*ResourceRuleItem),
|
||||||
}
|
}
|
||||||
for name, rules := range rc.NameRules {
|
for name, rules := range rc.NameRules {
|
||||||
for _, rule := range rules {
|
for _, rule := range rules {
|
||||||
rrs.GroupSet[rule.Group] = struct{}{}
|
rrs.GroupSet[rule.Group] = struct{}{}
|
||||||
rr := &ResourceRule{
|
rr := &ResourceRuleItem{
|
||||||
Group: rule.Group,
|
Group: rule.Group,
|
||||||
Id: rule.Id,
|
Id: rule.Id,
|
||||||
Rule: rule.Rule.DeepCopy(),
|
Rule: rule.Rule.DeepCopy(),
|
||||||
|
|||||||
@@ -5,27 +5,27 @@ import (
|
|||||||
"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
|
"kubesphere.io/kubesphere/pkg/api/customalerting/v1alpha1"
|
||||||
)
|
)
|
||||||
|
|
||||||
type ResourceRules struct {
|
type ResourceRuleCollection struct {
|
||||||
GroupSet map[string]struct{}
|
GroupSet map[string]struct{}
|
||||||
IdRules map[string]*ResourceRule
|
IdRules map[string]*ResourceRuleItem
|
||||||
NameRules map[string][]*ResourceRule
|
NameRules map[string][]*ResourceRuleItem
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResourceRule struct {
|
type ResourceRuleItem struct {
|
||||||
ResourceName string
|
ResourceName string
|
||||||
Group string
|
Group string
|
||||||
Id string
|
Id string
|
||||||
Rule *promresourcesv1.Rule
|
Rule *promresourcesv1.Rule
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResourceRuleSole struct {
|
type ResourceRule struct {
|
||||||
Level v1alpha1.RuleLevel
|
Level v1alpha1.RuleLevel
|
||||||
Custom bool
|
Custom bool
|
||||||
ResourceRule
|
ResourceRuleItem
|
||||||
}
|
}
|
||||||
|
|
||||||
type ResourceRuleChunk struct {
|
type ResourceRuleChunk struct {
|
||||||
Level v1alpha1.RuleLevel
|
Level v1alpha1.RuleLevel
|
||||||
Custom bool
|
Custom bool
|
||||||
ResourceRulesMap map[string]*ResourceRules
|
ResourceRulesMap map[string]*ResourceRuleCollection
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,12 +21,10 @@ import (
|
|||||||
const (
|
const (
|
||||||
ErrGenRuleId = "error generating rule id"
|
ErrGenRuleId = "error generating rule id"
|
||||||
|
|
||||||
LabelKeyInternalRuleGroup = "__rule_group__"
|
LabelKeyInternalRuleGroup = "__rule_group__"
|
||||||
LabelKeyInternalRuleName = "__rule_name__"
|
LabelKeyInternalRuleName = "__rule_name__"
|
||||||
LabelKeyInternalRuleQuery = "__rule_query__"
|
LabelKeyInternalRuleQuery = "__rule_query__"
|
||||||
LabelKeyInternalRuleDuration = "__rule_duration__"
|
LabelKeyInternalRuleDuration = "__rule_duration__"
|
||||||
LabelKeyInternalRuleAlias = "__rule_alias__"
|
|
||||||
LabelKeyInternalRuleDescription = "__rule_description__"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func FormatExpr(expr string) (string, error) {
|
func FormatExpr(expr string) (string, error) {
|
||||||
@@ -127,9 +125,9 @@ func GenEndpointRuleId(group string, epRule *customalerting.AlertingRule,
|
|||||||
return prommodel.Fingerprint(prommodel.LabelsToSignature(lbls)).String(), nil
|
return prommodel.Fingerprint(prommodel.LabelsToSignature(lbls)).String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// MixAlertingRules mix rules from prometheusrule custom resources and rules from endpoints.
|
// GetAlertingRulesStatus mix rules from prometheusrule custom resources and rules from endpoints.
|
||||||
// Use rules from prometheusrule custom resources as the main reference.
|
// Use rules from prometheusrule custom resources as the main reference.
|
||||||
func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRuleGroups []*customalerting.RuleGroup,
|
func GetAlertingRulesStatus(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRuleGroups []*customalerting.RuleGroup,
|
||||||
extLabels func() map[string]string) ([]*v1alpha1.GettableAlertingRule, error) {
|
extLabels func() map[string]string) ([]*v1alpha1.GettableAlertingRule, error) {
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -161,7 +159,7 @@ func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRule
|
|||||||
}
|
}
|
||||||
if ruleChunk.Custom {
|
if ruleChunk.Custom {
|
||||||
// guarantee the names of the custom alerting rules not to be repeated
|
// guarantee the names of the custom alerting rules not to be repeated
|
||||||
var m = make(map[string][]*ResourceRule)
|
var m = make(map[string][]*ResourceRuleItem)
|
||||||
for _, resourceRules := range ruleChunk.ResourceRulesMap {
|
for _, resourceRules := range ruleChunk.ResourceRulesMap {
|
||||||
for name, rrArr := range resourceRules.NameRules {
|
for name, rrArr := range resourceRules.NameRules {
|
||||||
m[name] = append(m[name], rrArr...)
|
m[name] = append(m[name], rrArr...)
|
||||||
@@ -176,7 +174,7 @@ func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRule
|
|||||||
}
|
}
|
||||||
resRule := rrArr[0]
|
resRule := rrArr[0]
|
||||||
epRule := idEpRules[resRule.Id]
|
epRule := idEpRules[resRule.Id]
|
||||||
if r := mixAlertingRule(resRule, epRule, ruleChunk.Custom, ruleChunk.Level); r != nil {
|
if r := getAlertingRuleStatus(resRule, epRule, ruleChunk.Custom, ruleChunk.Level); r != nil {
|
||||||
ret = append(ret, r)
|
ret = append(ret, r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,7 +184,7 @@ func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRule
|
|||||||
var m = make(map[string]*v1alpha1.GettableAlertingRule)
|
var m = make(map[string]*v1alpha1.GettableAlertingRule)
|
||||||
for _, resourceRules := range ruleChunk.ResourceRulesMap {
|
for _, resourceRules := range ruleChunk.ResourceRulesMap {
|
||||||
for id, rule := range resourceRules.IdRules {
|
for id, rule := range resourceRules.IdRules {
|
||||||
if r := mixAlertingRule(rule, idEpRules[id], ruleChunk.Custom, ruleChunk.Level); r != nil {
|
if r := getAlertingRuleStatus(rule, idEpRules[id], ruleChunk.Custom, ruleChunk.Level); r != nil {
|
||||||
m[id] = r
|
m[id] = r
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -199,7 +197,7 @@ func MixAlertingRules(ruleNamespace string, ruleChunk *ResourceRuleChunk, epRule
|
|||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func MixAlertingRule(ruleNamespace string, rule *ResourceRuleSole, epRuleGroups []*customalerting.RuleGroup,
|
func GetAlertingRuleStatus(ruleNamespace string, rule *ResourceRule, epRuleGroups []*customalerting.RuleGroup,
|
||||||
extLabels func() map[string]string) (*v1alpha1.GettableAlertingRule, error) {
|
extLabels func() map[string]string) (*v1alpha1.GettableAlertingRule, error) {
|
||||||
|
|
||||||
if rule == nil || rule.Rule == nil {
|
if rule == nil || rule.Rule == nil {
|
||||||
@@ -245,52 +243,27 @@ func MixAlertingRule(ruleNamespace string, rule *ResourceRuleSole, epRuleGroups
|
|||||||
epRule = epRules[rule.Id]
|
epRule = epRules[rule.Id]
|
||||||
}
|
}
|
||||||
|
|
||||||
return mixAlertingRule(&rule.ResourceRule, epRule, rule.Custom, rule.Level), nil
|
return getAlertingRuleStatus(&rule.ResourceRuleItem, epRule, rule.Custom, rule.Level), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func mixAlertingRule(resRule *ResourceRule, epRule *customalerting.AlertingRule,
|
func getAlertingRuleStatus(resRule *ResourceRuleItem, epRule *customalerting.AlertingRule,
|
||||||
custom bool, level v1alpha1.RuleLevel) *v1alpha1.GettableAlertingRule {
|
custom bool, level v1alpha1.RuleLevel) *v1alpha1.GettableAlertingRule {
|
||||||
|
|
||||||
if resRule == nil || resRule.Rule == nil {
|
if resRule == nil || resRule.Rule == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
|
||||||
alias string
|
|
||||||
descrption string
|
|
||||||
lbls map[string]string
|
|
||||||
)
|
|
||||||
if len(resRule.Rule.Labels) > 0 {
|
|
||||||
lbls = make(map[string]string)
|
|
||||||
for k, v := range resRule.Rule.Labels {
|
|
||||||
switch k {
|
|
||||||
case LabelKeyInternalRuleAlias:
|
|
||||||
alias = v
|
|
||||||
case LabelKeyInternalRuleDescription:
|
|
||||||
descrption = v
|
|
||||||
default:
|
|
||||||
lbls[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
rule := v1alpha1.GettableAlertingRule{
|
rule := v1alpha1.GettableAlertingRule{
|
||||||
AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
|
AlertingRule: v1alpha1.AlertingRule{
|
||||||
Id: resRule.Id,
|
Id: resRule.Id,
|
||||||
Name: resRule.Rule.Alert,
|
Name: resRule.Rule.Alert,
|
||||||
Custom: custom,
|
|
||||||
Level: level,
|
|
||||||
},
|
|
||||||
AlertingRuleProps: v1alpha1.AlertingRuleProps{
|
|
||||||
Query: resRule.Rule.Expr.String(),
|
Query: resRule.Rule.Expr.String(),
|
||||||
Duration: resRule.Rule.For,
|
Duration: resRule.Rule.For,
|
||||||
Labels: lbls,
|
Labels: resRule.Rule.Labels,
|
||||||
Annotations: resRule.Rule.Annotations,
|
Annotations: resRule.Rule.Annotations,
|
||||||
},
|
},
|
||||||
Alias: alias,
|
State: stateInactiveString,
|
||||||
Description: descrption,
|
Health: string(rules.HealthUnknown),
|
||||||
State: stateInactiveString,
|
|
||||||
Health: string(rules.HealthUnknown),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if epRule != nil {
|
if epRule != nil {
|
||||||
@@ -316,25 +289,15 @@ func mixAlertingRule(resRule *ResourceRule, epRule *customalerting.AlertingRule,
|
|||||||
rule.State = aState
|
rule.State = aState
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
var lbls map[string]string
|
|
||||||
if len(a.Labels) > 0 {
|
|
||||||
lbls = make(map[string]string)
|
|
||||||
for k, v := range a.Labels {
|
|
||||||
switch k {
|
|
||||||
case LabelKeyInternalRuleAlias, LabelKeyInternalRuleDescription:
|
|
||||||
default:
|
|
||||||
lbls[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
rule.Alerts = append(rule.Alerts, &v1alpha1.Alert{
|
rule.Alerts = append(rule.Alerts, &v1alpha1.Alert{
|
||||||
ActiveAt: a.ActiveAt,
|
ActiveAt: a.ActiveAt,
|
||||||
Labels: lbls,
|
Labels: a.Labels,
|
||||||
Annotations: a.Annotations,
|
Annotations: a.Annotations,
|
||||||
State: aState,
|
State: aState,
|
||||||
Value: a.Value,
|
Value: a.Value,
|
||||||
|
|
||||||
Rule: &rule.AlertingRuleQualifier,
|
RuleId: rule.Id,
|
||||||
|
RuleName: rule.Name,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -353,13 +316,9 @@ func ParseAlertingRules(epRuleGroups []*customalerting.RuleGroup, custom bool, l
|
|||||||
}
|
}
|
||||||
if filterFunc(g.Name, id, r) {
|
if filterFunc(g.Name, id, r) {
|
||||||
rule := &v1alpha1.GettableAlertingRule{
|
rule := &v1alpha1.GettableAlertingRule{
|
||||||
AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
|
AlertingRule: v1alpha1.AlertingRule{
|
||||||
Id: id,
|
Id: id,
|
||||||
Name: r.Name,
|
Name: r.Name,
|
||||||
Custom: custom,
|
|
||||||
Level: level,
|
|
||||||
},
|
|
||||||
AlertingRuleProps: v1alpha1.AlertingRuleProps{
|
|
||||||
Query: r.Query,
|
Query: r.Query,
|
||||||
Duration: parseDurationSeconds(r.Duration),
|
Duration: parseDurationSeconds(r.Duration),
|
||||||
Labels: r.Labels,
|
Labels: r.Labels,
|
||||||
@@ -392,7 +351,8 @@ func ParseAlertingRules(epRuleGroups []*customalerting.RuleGroup, custom bool, l
|
|||||||
State: aState,
|
State: aState,
|
||||||
Value: a.Value,
|
Value: a.Value,
|
||||||
|
|
||||||
Rule: &rule.AlertingRuleQualifier,
|
RuleId: rule.Id,
|
||||||
|
RuleName: rule.Name,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
ret = append(ret, rule)
|
ret = append(ret, rule)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
|
"kubesphere.io/kubesphere/pkg/simple/client/customalerting"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMixAlertingRules(t *testing.T) {
|
func TestGetAlertingRulesStatus(t *testing.T) {
|
||||||
var tests = []struct {
|
var tests = []struct {
|
||||||
description string
|
description string
|
||||||
ruleNamespace string
|
ruleNamespace string
|
||||||
@@ -20,26 +20,26 @@ func TestMixAlertingRules(t *testing.T) {
|
|||||||
extLabels func() map[string]string
|
extLabels func() map[string]string
|
||||||
expected []*v1alpha1.GettableAlertingRule
|
expected []*v1alpha1.GettableAlertingRule
|
||||||
}{{
|
}{{
|
||||||
description: "mix custom rules",
|
description: "get alerting rules status",
|
||||||
ruleNamespace: "test",
|
ruleNamespace: "test",
|
||||||
resourceRuleChunk: &ResourceRuleChunk{
|
resourceRuleChunk: &ResourceRuleChunk{
|
||||||
Level: v1alpha1.RuleLevelNamespace,
|
Level: v1alpha1.RuleLevelNamespace,
|
||||||
Custom: true,
|
Custom: true,
|
||||||
ResourceRulesMap: map[string]*ResourceRules{
|
ResourceRulesMap: map[string]*ResourceRuleCollection{
|
||||||
"custom-alerting-rule-jqbgn": &ResourceRules{
|
"custom-alerting-rule-jqbgn": &ResourceRuleCollection{
|
||||||
GroupSet: map[string]struct{}{"alerting.custom.defaults": struct{}{}},
|
GroupSet: map[string]struct{}{"alerting.custom.defaults": struct{}{}},
|
||||||
NameRules: map[string][]*ResourceRule{
|
NameRules: map[string][]*ResourceRuleItem{
|
||||||
"f89836879157ca88": []*ResourceRule{{
|
"ca7f09e76954e67c": []*ResourceRuleItem{{
|
||||||
ResourceName: "custom-alerting-rule-jqbgn",
|
ResourceName: "custom-alerting-rule-jqbgn",
|
||||||
Group: "alerting.custom.defaults",
|
Group: "alerting.custom.defaults",
|
||||||
Id: "f89836879157ca88",
|
Id: "ca7f09e76954e67c",
|
||||||
Rule: &promresourcesv1.Rule{
|
Rule: &promresourcesv1.Rule{
|
||||||
Alert: "TestCPUUsageHigh",
|
Alert: "TestCPUUsageHigh",
|
||||||
Expr: intstr.FromString(`namespace:workload_cpu_usage:sum{namespace="test"} > 1`),
|
Expr: intstr.FromString(`namespace:workload_cpu_usage:sum{namespace="test"} > 1`),
|
||||||
For: "1m",
|
For: "1m",
|
||||||
Labels: map[string]string{
|
Annotations: map[string]string{
|
||||||
LabelKeyInternalRuleAlias: "The alias is here",
|
"alias": "The alias is here",
|
||||||
LabelKeyInternalRuleDescription: "The description is here",
|
"description": "The description is here",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
@@ -56,34 +56,31 @@ func TestMixAlertingRules(t *testing.T) {
|
|||||||
Duration: 60,
|
Duration: 60,
|
||||||
Health: string(rules.HealthGood),
|
Health: string(rules.HealthGood),
|
||||||
State: stateInactiveString,
|
State: stateInactiveString,
|
||||||
Labels: map[string]string{
|
Annotations: map[string]string{
|
||||||
LabelKeyInternalRuleAlias: "The alias is here",
|
"alias": "The alias is here",
|
||||||
LabelKeyInternalRuleDescription: "The description is here",
|
"description": "The description is here",
|
||||||
},
|
},
|
||||||
}},
|
}},
|
||||||
}},
|
}},
|
||||||
expected: []*v1alpha1.GettableAlertingRule{{
|
expected: []*v1alpha1.GettableAlertingRule{{
|
||||||
AlertingRuleQualifier: v1alpha1.AlertingRuleQualifier{
|
AlertingRule: v1alpha1.AlertingRule{
|
||||||
Id: "f89836879157ca88",
|
Id: "ca7f09e76954e67c",
|
||||||
Name: "TestCPUUsageHigh",
|
Name: "TestCPUUsageHigh",
|
||||||
Level: v1alpha1.RuleLevelNamespace,
|
|
||||||
Custom: true,
|
|
||||||
},
|
|
||||||
AlertingRuleProps: v1alpha1.AlertingRuleProps{
|
|
||||||
Query: `namespace:workload_cpu_usage:sum{namespace="test"} > 1`,
|
Query: `namespace:workload_cpu_usage:sum{namespace="test"} > 1`,
|
||||||
Duration: "1m",
|
Duration: "1m",
|
||||||
Labels: map[string]string{},
|
Annotations: map[string]string{
|
||||||
|
"alias": "The alias is here",
|
||||||
|
"description": "The description is here",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
Alias: "The alias is here",
|
Health: string(rules.HealthGood),
|
||||||
Description: "The description is here",
|
State: stateInactiveString,
|
||||||
Health: string(rules.HealthGood),
|
|
||||||
State: stateInactiveString,
|
|
||||||
}},
|
}},
|
||||||
}}
|
}}
|
||||||
|
|
||||||
for _, test := range tests {
|
for _, test := range tests {
|
||||||
t.Run(test.description, func(t *testing.T) {
|
t.Run(test.description, func(t *testing.T) {
|
||||||
rules, err := MixAlertingRules(test.ruleNamespace, test.resourceRuleChunk, test.ruleGroups, test.extLabels)
|
rules, err := GetAlertingRulesStatus(test.ruleNamespace, test.resourceRuleChunk, test.ruleGroups, test.extLabels)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user