diff --git a/Makefile b/Makefile index 6d5c835cd1..decec5eadf 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ go-run-enterprise: ## Runs the enterprise go backend server SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN=tcp://127.0.0.1:9000 \ SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_CLUSTER=cluster \ go run -race \ - $(GO_BUILD_CONTEXT_ENTERPRISE)/*.go + $(GO_BUILD_CONTEXT_ENTERPRISE)/*.go server .PHONY: go-test go-test: ## Runs go unit tests diff --git a/ee/query-service/rules/anomaly.go b/ee/query-service/rules/anomaly.go index f4464d1290..6420374b33 100644 --- a/ee/query-service/rules/anomaly.go +++ b/ee/query-service/rules/anomaly.go @@ -246,7 +246,9 @@ func (r *AnomalyRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, t continue } } - results, err := r.Threshold.ShouldAlert(*series, r.Unit()) + results, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{ + ActiveAlerts: r.ActiveAlertsLabelFP(), + }) if err != nil { return nil, err } @@ -296,7 +298,9 @@ func (r *AnomalyRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUID, continue } } - results, err := r.Threshold.ShouldAlert(*series, r.Unit()) + results, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{ + ActiveAlerts: r.ActiveAlertsLabelFP(), + }) if err != nil { return nil, err } @@ -410,6 +414,7 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (interface{}, erro GeneratorURL: r.GeneratorURL(), Receivers: ruleReceiverMap[lbs.Map()[ruletypes.LabelThresholdName]], Missing: smpl.IsMissing, + IsRecovering: smpl.IsRecovering, } } @@ -422,6 +427,9 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (interface{}, erro alert.Value = a.Value alert.Annotations = a.Annotations + // Update the recovering and missing state of existing alert + alert.IsRecovering = a.IsRecovering + alert.Missing = a.Missing if v, ok := alert.Labels.Map()[ruletypes.LabelThresholdName]; ok { alert.Receivers = ruleReceiverMap[v] } @@ -480,6 +488,30 @@ func (r *AnomalyRule) Eval(ctx context.Context, ts time.Time) (interface{}, erro Value: a.Value, }) } + + // We need to change firing alert to recovering if the returned sample meets recovery threshold + changeFiringToRecovering := a.State == model.StateFiring && a.IsRecovering + // We need to change recovering alerts to firing if the returned sample meets target threshold + changeRecoveringToFiring := a.State == model.StateRecovering && !a.IsRecovering && !a.Missing + // in any of the above case we need to update the status of alert + if changeFiringToRecovering || changeRecoveringToFiring { + state := model.StateRecovering + if changeRecoveringToFiring { + state = model.StateFiring + } + a.State = state + r.logger.DebugContext(ctx, "converting alert state", "name", r.Name(), "state", state) + itemsToAdd = append(itemsToAdd, model.RuleStateHistory{ + RuleID: r.ID(), + RuleName: r.Name(), + State: state, + StateChanged: true, + UnixMilli: ts.UnixMilli(), + Labels: model.LabelsString(labelsJSON), + Fingerprint: a.QueryResultLables.Hash(), + Value: a.Value, + }) + } } currentState := r.State() diff --git a/pkg/query-service/model/alerting.go b/pkg/query-service/model/alerting.go index 969cdac7d3..5d2d6858c1 100644 --- a/pkg/query-service/model/alerting.go +++ b/pkg/query-service/model/alerting.go @@ -12,9 +12,12 @@ import ( // AlertState denotes the state of an active alert. type AlertState int +// The enum values are ordered by priority (lowest to highest). +// When determining overall rule state, higher numeric values take precedence. const ( StateInactive AlertState = iota StatePending + StateRecovering StateFiring StateNoData StateDisabled @@ -32,6 +35,8 @@ func (s AlertState) String() string { return "nodata" case StateDisabled: return "disabled" + case StateRecovering: + return "recovering" } panic(errors.Errorf("unknown alert state: %d", s)) } @@ -58,6 +63,8 @@ func (s *AlertState) UnmarshalJSON(b []byte) error { *s = StateNoData case "disabled": *s = StateDisabled + case "recovering": + *s = StateRecovering default: *s = StateInactive } @@ -83,6 +90,8 @@ func (s *AlertState) Scan(value interface{}) error { *s = StateNoData case "disabled": *s = StateDisabled + case "recovering": + *s = StateRecovering } return nil } diff --git a/pkg/query-service/rules/base_rule.go b/pkg/query-service/rules/base_rule.go index 62669fad17..80d896d787 100644 --- a/pkg/query-service/rules/base_rule.go +++ b/pkg/query-service/rules/base_rule.go @@ -191,6 +191,26 @@ func (r *BaseRule) currentAlerts() []*ruletypes.Alert { return alerts } +// ActiveAlertsLabelFP returns a map of active alert labels fingerprint and +// the fingerprint is computed using the QueryResultLables.Hash() method. +// We use the QueryResultLables instead of labels as these labels are raw labels +// that we get from the sample. +// This is useful in cases where we want to check if an alert is still active +// based on the labels we have. +func (r *BaseRule) ActiveAlertsLabelFP() map[uint64]struct{} { + r.mtx.Lock() + defer r.mtx.Unlock() + + activeAlerts := make(map[uint64]struct{}, len(r.Active)) + for _, alert := range r.Active { + if alert == nil || alert.QueryResultLables == nil { + continue + } + activeAlerts[alert.QueryResultLables.Hash()] = struct{}{} + } + return activeAlerts +} + func (r *BaseRule) EvalDelay() time.Duration { return r.evalDelay } diff --git a/pkg/query-service/rules/base_rule_test.go b/pkg/query-service/rules/base_rule_test.go index 8391ded1fc..9618762cc2 100644 --- a/pkg/query-service/rules/base_rule_test.go +++ b/pkg/query-service/rules/base_rule_test.go @@ -1,9 +1,10 @@ package rules import ( - "github.com/stretchr/testify/require" "testing" + "github.com/stretchr/testify/require" + v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" ruletypes "github.com/SigNoz/signoz/pkg/types/ruletypes" ) @@ -74,7 +75,7 @@ func TestBaseRule_RequireMinPoints(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - _, err := test.rule.Threshold.ShouldAlert(*test.series, "") + _, err := test.rule.Threshold.Eval(*test.series, "", ruletypes.EvalData{}) require.NoError(t, err) require.Equal(t, len(test.series.Points) >= test.rule.ruleCondition.RequiredNumPoints, test.shouldAlert) }) diff --git a/pkg/query-service/rules/manager.go b/pkg/query-service/rules/manager.go index a935aa2590..608d5daa13 100644 --- a/pkg/query-service/rules/manager.go +++ b/pkg/query-service/rules/manager.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "fmt" - "github.com/SigNoz/signoz/pkg/query-service/utils/labels" "log/slog" "sort" "strings" "sync" "time" + "github.com/SigNoz/signoz/pkg/query-service/utils/labels" + "go.uber.org/zap" "github.com/go-openapi/strfmt" diff --git a/pkg/query-service/rules/prom_rule.go b/pkg/query-service/rules/prom_rule.go index a880b98d4c..0f46d76068 100644 --- a/pkg/query-service/rules/prom_rule.go +++ b/pkg/query-service/rules/prom_rule.go @@ -159,7 +159,9 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error) continue } - results, err := r.Threshold.ShouldAlert(toCommonSeries(series), r.Unit()) + results, err := r.Threshold.Eval(toCommonSeries(series), r.Unit(), ruletypes.EvalData{ + ActiveAlerts: r.ActiveAlertsLabelFP(), + }) if err != nil { return nil, err } @@ -233,6 +235,7 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error) Value: result.V, GeneratorURL: r.GeneratorURL(), Receivers: ruleReceiverMap[lbs.Map()[ruletypes.LabelThresholdName]], + IsRecovering: result.IsRecovering, } } } @@ -245,6 +248,9 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error) if alert, ok := r.Active[h]; ok && alert.State != model.StateInactive { alert.Value = a.Value alert.Annotations = a.Annotations + // Update the recovering and missing state of existing alert + alert.IsRecovering = a.IsRecovering + alert.Missing = a.Missing if v, ok := alert.Labels.Map()[ruletypes.LabelThresholdName]; ok { alert.Receivers = ruleReceiverMap[v] } @@ -304,6 +310,29 @@ func (r *PromRule) Eval(ctx context.Context, ts time.Time) (interface{}, error) }) } + // We need to change firing alert to recovering if the returned sample meets recovery threshold + changeAlertingToRecovering := a.State == model.StateFiring && a.IsRecovering + // We need to change recovering alerts to firing if the returned sample meets target threshold + changeRecoveringToFiring := a.State == model.StateRecovering && !a.IsRecovering && !a.Missing + // in any of the above case we need to update the status of alert + if changeAlertingToRecovering || changeRecoveringToFiring { + state := model.StateRecovering + if changeRecoveringToFiring { + state = model.StateFiring + } + a.State = state + r.logger.DebugContext(ctx, "converting alert state", "name", r.Name(), "state", state) + itemsToAdd = append(itemsToAdd, model.RuleStateHistory{ + RuleID: r.ID(), + RuleName: r.Name(), + State: state, + StateChanged: true, + UnixMilli: ts.UnixMilli(), + Labels: model.LabelsString(labelsJSON), + Fingerprint: a.QueryResultLables.Hash(), + Value: a.Value, + }) + } } r.health = ruletypes.HealthGood r.lastError = err diff --git a/pkg/query-service/rules/promrule_test.go b/pkg/query-service/rules/promrule_test.go index ef0dbcab32..a1a1cc69ae 100644 --- a/pkg/query-service/rules/promrule_test.go +++ b/pkg/query-service/rules/promrule_test.go @@ -23,7 +23,7 @@ func getVectorValues(vectors []ruletypes.Sample) []float64 { return values } -func TestPromRuleShouldAlert(t *testing.T) { +func TestPromRuleEval(t *testing.T) { postableRule := ruletypes.PostableRule{ AlertName: "Test Rule", AlertType: ruletypes.AlertTypeMetric, @@ -696,7 +696,7 @@ func TestPromRuleShouldAlert(t *testing.T) { assert.NoError(t, err) } - resultVectors, err := rule.Threshold.ShouldAlert(toCommonSeries(c.values), rule.Unit()) + resultVectors, err := rule.Threshold.Eval(toCommonSeries(c.values), rule.Unit(), ruletypes.EvalData{}) assert.NoError(t, err) // Compare full result vector with expected vector diff --git a/pkg/query-service/rules/rule.go b/pkg/query-service/rules/rule.go index 850b01879a..f9580d8d97 100644 --- a/pkg/query-service/rules/rule.go +++ b/pkg/query-service/rules/rule.go @@ -24,6 +24,8 @@ type Rule interface { HoldDuration() time.Duration State() model.AlertState ActiveAlerts() []*ruletypes.Alert + // ActiveAlertsLabelFP returns a map of active alert labels fingerprint + ActiveAlertsLabelFP() map[uint64]struct{} PreferredChannels() []string diff --git a/pkg/query-service/rules/threshold_rule.go b/pkg/query-service/rules/threshold_rule.go index 2b873c4320..17f0eb18d4 100644 --- a/pkg/query-service/rules/threshold_rule.go +++ b/pkg/query-service/rules/threshold_rule.go @@ -488,7 +488,9 @@ func (r *ThresholdRule) buildAndRunQuery(ctx context.Context, orgID valuer.UUID, continue } } - resultSeries, err := r.Threshold.ShouldAlert(*series, r.Unit()) + resultSeries, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{ + ActiveAlerts: r.ActiveAlertsLabelFP(), + }) if err != nil { return nil, err } @@ -565,7 +567,9 @@ func (r *ThresholdRule) buildAndRunQueryV5(ctx context.Context, orgID valuer.UUI continue } } - resultSeries, err := r.Threshold.ShouldAlert(*series, r.Unit()) + resultSeries, err := r.Threshold.Eval(*series, r.Unit(), ruletypes.EvalData{ + ActiveAlerts: r.ActiveAlertsLabelFP(), + }) if err != nil { return nil, err } @@ -666,13 +670,14 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er // Links with timestamps should go in annotations since labels // is used alert grouping, and we want to group alerts with the same // label set, but different timestamps, together. - if r.typ == ruletypes.AlertTypeTraces { + switch r.typ { + case ruletypes.AlertTypeTraces: link := r.prepareLinksToTraces(ctx, ts, smpl.Metric) if link != "" && r.hostFromSource() != "" { r.logger.InfoContext(ctx, "adding traces link to annotations", "link", fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)) annotations = append(annotations, labels.Label{Name: "related_traces", Value: fmt.Sprintf("%s/traces-explorer?%s", r.hostFromSource(), link)}) } - } else if r.typ == ruletypes.AlertTypeLogs { + case ruletypes.AlertTypeLogs: link := r.prepareLinksToLogs(ctx, ts, smpl.Metric) if link != "" && r.hostFromSource() != "" { r.logger.InfoContext(ctx, "adding logs link to annotations", "link", fmt.Sprintf("%s/logs/logs-explorer?%s", r.hostFromSource(), link)) @@ -698,6 +703,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er GeneratorURL: r.GeneratorURL(), Receivers: ruleReceiverMap[lbs.Map()[ruletypes.LabelThresholdName]], Missing: smpl.IsMissing, + IsRecovering: smpl.IsRecovering, } } @@ -711,6 +717,9 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er alert.Value = a.Value alert.Annotations = a.Annotations + // Update the recovering and missing state of existing alert + alert.IsRecovering = a.IsRecovering + alert.Missing = a.Missing if v, ok := alert.Labels.Map()[ruletypes.LabelThresholdName]; ok { alert.Receivers = ruleReceiverMap[v] } @@ -735,6 +744,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er delete(r.Active, fp) } if a.State != model.StateInactive { + r.logger.DebugContext(ctx, "converting firing alert to inActive", "name", r.Name()) a.State = model.StateInactive a.ResolvedAt = ts itemsToAdd = append(itemsToAdd, model.RuleStateHistory{ @@ -752,6 +762,7 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er } if a.State == model.StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration { + r.logger.DebugContext(ctx, "converting pending alert to firing", "name", r.Name()) a.State = model.StateFiring a.FiredAt = ts state := model.StateFiring @@ -769,6 +780,30 @@ func (r *ThresholdRule) Eval(ctx context.Context, ts time.Time) (interface{}, er Value: a.Value, }) } + + // We need to change firing alert to recovering if the returned sample meets recovery threshold + changeAlertingToRecovering := a.State == model.StateFiring && a.IsRecovering + // We need to change recovering alerts to firing if the returned sample meets target threshold + changeRecoveringToFiring := a.State == model.StateRecovering && !a.IsRecovering && !a.Missing + // in any of the above case we need to update the status of alert + if changeAlertingToRecovering || changeRecoveringToFiring { + state := model.StateRecovering + if changeRecoveringToFiring { + state = model.StateFiring + } + a.State = state + r.logger.DebugContext(ctx, "converting alert state", "name", r.Name(), "state", state) + itemsToAdd = append(itemsToAdd, model.RuleStateHistory{ + RuleID: r.ID(), + RuleName: r.Name(), + State: state, + StateChanged: true, + UnixMilli: ts.UnixMilli(), + Labels: model.LabelsString(labelsJSON), + Fingerprint: a.QueryResultLables.Hash(), + Value: a.Value, + }) + } } currentState := r.State() diff --git a/pkg/query-service/rules/threshold_rule_test.go b/pkg/query-service/rules/threshold_rule_test.go index 2e75236691..c0689a312d 100644 --- a/pkg/query-service/rules/threshold_rule_test.go +++ b/pkg/query-service/rules/threshold_rule_test.go @@ -29,9 +29,9 @@ import ( qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5" ) -func TestThresholdRuleShouldAlert(t *testing.T) { +func TestThresholdRuleEvalBackwardCompat(t *testing.T) { postableRule := ruletypes.PostableRule{ - AlertName: "Tricky Condition Tests", + AlertName: "Eval Backward Compatibility Test without recovery target", AlertType: ruletypes.AlertTypeMetric, RuleType: ruletypes.RuleTypeThreshold, Evaluation: &ruletypes.EvaluationEnvelope{ruletypes.RollingEvaluation, ruletypes.RollingWindow{ @@ -59,750 +59,7 @@ func TestThresholdRuleShouldAlert(t *testing.T) { logger := instrumentationtest.New().Logger() - cases := []struct { - values v3.Series - expectAlert bool - compareOp string - matchType string - target float64 - expectedAlertSample v3.Point - }{ - // Test cases for Equals Always - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "2", // Always - target: 0.0, - expectedAlertSample: v3.Point{Value: 0.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 1.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "2", // Always - target: 0.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "2", // Always - target: 0.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "2", // Always - target: 0.0, - }, - // Test cases for Equals Once - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 0.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 1.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 0.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 0.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "1", // Once - target: 0.0, - }, - // Test cases for Greater Than Always - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Greater Than - matchType: "2", // Always - target: 1.5, - expectedAlertSample: v3.Point{Value: 2.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: false, - compareOp: "1", // Greater Than - matchType: "2", // Always - target: 4.5, - }, - // Test cases for Greater Than Once - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Greater Than - matchType: "1", // Once - target: 4.5, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 4.0}, - {Value: 4.0}, - {Value: 4.0}, - {Value: 4.0}, - {Value: 4.0}, - }, - }, - expectAlert: false, - compareOp: "1", // Greater Than - matchType: "1", // Once - target: 4.5, - }, - // Test cases for Not Equals Always - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "2", // Always - target: 0.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 0.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "2", // Always - target: 0.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "2", // Always - target: 0.0, - expectedAlertSample: v3.Point{Value: 1.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "2", // Always - target: 0.0, - }, - // Test cases for Not Equals Once - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 1.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - {Value: 0.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "1", // Once - target: 0.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 0.0}, - {Value: 0.0}, - {Value: 1.0}, - {Value: 0.0}, - {Value: 1.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 1.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - {Value: 1.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "1", // Once - target: 0.0, - expectedAlertSample: v3.Point{Value: 1.0}, - }, - // Test cases for Less Than Always - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.5}, - {Value: 1.5}, - {Value: 1.5}, - {Value: 1.5}, - {Value: 1.5}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "2", // Always - target: 4, - expectedAlertSample: v3.Point{Value: 1.5}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 1.5}, - {Value: 2.5}, - {Value: 1.5}, - {Value: 3.5}, - {Value: 1.5}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "2", // Always - target: 4, - expectedAlertSample: v3.Point{Value: 3.5}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - }, - }, - expectAlert: false, - compareOp: "2", // Less Than - matchType: "2", // Always - target: 4, - }, - // Test cases for Less Than Once - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 2.5}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "1", // Once - target: 4, - expectedAlertSample: v3.Point{Value: 2.5}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - {Value: 4.5}, - }, - }, - expectAlert: false, - compareOp: "2", // Less Than - matchType: "1", // Once - target: 4, - }, - // Test cases for OnAverage - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "3", // OnAverage - target: 6.0, - expectedAlertSample: v3.Point{Value: 6.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "3", // OnAverage - target: 4.5, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "3", // OnAverage - target: 4.5, - expectedAlertSample: v3.Point{Value: 6.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "3", // OnAverage - target: 6.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Greater Than - matchType: "3", // OnAverage - target: 4.5, - expectedAlertSample: v3.Point{Value: 6.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 11.0}, - {Value: 4.0}, - {Value: 3.0}, - {Value: 7.0}, - {Value: 12.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Above - matchType: "2", // Always - target: 2.0, - expectedAlertSample: v3.Point{Value: 3.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 11.0}, - {Value: 4.0}, - {Value: 3.0}, - {Value: 7.0}, - {Value: 12.0}, - }, - }, - expectAlert: true, - compareOp: "2", // Below - matchType: "2", // Always - target: 13.0, - expectedAlertSample: v3.Point{Value: 12.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "3", // OnAverage - target: 12.0, - expectedAlertSample: v3.Point{Value: 6.0}, - }, - // Test cases for InTotal - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "4", // InTotal - target: 30.0, - expectedAlertSample: v3.Point{Value: 30.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 4.0}, - {Value: 6.0}, - {Value: 8.0}, - {Value: 2.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "4", // InTotal - target: 20.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "4", // InTotal - target: 9.0, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "4", // InTotal - target: 10.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Greater Than - matchType: "4", // InTotal - target: 10.0, - expectedAlertSample: v3.Point{Value: 20.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "1", // Greater Than - matchType: "4", // InTotal - target: 20.0, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "4", // InTotal - target: 30.0, - expectedAlertSample: v3.Point{Value: 20.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "2", // Less Than - matchType: "4", // InTotal - target: 20.0, - }, - // Test cases for Last - // greater than last - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "1", // Greater Than - matchType: "5", // Last - target: 5.0, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "1", // Greater Than - matchType: "5", // Last - target: 20.0, - }, - // less than last - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "2", // Less Than - matchType: "5", // Last - target: 15.0, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "2", // Less Than - matchType: "5", // Last - target: 5.0, - }, - // equals last - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "3", // Equals - matchType: "5", // Last - target: 10.0, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "3", // Equals - matchType: "5", // Last - target: 5.0, - }, - // not equals last - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: true, - compareOp: "4", // Not Equals - matchType: "5", // Last - target: 5.0, - expectedAlertSample: v3.Point{Value: 10.0}, - }, - { - values: v3.Series{ - Points: []v3.Point{ - {Value: 10.0}, - {Value: 10.0}, - }, - }, - expectAlert: false, - compareOp: "4", // Not Equals - matchType: "5", // Last - target: 10.0, - }, - } - - for idx, c := range cases { + for idx, c := range tcThresholdRuleEvalNoRecoveryTarget { postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{ Kind: ruletypes.BasicThresholdKind, Spec: ruletypes.BasicRuleThresholds{ @@ -824,7 +81,9 @@ func TestThresholdRuleShouldAlert(t *testing.T) { values.Points[i].Timestamp = time.Now().UnixMilli() } - resultVectors, err := rule.Threshold.ShouldAlert(c.values, rule.Unit()) + resultVectors, err := rule.Threshold.Eval(c.values, rule.Unit(), ruletypes.EvalData{ + ActiveAlerts: map[uint64]struct{}{}, + }) assert.NoError(t, err, "Test case %d", idx) // Compare result vectors with expected behavior @@ -1201,7 +460,7 @@ func TestThresholdRuleLabelNormalization(t *testing.T) { values.Points[i].Timestamp = time.Now().UnixMilli() } - vector, err := rule.Threshold.ShouldAlert(c.values, rule.Unit()) + vector, err := rule.Threshold.Eval(c.values, rule.Unit(), ruletypes.EvalData{}) assert.NoError(t, err) for name, value := range c.values.Labels { @@ -1211,7 +470,7 @@ func TestThresholdRuleLabelNormalization(t *testing.T) { } // Get result vectors from threshold evaluation - resultVectors, err := rule.Threshold.ShouldAlert(c.values, rule.Unit()) + resultVectors, err := rule.Threshold.Eval(c.values, rule.Unit(), ruletypes.EvalData{}) assert.NoError(t, err, "Test case %d", idx) // Compare result vectors with expected behavior @@ -2171,3 +1430,330 @@ func TestMultipleThresholdRule(t *testing.T) { } } } + +func TestThresholdRuleEval_BasicCases(t *testing.T) { + postableRule := ruletypes.PostableRule{ + AlertName: "Eval Recovery Threshold Test", + AlertType: ruletypes.AlertTypeMetric, + RuleType: ruletypes.RuleTypeThreshold, + Evaluation: &ruletypes.EvaluationEnvelope{Kind: ruletypes.RollingEvaluation, Spec: ruletypes.RollingWindow{ + EvalWindow: ruletypes.Duration(5 * time.Minute), + Frequency: ruletypes.Duration(1 * time.Minute), + }}, + RuleCondition: &ruletypes.RuleCondition{ + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{ + Key: "probe_success", + }, + AggregateOperator: v3.AggregateOperatorNoOp, + DataSource: v3.DataSourceMetrics, + Expression: "A", + }, + }, + }, + }, + } + + runEvalTests(t, postableRule, tcThresholdRuleEval) + +} + +func TestThresholdRuleEval_MatchPlusCompareOps(t *testing.T) { + postableRule := ruletypes.PostableRule{ + AlertName: "Eval Match Plus Compare Ops Threshold Test", + AlertType: ruletypes.AlertTypeMetric, + RuleType: ruletypes.RuleTypeThreshold, + Evaluation: &ruletypes.EvaluationEnvelope{Kind: ruletypes.RollingEvaluation, Spec: ruletypes.RollingWindow{ + EvalWindow: ruletypes.Duration(5 * time.Minute), + Frequency: ruletypes.Duration(1 * time.Minute), + }}, + RuleCondition: &ruletypes.RuleCondition{ + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{ + Key: "probe_success", + }, + AggregateOperator: v3.AggregateOperatorNoOp, + DataSource: v3.DataSourceMetrics, + Expression: "A", + }, + }, + }, + }, + } + + runEvalTests(t, postableRule, tcThresholdRuleEvalMatchPlusCompareOps) + +} + +func runEvalTests(t *testing.T, postableRule ruletypes.PostableRule, testCases []recoveryTestCase) { + logger := instrumentationtest.New().Logger() + for _, c := range testCases { + t.Run(c.description, func(t *testing.T) { + // Prepare threshold with recovery target + threshold := ruletypes.BasicRuleThreshold{ + Name: c.thresholdName, + TargetValue: &c.target, + RecoveryTarget: c.recoveryTarget, + MatchType: ruletypes.MatchType(c.matchType), + CompareOp: ruletypes.CompareOp(c.compareOp), + } + + // Build thresholds list + thresholds := ruletypes.BasicRuleThresholds{threshold} + + // Add additional thresholds if specified + for _, addThreshold := range c.additionalThresholds { + thresholds = append(thresholds, ruletypes.BasicRuleThreshold{ + Name: addThreshold.name, + TargetValue: &addThreshold.target, + RecoveryTarget: addThreshold.recoveryTarget, + MatchType: ruletypes.MatchType(addThreshold.matchType), + CompareOp: ruletypes.CompareOp(addThreshold.compareOp), + }) + } + + postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{ + Kind: ruletypes.BasicThresholdKind, + Spec: thresholds, + } + + rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute)) + if err != nil { + assert.NoError(t, err) + return + } + + values := c.values + for i := range values.Points { + values.Points[i].Timestamp = time.Now().UnixMilli() + } + + // Prepare activeAlerts: if nil, auto-calculate from labels + thresholdName + activeAlerts := c.activeAlerts + if activeAlerts == nil { + sampleLabels := ruletypes.PrepareSampleLabelsForRule(values.Labels, c.thresholdName) + alertHash := sampleLabels.Hash() + activeAlerts = map[uint64]struct{}{alertHash: {}} + // Handle other thresholds + for _, addThreshold := range c.additionalThresholds { + sampleLabels := ruletypes.PrepareSampleLabelsForRule(values.Labels, addThreshold.name) + alertHash := sampleLabels.Hash() + activeAlerts[alertHash] = struct{}{} + } + } + + evalData := ruletypes.EvalData{ + ActiveAlerts: activeAlerts, + } + + resultVectors, err := rule.Threshold.Eval(values, rule.Unit(), evalData) + assert.NoError(t, err) + + // Verify results + if c.expectAlert || c.expectRecovery { + // Either a new alert fires or recovery happens - both return result vectors + assert.NotEmpty(t, resultVectors, "Expected alert or recovery but got no result vectors") + if len(resultVectors) > 0 { + found := false + for _, sample := range resultVectors { + // Check if this is the expected sample + if sample.V == c.expectedAlertSample.Value { + found = true + // Verify IsRecovering flag + assert.Equal(t, c.expectRecovery, sample.IsRecovering, "IsRecovering flag mismatch") + // Verify target value + if c.expectedTarget != 0 || sample.Target != 0 { + assert.InDelta(t, c.expectedTarget, sample.Target, 0.01, "Target value mismatch") + } + if sample.RecoveryTarget != nil { + assert.InDelta(t, *sample.RecoveryTarget, c.expectedRecoveryTarget, 0.01, "Recovery target value mismatch") + } + break + } + } + assert.True(t, found, "Expected alert sample value %.2f not found in result vectors. Got values: %v", c.expectedAlertSample.Value, getVectorValues(resultVectors)) + } + } else { + // No alert and no recovery expected - should be empty + assert.Empty(t, resultVectors, "Expected no alert but got result vectors: %v", resultVectors) + } + }) + } +} + +// runMultiThresholdEvalTests runs tests for multiple threshold scenarios +// where each threshold can be in a different state (firing, recovering, resolved) +func runMultiThresholdEvalTests(t *testing.T, postableRule ruletypes.PostableRule, testCases []multiThresholdTestCase) { + logger := instrumentationtest.New().Logger() + for _, c := range testCases { + t.Run(c.description, func(t *testing.T) { + // Prepare primary threshold + threshold := ruletypes.BasicRuleThreshold{ + Name: c.thresholdName, + TargetValue: &c.target, + RecoveryTarget: c.recoveryTarget, + MatchType: ruletypes.MatchType(c.matchType), + CompareOp: ruletypes.CompareOp(c.compareOp), + } + + // Build thresholds list + thresholds := ruletypes.BasicRuleThresholds{threshold} + + // Add additional thresholds + for _, addThreshold := range c.additionalThresholds { + thresholds = append(thresholds, ruletypes.BasicRuleThreshold{ + Name: addThreshold.name, + TargetValue: &addThreshold.target, + RecoveryTarget: addThreshold.recoveryTarget, + MatchType: ruletypes.MatchType(addThreshold.matchType), + CompareOp: ruletypes.CompareOp(addThreshold.compareOp), + }) + } + + postableRule.RuleCondition.Thresholds = &ruletypes.RuleThresholdData{ + Kind: ruletypes.BasicThresholdKind, + Spec: thresholds, + } + + rule, err := NewThresholdRule("69", valuer.GenerateUUID(), &postableRule, nil, nil, logger, WithEvalDelay(2*time.Minute)) + if err != nil { + assert.NoError(t, err) + return + } + + values := c.values + for i := range values.Points { + values.Points[i].Timestamp = time.Now().UnixMilli() + } + + // Prepare activeAlerts: if nil, auto-calculate from labels + all threshold names + activeAlerts := c.activeAlerts + if activeAlerts == nil { + activeAlerts = make(map[uint64]struct{}) + // Add primary threshold + sampleLabels := ruletypes.PrepareSampleLabelsForRule(values.Labels, c.thresholdName) + alertHash := sampleLabels.Hash() + activeAlerts[alertHash] = struct{}{} + // Add additional thresholds + for _, addThreshold := range c.additionalThresholds { + sampleLabels := ruletypes.PrepareSampleLabelsForRule(values.Labels, addThreshold.name) + alertHash := sampleLabels.Hash() + activeAlerts[alertHash] = struct{}{} + } + } + + evalData := ruletypes.EvalData{ + ActiveAlerts: activeAlerts, + } + + resultVectors, err := rule.Threshold.Eval(values, rule.Unit(), evalData) + assert.NoError(t, err) + + // Validate total sample count + assert.Equal(t, c.ExpectedSampleCount, len(resultVectors), + "Expected %d samples but got %d. Sample values: %v", + c.ExpectedSampleCount, len(resultVectors), getVectorValues(resultVectors)) + + // Build a map of threshold name -> sample for easy lookup + samplesByThreshold := make(map[string]ruletypes.Sample) + for _, sample := range resultVectors { + thresholdName := sample.Metric.Get(ruletypes.LabelThresholdName) + samplesByThreshold[thresholdName] = sample + } + + // Validate each threshold's expected result + for thresholdName, expectation := range c.ExpectedResults { + sample, found := samplesByThreshold[thresholdName] + + if expectation.ShouldReturnSample { + assert.True(t, found, "Expected sample for threshold '%s' but not found in results", thresholdName) + if !found { + continue + } + + // Validate IsRecovering flag + assert.Equal(t, expectation.IsRecovering, sample.IsRecovering, + "Threshold '%s': IsRecovering flag mismatch", thresholdName) + + // Validate sample value + assert.InDelta(t, expectation.SampleValue, sample.V, 0.01, + "Threshold '%s': Sample value mismatch", thresholdName) + + // Validate target value + assert.InDelta(t, expectation.TargetValue, sample.Target, 0.01, + "Threshold '%s': Target value mismatch", thresholdName) + + // Validate recovery target value + if expectation.RecoveryValue != nil { + assert.NotNil(t, sample.RecoveryTarget, + "Threshold '%s': Expected RecoveryTarget to be set but it was nil", thresholdName) + if sample.RecoveryTarget != nil { + assert.InDelta(t, *expectation.RecoveryValue, *sample.RecoveryTarget, 0.01, + "Threshold '%s': RecoveryTarget value mismatch", thresholdName) + } + } + } else { + assert.False(t, found, "Expected NO sample for threshold '%s' but found one with value %.2f", + thresholdName, sample.V) + } + } + + // Validate sample order if specified + if len(c.ExpectedSampleOrder) > 0 { + assert.Equal(t, len(c.ExpectedSampleOrder), len(resultVectors), + "Expected sample order length mismatch") + for i, expectedName := range c.ExpectedSampleOrder { + if i < len(resultVectors) { + actualName := resultVectors[i].Metric.Get(ruletypes.LabelThresholdName) + assert.Equal(t, expectedName, actualName, + "Sample order mismatch at index %d: expected '%s', got '%s'", + i, expectedName, actualName) + } + } + } + }) + } +} + +// TestThresholdRuleEval_MultiThreshold tests multiple threshold scenarios +// where each threshold can be in a different state (firing, recovering, resolved) +func TestThresholdRuleEval_MultiThreshold(t *testing.T) { + postableRule := ruletypes.PostableRule{ + AlertName: "Multi-Threshold Recovery Test", + AlertType: ruletypes.AlertTypeMetric, + RuleType: ruletypes.RuleTypeThreshold, + Evaluation: &ruletypes.EvaluationEnvelope{Kind: ruletypes.RollingEvaluation, Spec: ruletypes.RollingWindow{ + EvalWindow: ruletypes.Duration(5 * time.Minute), + Frequency: ruletypes.Duration(1 * time.Minute), + }}, + RuleCondition: &ruletypes.RuleCondition{ + CompositeQuery: &v3.CompositeQuery{ + QueryType: v3.QueryTypeBuilder, + BuilderQueries: map[string]*v3.BuilderQuery{ + "A": { + QueryName: "A", + StepInterval: 60, + AggregateAttribute: v3.AttributeKey{ + Key: "probe_success", + }, + AggregateOperator: v3.AggregateOperatorNoOp, + DataSource: v3.DataSourceMetrics, + Expression: "A", + }, + }, + }, + }, + } + + runMultiThresholdEvalTests(t, postableRule, tcThresholdRuleEvalMultiThreshold) +} diff --git a/pkg/query-service/rules/threshold_rule_test_data.go b/pkg/query-service/rules/threshold_rule_test_data.go index 8e447e0a45..9552191759 100644 --- a/pkg/query-service/rules/threshold_rule_test_data.go +++ b/pkg/query-service/rules/threshold_rule_test_data.go @@ -1,6 +1,64 @@ package rules -import "time" +import ( + "time" + + v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" +) + +type recoveryTestCase struct { + description string + values v3.Series + expectAlert bool + expectRecovery bool // IsRecovering flag check + compareOp string + matchType string + target float64 + recoveryTarget *float64 // nil to test case where only target value is checked + activeAlerts map[uint64]struct{} // simulates active alert fingerprints. nil = auto-calculate from labels+thresholdName, empty map = no active alerts + additionalThresholds []struct { // additional thresholds to add to the rule (for testing multiple thresholds) + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + } + expectedAlertSample v3.Point + expectedTarget float64 + expectedRecoveryTarget float64 + thresholdName string // for hash calculation +} + +// thresholdExpectation defines expected behavior for a single threshold in multi-threshold tests +type thresholdExpectation struct { + ShouldReturnSample bool // Should this threshold return a sample? + IsRecovering bool // Should IsRecovering be true? + SampleValue float64 // Expected sample value + TargetValue float64 // Expected Target field in sample + RecoveryValue *float64 // Expected RecoveryTarget field in sample (can be nil) +} + +// multiThresholdTestCase extends recoveryTestCase for testing multiple thresholds with detailed per-threshold expectations +type multiThresholdTestCase struct { + // Embed the base struct to reuse all fields + recoveryTestCase + + // ============================================================ + // Multi-threshold expectations + // ============================================================ + + // Map of threshold name → expected behavior + // Key: threshold name (matches thresholdName or additionalThresholds[].name) + // Value: what we expect for that specific threshold + ExpectedResults map[string]thresholdExpectation + + // Total expected samples (for quick validation) + ExpectedSampleCount int + + // Optional: Expected order of samples (by threshold name) + // Used to verify sorting is correct + ExpectedSampleOrder []string +} var ( testCases = []struct { @@ -108,4 +166,2764 @@ var ( target: 200, // 200 GB }, } + + tcThresholdRuleEvalNoRecoveryTarget = []recoveryTestCase{ + // Test cases for Equals Always + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "2", // Always + target: 0.0, + expectedAlertSample: v3.Point{Value: 0.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 1.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "2", // Always + target: 0.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "2", // Always + target: 0.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "2", // Always + target: 0.0, + }, + // Test cases for Equals Once + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 0.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 1.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 0.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 0.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "1", // Once + target: 0.0, + }, + // Test cases for Greater Than Always + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Greater Than + matchType: "2", // Always + target: 1.5, + expectedAlertSample: v3.Point{Value: 2.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: false, + compareOp: "1", // Greater Than + matchType: "2", // Always + target: 4.5, + }, + // Test cases for Greater Than Once + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Greater Than + matchType: "1", // Once + target: 4.5, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 4.0}, + {Value: 4.0}, + {Value: 4.0}, + {Value: 4.0}, + {Value: 4.0}, + }, + }, + expectAlert: false, + compareOp: "1", // Greater Than + matchType: "1", // Once + target: 4.5, + }, + // Test cases for Not Equals Always + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "2", // Always + target: 0.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 0.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "2", // Always + target: 0.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "2", // Always + target: 0.0, + expectedAlertSample: v3.Point{Value: 1.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "2", // Always + target: 0.0, + }, + // Test cases for Not Equals Once + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 1.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + {Value: 0.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "1", // Once + target: 0.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 0.0}, + {Value: 0.0}, + {Value: 1.0}, + {Value: 0.0}, + {Value: 1.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 1.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + {Value: 1.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "1", // Once + target: 0.0, + expectedAlertSample: v3.Point{Value: 1.0}, + }, + // Test cases for Less Than Always + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.5}, + {Value: 1.5}, + {Value: 1.5}, + {Value: 1.5}, + {Value: 1.5}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "2", // Always + target: 4, + expectedAlertSample: v3.Point{Value: 1.5}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.5}, + {Value: 2.5}, + {Value: 1.5}, + {Value: 3.5}, + {Value: 1.5}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "2", // Always + target: 4, + expectedAlertSample: v3.Point{Value: 3.5}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + }, + }, + expectAlert: false, + compareOp: "2", // Less Than + matchType: "2", // Always + target: 4, + }, + // Test cases for Less Than Once + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 2.5}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "1", // Once + target: 4, + expectedAlertSample: v3.Point{Value: 2.5}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + {Value: 4.5}, + }, + }, + expectAlert: false, + compareOp: "2", // Less Than + matchType: "1", // Once + target: 4, + }, + // Test cases for OnAverage + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "3", // OnAverage + target: 6.0, + expectedAlertSample: v3.Point{Value: 6.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "3", // OnAverage + target: 4.5, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "3", // OnAverage + target: 4.5, + expectedAlertSample: v3.Point{Value: 6.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "3", // OnAverage + target: 6.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Greater Than + matchType: "3", // OnAverage + target: 4.5, + expectedAlertSample: v3.Point{Value: 6.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 11.0}, + {Value: 4.0}, + {Value: 3.0}, + {Value: 7.0}, + {Value: 12.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Above + matchType: "2", // Always + target: 2.0, + expectedAlertSample: v3.Point{Value: 3.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 11.0}, + {Value: 4.0}, + {Value: 3.0}, + {Value: 7.0}, + {Value: 12.0}, + }, + }, + expectAlert: true, + compareOp: "2", // Below + matchType: "2", // Always + target: 13.0, + expectedAlertSample: v3.Point{Value: 12.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "3", // OnAverage + target: 12.0, + expectedAlertSample: v3.Point{Value: 6.0}, + }, + // Test cases for InTotal + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "4", // InTotal + target: 30.0, + expectedAlertSample: v3.Point{Value: 30.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 4.0}, + {Value: 6.0}, + {Value: 8.0}, + {Value: 2.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "4", // InTotal + target: 20.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "4", // InTotal + target: 9.0, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "4", // InTotal + target: 10.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Greater Than + matchType: "4", // InTotal + target: 10.0, + expectedAlertSample: v3.Point{Value: 20.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "1", // Greater Than + matchType: "4", // InTotal + target: 20.0, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "4", // InTotal + target: 30.0, + expectedAlertSample: v3.Point{Value: 20.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "2", // Less Than + matchType: "4", // InTotal + target: 20.0, + }, + // Test cases for Last + // greater than last + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "1", // Greater Than + matchType: "5", // Last + target: 5.0, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "1", // Greater Than + matchType: "5", // Last + target: 20.0, + }, + // less than last + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "2", // Less Than + matchType: "5", // Last + target: 15.0, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "2", // Less Than + matchType: "5", // Last + target: 5.0, + }, + // equals last + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "3", // Equals + matchType: "5", // Last + target: 10.0, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "3", // Equals + matchType: "5", // Last + target: 5.0, + }, + // not equals last + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: true, + compareOp: "4", // Not Equals + matchType: "5", // Last + target: 5.0, + expectedAlertSample: v3.Point{Value: 10.0}, + }, + { + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, + {Value: 10.0}, + }, + }, + expectAlert: false, + compareOp: "4", // Not Equals + matchType: "5", // Last + target: 10.0, + }, + } + + tcThresholdRuleEval = []recoveryTestCase{ + // ============================================================ + // Category 1: No Active Alert - Recovery Zone Match + // ============================================================ + // Purpose: Verify recovery threshold is IGNORED when there's no active alert + // Behavior: Even if value is in recovery zone (between target and recovery threshold), + // no alert should be returned because recovery only applies to existing alerts + // Expected: expectAlert=false, expectRecovery=false for all cases + { + description: "Cat1: Above operator - value in recovery zone, no active alert → no alert returned", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: map[uint64]struct{}{}, // No active alerts + thresholdName: "test_threshold", + }, + { + description: "Cat1: Below operator - value in recovery zone, no active alert → no alert returned", + values: v3.Series{ + Points: []v3.Point{ + {Value: 60.0}, + }, + Labels: map[string]string{ + "service": "backend", + }, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: map[uint64]struct{}{}, // No active alerts + thresholdName: "test_threshold", + }, + { + description: "Cat1: NotEq operator - value in recovery zone, no active alert → no alert returned", + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + }, + Labels: map[string]string{ + "service": "api", + }, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "4", // NotEq + matchType: "1", // AtleastOnce + target: 1.0, + recoveryTarget: func() *float64 { v := 0.0; return &v }(), + activeAlerts: map[uint64]struct{}{}, // No active alerts + thresholdName: "test_threshold", + }, + // ============================================================ + // Category 2: Active Alert - In Recovery Zone + // ============================================================ + // Purpose: Verify IsRecovering=true when alert is active and value is in recovery zone + // Behavior: Value has improved (no longer breaches target) but hasn't fully recovered + // (still breaches recovery threshold). This is the "improving" state. + // Expected: expectAlert=true, expectRecovery=true, IsRecovering=true + // Sample uses recovery target value, not main target + { + description: "Cat2: Above operator - active alert, value below target but above recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 90.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "test_threshold_above", + }, + { + description: "Cat2: Below operator - active alert, value above target but below recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 60.0}, + }, + Labels: map[string]string{ + "service": "backend", + }, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 60.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "test_threshold_below", + }, + { + description: "Cat2: NotEq operator - active alert, value equals target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + }, + Labels: map[string]string{ + "service": "api", + }, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEq + matchType: "1", // AtleastOnce + target: 1.0, + recoveryTarget: func() *float64 { v := 0.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 1.0}, + expectedTarget: 1.0, + expectedRecoveryTarget: 0.0, + thresholdName: "test_threshold_noteq", + }, + // ============================================================ + // Category 3: Active Alert - Still Breaching Target + // ============================================================ + // Purpose: Verify normal alert behavior when target threshold is still breached + // Behavior: Value still breaches the main target threshold, so alert continues firing + // normally. Recovery threshold is not checked when target still breaches. + // Expected: expectAlert=true, expectRecovery=false (normal firing alert) + // Sample uses main target value, not recovery target + { + description: "Cat3: Above operator - active alert, value still above target → normal firing alert", + values: v3.Series{ + Points: []v3.Point{ + {Value: 110.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: true, + expectRecovery: false, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 110.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "test_threshold_still_alerting_above", + }, + { + description: "Cat3: Below operator - active alert, value still below target → normal firing alert", + values: v3.Series{ + Points: []v3.Point{ + {Value: 40.0}, + }, + Labels: map[string]string{ + "service": "backend", + }, + }, + expectAlert: true, + expectRecovery: false, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 40.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "test_threshold_still_alerting_below", + }, + { + description: "Cat3: Above operator - active alert, value fully recovered (below recovery) → alert resolved", + values: v3.Series{ + Points: []v3.Point{ + {Value: 75.0}, + }, + Labels: map[string]string{ + "service": "api", + }, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + thresholdName: "test_threshold_no_match", + }, + // ============================================================ + // Category 4: Alert Identity & Fingerprint Matching + // ============================================================ + // Purpose: Verify recovery only applies to alerts with matching fingerprints + // Behavior: Alert fingerprint = hash(series labels + threshold name) + // Recovery requires exact fingerprint match with active alert + // Expected: Recovery only triggers when alert fingerprint matches active alert + { + description: "Cat4: Wrong alert fingerprint - value in recovery zone but different active alert → no recovery", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: map[uint64]struct{}{12345: {}}, // Wrong hash (not matching this series) + thresholdName: "test_threshold_wrong_hash", + }, + { + description: "Cat4: Correct alert fingerprint - value in recovery zone and matching active alert → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName + expectedAlertSample: v3.Point{Value: 90.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "test_threshold_correct_hash", + }, + { + description: "Cat4: Multiple thresholds - each tracks recovery independently based on its own fingerprint", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + }, + Labels: map[string]string{ + "service": "frontend", + }, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 120.0, + recoveryTarget: func() *float64 { v := 100.0; return &v }(), + activeAlerts: nil, // Auto-calculate from labels+thresholdName (only for first threshold) + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + // this will match the second recovery threshold + { + name: "test_threshold_multiple_second", + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "1", // Above + }, + }, + expectedAlertSample: v3.Point{Value: 90.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "test_threshold_multiple", + }, + // Test fully recovered (value past recovery threshold) + { + description: "Cat4: Above operator - active alert, value fully recovered (below recovery) → alert resolves", + values: v3.Series{ + Points: []v3.Point{ + {Value: 75.0}, // below recovery threshold + }, + Labels: map[string]string{"service": "test30"}, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + thresholdName: "cat4_fully_recovered", + }, + { + description: "Cat4: Below operator - active alert, value fully recovered (above recovery) → alert resolves", + values: v3.Series{ + Points: []v3.Point{ + {Value: 75.0}, // above recovery threshold + }, + Labels: map[string]string{"service": "test31"}, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + thresholdName: "cat4_fully_recovered_below", + }, + } + + tcThresholdRuleEvalMatchPlusCompareOps = []recoveryTestCase{ + // ============================================================ + // Category 1: MatchType - AtleastOnce with All CompareOps + // ============================================================ + // Purpose: Verify "at least one point matches" works with recovery for all operators + // Behavior: If ANY point in series matches the condition, alert fires/recovers + // ============================================================ + + { + description: "Cat1: AtleastOnce + Above - active alert, one value in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 75.0}, // below recovery + {Value: 85.0}, // in recovery zone (between 80 and 100) + {Value: 70.0}, // below recovery + }, + Labels: map[string]string{"service": "test1"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 85.0}, // first matching value + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat1_atleastonce_above_recovery", + }, + { + description: "Cat1: AtleastOnce + Below - active alert, one value in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 80.0}, // above recovery + {Value: 60.0}, // in recovery zone (between 50 and 70) + {Value: 75.0}, // above recovery + }, + Labels: map[string]string{"service": "test2"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 60.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "cat1_atleastonce_below_recovery", + }, + { + description: "Cat1: AtleastOnce + Equals - active alert, one value equals recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, // doesn't equal recovery (5.0) + {Value: 5.0}, // equals recovery + {Value: 2.0}, // doesn't equal recovery + }, + Labels: map[string]string{"service": "test3"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "3", // Equals + matchType: "1", // AtleastOnce + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 5.0}, + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat1_atleastonce_equals_recovery", + }, + { + description: "Cat1: AtleastOnce + NotEquals - active alert, values equal target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, // equals target (doesn't breach target) + {Value: 10.0}, // equals target (doesn't breach target) + {Value: 10.0}, // equals target (doesn't breach target) + }, + Labels: map[string]string{"service": "test4"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEquals + matchType: "1", // AtleastOnce + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 10.0}, // All values = 10, which != 5 (recovery condition met) + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat1_atleastonce_noteq_recovery", + }, + { + description: "Cat1: AtleastOnce + OutsideBounds - active alert, |value| in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 85.0}, // |85| >= recovery (80) + }, + Labels: map[string]string{"service": "test26"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "7", // OutsideBounds + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 85.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat1_atleastonce_outsidebounds_recovery", + }, + { + description: "Cat1: AtleastOnce + OutsideBounds - active alert, negative value in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: -85.0}, // |-85| = 85 >= recovery (80) + }, + Labels: map[string]string{"service": "test27"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "7", // OutsideBounds + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: -85.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat1_atleastonce_outsidebounds_negative_recovery", + }, + + // ============================================================ + // Category 2: MatchType - AllTheTimes with All CompareOps + // ============================================================ + // Purpose: Verify "all points must match" works with recovery for all operators + // Behavior: ALL points must match the condition for alert to fire/recover + // ============================================================ + + { + description: "Cat2: AllTheTimes + Above - active alert, all values in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 85.0}, // in recovery zone + {Value: 90.0}, // in recovery zone + {Value: 82.0}, // in recovery zone + }, + Labels: map[string]string{"service": "test5"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "2", // AllTheTimes + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 82.0}, // min value for Above + AllTheTimes + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat2_allthetimes_above_recovery", + }, + { + description: "Cat2: AllTheTimes + Above - active alert, one value below recovery → no recovery", + values: v3.Series{ + Points: []v3.Point{ + {Value: 85.0}, // in recovery zone + {Value: 75.0}, // below recovery (breaks AllTheTimes) + {Value: 90.0}, // in recovery zone + }, + Labels: map[string]string{"service": "test6"}, + }, + expectAlert: false, + expectRecovery: false, + compareOp: "1", // Above + matchType: "2", // AllTheTimes + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + thresholdName: "cat2_allthetimes_above_no_recovery", + }, + { + description: "Cat2: AllTheTimes + Below - active alert, all values in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 60.0}, // in recovery zone + {Value: 55.0}, // in recovery zone + {Value: 65.0}, // in recovery zone + }, + Labels: map[string]string{"service": "test7"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "2", // AllTheTimes + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 65.0}, // max value for Below + AllTheTimes + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "cat2_allthetimes_below_recovery", + }, + { + description: "Cat2: AllTheTimes + Equals - active alert, all values equal recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 5.0}, + {Value: 5.0}, + {Value: 5.0}, + }, + Labels: map[string]string{"service": "test8"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "3", // Equals + matchType: "2", // AllTheTimes + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 5.0}, + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat2_allthetimes_equals_recovery", + }, + { + description: "Cat2: AllTheTimes + NotEquals - active alert, all values equal target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, // equals target (doesn't breach) + {Value: 10.0}, // equals target (doesn't breach) + {Value: 10.0}, // equals target (doesn't breach) + }, + Labels: map[string]string{"service": "test9"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEquals + matchType: "2", // AllTheTimes + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 10.0}, // All equal target, all != recovery + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat2_allthetimes_noteq_recovery", + }, + + // ============================================================ + // Category 3: MatchType - OnAverage with All CompareOps + // ============================================================ + // Purpose: Verify average-based matching works with recovery for all operators + // Behavior: Average of all points is compared against threshold + // ============================================================ + + { + description: "Cat3: OnAverage + Above - active alert, avg in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 85.0}, + {Value: 90.0}, + {Value: 85.0}, + }, // avg = 86.67 + Labels: map[string]string{"service": "test10"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "3", // OnAverage + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 86.66666666666667}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat3_onaverage_above_recovery", + }, + { + description: "Cat3: OnAverage + Below - active alert, avg in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 60.0}, + {Value: 65.0}, + {Value: 55.0}, + }, // avg = 60 + Labels: map[string]string{"service": "test11"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "3", // OnAverage + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 60.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "cat3_onaverage_below_recovery", + }, + { + description: "Cat3: OnAverage + Equals - active alert, avg equals recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 4.0}, + {Value: 5.0}, + {Value: 6.0}, + }, // avg = 5.0 + Labels: map[string]string{"service": "test12"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "3", // Equals + matchType: "3", // OnAverage + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 5.0}, + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat3_onaverage_equals_recovery", + }, + { + description: "Cat3: OnAverage + NotEquals - active alert, avg equals target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 8.0}, + {Value: 10.0}, + {Value: 12.0}, + }, // avg = 10.0 (equals target, not equal to recovery 5.0) + Labels: map[string]string{"service": "test13"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEquals + matchType: "3", // OnAverage + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 10.0}, // avg = 10.0 + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat3_onaverage_noteq_recovery", + }, + { + description: "Cat3: OnAverage + OutsideBounds - active alert, avg |value| in recovery zone → IsRecovering=false", + values: v3.Series{ + Points: []v3.Point{ + {Value: -90.0}, + {Value: 85.0}, + {Value: -80.0}, + }, // avg = -28.33, |avg| = 28.33 + Labels: map[string]string{"service": "test28"}, + }, + expectAlert: false, + expectRecovery: false, // This will not match as 28.33 >= 80.0 is not true + compareOp: "7", // OutsideBounds + matchType: "3", // OnAverage + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + thresholdName: "cat3_onaverage_outsidebounds_no_recovery", + }, + + // ============================================================ + // Category 4: MatchType - InTotal with All CompareOps + // ============================================================ + // Purpose: Verify sum-based matching works with recovery for all operators + // Behavior: Sum of all points is compared against threshold + // ============================================================ + + { + description: "Cat4: InTotal + Above - active alert, sum in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 30.0}, + {Value: 35.0}, + {Value: 25.0}, + }, // sum = 90 + Labels: map[string]string{"service": "test14"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "4", // InTotal + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 90.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat4_intotal_above_recovery", + }, + { + description: "Cat4: InTotal + Below - active alert, sum in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 20.0}, + {Value: 25.0}, + {Value: 15.0}, + }, // sum = 60 + Labels: map[string]string{"service": "test15"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "4", // InTotal + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 60.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "cat4_intotal_below_recovery", + }, + { + description: "Cat4: InTotal + Equals - active alert, sum equals recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 1.0}, + {Value: 2.0}, + {Value: 2.0}, + }, // sum = 5.0 + Labels: map[string]string{"service": "test16"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "3", // Equals + matchType: "4", // InTotal + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 5.0}, + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat4_intotal_equals_recovery", + }, + { + description: "Cat4: InTotal + NotEquals - active alert, sum equals target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 3.0}, + {Value: 3.0}, + {Value: 4.0}, + }, // sum = 10.0 (equals target, not equal to recovery 5.0) + Labels: map[string]string{"service": "test17"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEquals + matchType: "4", // InTotal + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 10.0}, // sum = 10.0 + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat4_intotal_noteq_recovery", + }, + + // ============================================================ + // Category 5: MatchType - Last with All CompareOps + // ============================================================ + // Purpose: Verify last-point matching works with recovery for all operators + // Behavior: Only the last point is compared against threshold + // ============================================================ + + { + description: "Cat5: Last + Above - active alert, last value in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 110.0}, // above target (ignored) + {Value: 75.0}, // below recovery (ignored) + {Value: 85.0}, // last: in recovery zone + }, + Labels: map[string]string{"service": "test18"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "1", // Above + matchType: "5", // Last + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 85.0}, + expectedTarget: 100.0, + expectedRecoveryTarget: 80.0, + thresholdName: "cat5_last_above_recovery", + }, + { + description: "Cat5: Last + Below - active alert, last value in recovery zone → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 40.0}, // below target (ignored) + {Value: 80.0}, // above recovery (ignored) + {Value: 60.0}, // last: in recovery zone + }, + Labels: map[string]string{"service": "test19"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "2", // Below + matchType: "5", // Last + target: 50.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 60.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 70.0, + thresholdName: "cat5_last_below_recovery", + }, + { + description: "Cat5: Last + Equals - active alert, last value equals recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 10.0}, // equals target (ignored) + {Value: 1.0}, // not equal (ignored) + {Value: 5.0}, // last: equals recovery + }, + Labels: map[string]string{"service": "test20"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "3", // Equals + matchType: "5", // Last + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 5.0}, + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat5_last_equals_recovery", + }, + { + description: "Cat5: Last + NotEquals - active alert, last value equals target but not recovery → IsRecovering=true", + values: v3.Series{ + Points: []v3.Point{ + {Value: 5.0}, // equals recovery (ignored) + {Value: 3.0}, // not equal to either (ignored) + {Value: 10.0}, // last: equals target, not equal recovery + }, + Labels: map[string]string{"service": "test21"}, + }, + expectAlert: true, + expectRecovery: true, + compareOp: "4", // NotEquals + matchType: "5", // Last + target: 10.0, + recoveryTarget: func() *float64 { v := 5.0; return &v }(), + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 10.0}, // last = 10.0 + expectedTarget: 10.0, + expectedRecoveryTarget: 5.0, + thresholdName: "cat5_last_noteq_recovery", + }, + + // ============================================================ + // Category 6: Additional Comprehensive Test Cases + // ============================================================ + + // Test no recovery target (backward compatibility) + { + description: "Cat6: No recovery target - Below operator, normal alert behavior", + values: v3.Series{ + Points: []v3.Point{ + {Value: 40.0}, + }, + Labels: map[string]string{"service": "test29"}, + }, + expectAlert: true, + expectRecovery: false, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 50.0, + recoveryTarget: nil, // No recovery target + activeAlerts: nil, + expectedAlertSample: v3.Point{Value: 40.0}, + expectedTarget: 50.0, + expectedRecoveryTarget: 0, + thresholdName: "cat6_no_recovery_target", + }, + } + + // ============================================================ + // Multi-Threshold Test Cases + // ============================================================ + // These test cases validate behavior when multiple thresholds are configured + // Each threshold can be in a different state (firing, recovering, resolved) + + tcThresholdRuleEvalMultiThreshold = []multiThresholdTestCase{ + // ============================================================ + // Test 1: Independent State Tracking - Critical recovering, Warning firing, Info firing + // ============================================================ + // Value: 85.0 + // Critical (target=100, recovery=80): 85 < 100 (not breaching) AND 85 > 80 (in recovery zone) → IsRecovering=true + // Warning (target=90, recovery=70): 85 < 90 (not breaching) AND 85 > 70 (in recovery zone) → IsRecovering=true + // Info (target=80, recovery=60): 85 > 80 (breaching target) → Firing (not recovering) + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Critical recovering, Warning recovering, Info firing - demonstrates independent state tracking", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "payment"}, + }, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + { + name: "info", + target: 80.0, + recoveryTarget: func() *float64 { v := 60.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, // Auto-calculate for all thresholds + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 85 < 100 (not breaching) AND 85 > 80 (in recovery zone) + SampleValue: 85.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 85 < 90 (not breaching) AND 85 > 70 (in recovery zone) + SampleValue: 85.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + "info": { + ShouldReturnSample: true, + IsRecovering: false, // 85 > 80 (still breaching target) → Firing + SampleValue: 85.0, + TargetValue: 80.0, + RecoveryValue: func() *float64 { v := 60.0; return &v }(), + }, + }, + ExpectedSampleCount: 3, // All three thresholds return samples + ExpectedSampleOrder: []string{"critical", "warning", "info"}, // Sorted by target (descending for Above: 100, 90, 80) + }, + + // ============================================================ + // Test 2: Threshold Priority & Sorting - Above operator + // ============================================================ + // Value: 95.0 + // Critical (target=100, recovery=90): 95 < 100 (not breaching) AND 95 > 90 (in recovery zone) → IsRecovering=true + // Warning (target=80, recovery=70): 95 > 80 (breaching target) → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Above - value 95 matches Critical recovery and Warning firing, verify sorting order", + values: v3.Series{ + Points: []v3.Point{{Value: 95.0}}, + Labels: map[string]string{"service": "api"}, + }, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 90.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 80.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 95 < 100 AND 95 > 90 + SampleValue: 95.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 90.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // 95 > 80 (still firing) + SampleValue: 95.0, + TargetValue: 80.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending: 100, 80 + }, + + // ============================================================ + // Test 3: Threshold Priority & Sorting - Below operator + // ============================================================ + // Value: 15.0 + // Critical (target=10, recovery=20): 15 > 10 (not breaching) AND 15 < 20 (in recovery zone) → IsRecovering=true + // Warning (target=30, recovery=40): 15 < 30 (breaching target) → Firing (not recovering) + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Below - value 15 matches both Critical and Warning recovery zones, verify ascending sort", + values: v3.Series{ + Points: []v3.Point{{Value: 15.0}}, + Labels: map[string]string{"service": "database"}, + }, + compareOp: "2", // Below + matchType: "1", // AtleastOnce + target: 10.0, + recoveryTarget: func() *float64 { v := 20.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 30.0, + recoveryTarget: func() *float64 { v := 40.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "2", // CompareOp: ValueIsBelow (2) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 15 > 10 AND 15 < 20 + SampleValue: 15.0, + TargetValue: 10.0, + RecoveryValue: func() *float64 { v := 20.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // 15 < 30 (still breaching target) → Firing + SampleValue: 15.0, + TargetValue: 30.0, + RecoveryValue: func() *float64 { v := 40.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted ascending for Below: 10, 30 + }, + + // ============================================================ + // Test 4: Independent Recovery States - One firing, one recovering, one resolved + // ============================================================ + // Value: 85.0 + // Critical (target=80, recovery=70): 85 > 80 (breaching) → Firing + // Warning (target=90, recovery=80): 85 < 90 (not breaching) AND 85 > 80 (in recovery zone) → IsRecovering=true + // Info (target=100, recovery=90): 85 < 90 (below recovery) → Fully resolved + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Critical firing, Warning recovering, Info resolved - independent state tracking", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "payment"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 80.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "1", // Above + }, + { + name: "info", + target: 100.0, + recoveryTarget: func() *float64 { v := 90.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "1", // Above + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: false, // 85 > 80 (firing) + SampleValue: 85.0, + TargetValue: 80.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 85 < 90 AND 85 > 80 + SampleValue: 85.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "info": { + ShouldReturnSample: false, // 85 < 90 (fully recovered) + IsRecovering: false, + SampleValue: 0, + TargetValue: 0, + RecoveryValue: nil, + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"warning", "critical"}, // Sorted descending: 100, 90, 80 (info not in result, so only warning, critical) + }, + + // ============================================================ + // Test 5: Overlapping Recovery Zones - Nested zones + // ============================================================ + // Value: 85.0 + // Critical (target=100, recovery=80): 85 < 100 AND 85 > 80 → IsRecovering=true + // Warning (target=90, recovery=70): 85 < 90 AND 85 > 70 → IsRecovering=true + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Nested recovery zones - value 85 in both Critical and Warning recovery zones", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "checkout"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, + SampleValue: 85.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, + SampleValue: 85.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending: 100, 90 + }, + + // ============================================================ + // Test 6: Non-overlapping Recovery Zones + // ============================================================ + // Value: 75.0 + // Critical (target=100, recovery=80): 75 < 80 (below recovery) → Fully recovered + // Warning (target=90, recovery=70): 75 < 90 AND 75 > 70 → IsRecovering=true + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Non-overlapping zones - value 75 only in Warning recovery, Critical fully recovered", + values: v3.Series{ + Points: []v3.Point{{Value: 75.0}}, + Labels: map[string]string{"service": "inventory"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: false, // 75 < 80 (fully recovered) + IsRecovering: false, + SampleValue: 0, + TargetValue: 0, + RecoveryValue: nil, + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 75 < 90 AND 75 > 70 + SampleValue: 75.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + }, + ExpectedSampleCount: 1, + ExpectedSampleOrder: []string{"warning"}, // Only warning in result + }, + + // ============================================================ + // Test 7: Different MatchTypes - Once vs Always + // ============================================================ + // Values: [85, 95, 88] + // Critical (Once, target=100, recovery=80): + // - shouldAlert: Check if ANY value > 100 → None match → false + // - matchesRecovery: Check if ANY value > 80 → 85 > 80 → true (returns first match: 85) + // Warning (Always, target=90, recovery=70): + // - shouldAlert: Check if ALL values > 90 → 85 ≤ 90 (fails) → false + // - matchesRecovery: Check if ALL values > 70 → All [85,95,88] > 70 → true (returns min: 85) + // Result: Both thresholds are recovering + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Different MatchTypes - Critical(Once) and Warning(Always) both recovering", + values: v3.Series{ + Points: []v3.Point{ + {Value: 85.0}, + {Value: 95.0}, + {Value: 88.0}, + }, + Labels: map[string]string{"service": "search"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "2", // MatchType: AllTheTimes (2) - all points must match + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // At least one value > 80 (recovery target) + SampleValue: 85.0, // First value that matches: 85 > 80 + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // All values > 70 (recovery target) + SampleValue: 85.0, // Min value for AllTheTimes: min(85, 95, 88) = 85 + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending by target: 100, 90 + }, + + // ============================================================ + // Test 8: Mixed Recovery Config - One has recovery, one doesn't + // ============================================================ + // Value: 85.0 + // Critical (target=100, recovery=80): 85 < 100 AND 85 > 80 → IsRecovering=true + // Warning (target=90, recovery=nil): 85 < 90 → Immediately resolved (no recovery zone) + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Mixed recovery config - Critical has recovery target, Warning doesn't", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "notification"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: nil, // No recovery target + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, + SampleValue: 85.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: false, // No recovery target, immediately resolved + IsRecovering: false, + SampleValue: 0, + TargetValue: 0, + RecoveryValue: nil, + }, + }, + ExpectedSampleCount: 1, + ExpectedSampleOrder: []string{"critical"}, // Only critical + }, + + // ============================================================ + // Test 9: All Thresholds Firing + // ============================================================ + // Value: 150.0 + // Critical (target=100): 150 > 100 → Firing + // Warning (target=80): 150 > 80 → Firing + // Info (target=60): 150 > 60 → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: All firing - value 150 breaches all three thresholds", + values: v3.Series{ + Points: []v3.Point{{Value: 150.0}}, + Labels: map[string]string{"service": "cache"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 90.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 80.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + { + name: "info", + target: 60.0, + recoveryTarget: func() *float64 { v := 50.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: false, // 150 > 100 (firing) + SampleValue: 150.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 90.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // 150 > 80 (firing) + SampleValue: 150.0, + TargetValue: 80.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + "info": { + ShouldReturnSample: true, + IsRecovering: false, // 150 > 60 (firing) + SampleValue: 150.0, + TargetValue: 60.0, + RecoveryValue: func() *float64 { v := 50.0; return &v }(), + }, + }, + ExpectedSampleCount: 3, + ExpectedSampleOrder: []string{"critical", "warning", "info"}, // Sorted descending: 100, 80, 60 + }, + + // ============================================================ + // Test 10: All Thresholds Recovering (with one firing) + // ============================================================ + // Value: 75.0 + // Critical (target=100, recovery=70): 75 < 100 AND 75 > 70 → IsRecovering=true + // Warning (target=80, recovery=65): 75 < 80 AND 75 > 65 → IsRecovering=true + // Info (target=60, recovery=50): 75 > 60 → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Two recovering, one firing - value 75 in recovery zones", + values: v3.Series{ + Points: []v3.Point{{Value: 75.0}}, + Labels: map[string]string{"service": "queue"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) - alerts when value > target + matchType: "1", // MatchType: AtleastOnce (1) - at least one point must match + target: 100.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 80.0, + recoveryTarget: func() *float64 { v := 65.0; return &v }(), + matchType: "1", + compareOp: "1", + }, + { + name: "info", + target: 60.0, + recoveryTarget: func() *float64 { v := 50.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "1", // CompareOp: ValueIsAbove (1) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 75 < 100 AND 75 > 70 + SampleValue: 75.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 75 < 80 AND 75 > 65 + SampleValue: 75.0, + TargetValue: 80.0, + RecoveryValue: func() *float64 { v := 65.0; return &v }(), + }, + "info": { + ShouldReturnSample: true, + IsRecovering: false, // 75 > 60 (firing) + SampleValue: 75.0, + TargetValue: 60.0, + RecoveryValue: func() *float64 { v := 50.0; return &v }(), + }, + }, + ExpectedSampleCount: 3, + ExpectedSampleOrder: []string{"critical", "warning", "info"}, // Sorted descending: 100, 80, 60 + }, + + // ============================================================ + // Test 11: Mixed Operators - Above and Below on same metric + // ============================================================ + // Value: 85.0 + // High CPU (Above, target=90, recovery=80): 85 < 90 AND 85 > 80 → IsRecovering=true + // Low CPU (Below, target=10, recovery=20): 85 > 10 AND 85 > 20 → Fully recovered + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Mixed operators - CPU Above 90 (high) and Below 10 (low) thresholds", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "worker"}, + }, + compareOp: "1", // Above + matchType: "1", // AtleastOnce + target: 90.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "high_cpu", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "low_cpu", + target: 10.0, + recoveryTarget: func() *float64 { v := 20.0; return &v }(), + matchType: "1", // MatchType: AtleastOnce (1) + compareOp: "2", // CompareOp: ValueIsBelow (2) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "high_cpu": { + ShouldReturnSample: true, + IsRecovering: true, // 85 < 90 AND 85 > 80 + SampleValue: 85.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "low_cpu": { + ShouldReturnSample: false, // 85 > 20 (fully recovered) + IsRecovering: false, + SampleValue: 0, + TargetValue: 0, + RecoveryValue: nil, + }, + }, + ExpectedSampleCount: 1, + ExpectedSampleOrder: []string{"high_cpu"}, // Only high_cpu in result + }, + + // ============================================================ + // Test 12: OnAverage MatchType with Above operator + // ============================================================ + // Values: [70, 90, 100] + // Critical (OnAverage, Above, target=100, recovery=80): avg=86.67 < 100 AND avg > 80 → IsRecovering=true + // Warning (AtleastOnce, Above, target=95, recovery=85): 100 > 95 → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: OnAverage vs AtleastOnce - Critical(OnAverage) recovering, Warning(Once) firing", + values: v3.Series{ + Points: []v3.Point{ + {Value: 70.0}, + {Value: 90.0}, + {Value: 100.0}, + }, + Labels: map[string]string{"service": "analytics"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) + matchType: "3", // MatchType: OnAverage (3) - average of all points + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 95.0, + recoveryTarget: func() *float64 { v := 85.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "1", // Above + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // avg(70,90,100)=86.67 < 100 AND > 80 + SampleValue: 86.67, // Average value (rounded) + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // 100 > 95 (firing) + SampleValue: 100.0, + TargetValue: 95.0, + RecoveryValue: func() *float64 { v := 85.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending: 100, 95 + }, + + // ============================================================ + // Test 13: Last MatchType with Below operator + // ============================================================ + // Values: [100, 90, 15] + // Critical (Last, Below, target=10, recovery=20): last=15 > 10 AND 15 < 20 → IsRecovering=true + // Warning (Last, Below, target=30, recovery=40): last=15 < 30 → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Last MatchType - Critical(Last) recovering, Warning(Last) firing", + values: v3.Series{ + Points: []v3.Point{ + {Value: 100.0}, + {Value: 90.0}, + {Value: 15.0}, + }, + Labels: map[string]string{"service": "memory"}, + }, + compareOp: "2", // CompareOp: ValueIsBelow (2) + matchType: "5", // MatchType: Last (5) - only last point matters + target: 10.0, + recoveryTarget: func() *float64 { v := 20.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 30.0, + recoveryTarget: func() *float64 { v := 40.0; return &v }(), + matchType: "5", // Last + compareOp: "2", // Below + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 15 > 10 AND 15 < 20 + SampleValue: 15.0, // Last value + TargetValue: 10.0, + RecoveryValue: func() *float64 { v := 20.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // 15 < 30 (firing) + SampleValue: 15.0, // Last value + TargetValue: 30.0, + RecoveryValue: func() *float64 { v := 40.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted ascending for Below: 10, 30 + }, + + // ============================================================ + // Test 14: Boundary Value Testing - Above vs Below at same value + // ============================================================ + // Value: 90.0 + // Critical (Above, target=90, recovery=80): 90 > 90 → false, check recovery: 90 > 80 → IsRecovering=true + // Warning (Below, target=90, recovery=100): 90 < 90 → false, check recovery: 90 < 100 → IsRecovering=true + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Boundary value - both recovering at exact target value", + values: v3.Series{ + Points: []v3.Point{{Value: 90.0}}, + Labels: map[string]string{"service": "boundary"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) + matchType: "1", // AtleastOnce + target: 90.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 100.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "2", // CompareOp: ValueIsBelow (2) + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // 90 not > 90 (not firing), but 90 > 80 (recovering) + SampleValue: 90.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 90 not < 90 (not firing), but 90 < 100 (recovering) + SampleValue: 90.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 100.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Both have same target + }, + + // ============================================================ + // Test 15: InTotal MatchType with Above operator + // ============================================================ + // Values: [30, 40, 50] + // Critical (InTotal, Above, target=150, recovery=100): sum=120 < 150 AND 120 > 100 → IsRecovering=true + // Warning (InTotal, Above, target=100, recovery=80): sum=120 > 100 → Firing + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: InTotal MatchType - Critical recovering, Warning firing based on sum", + values: v3.Series{ + Points: []v3.Point{ + {Value: 30.0}, + {Value: 40.0}, + {Value: 50.0}, + }, + Labels: map[string]string{"service": "requests"}, + }, + compareOp: "1", // CompareOp: ValueIsAbove (1) + matchType: "4", // MatchType: InTotal (4) - sum of all points + target: 150.0, + recoveryTarget: func() *float64 { v := 100.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 100.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + matchType: "4", // InTotal + compareOp: "1", // Above + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // sum=120 < 150 AND 120 > 100 + SampleValue: 120.0, // Sum of all values + TargetValue: 150.0, + RecoveryValue: func() *float64 { v := 100.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // sum=120 > 100 (firing) + SampleValue: 120.0, // Sum of all values + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending: 150, 100 + }, + + // ============================================================ + // Test 16: Mixed MatchTypes - OnAverage, Last, and AllTheTimes + // ============================================================ + // Values: [60, 80, 100] + // Critical (OnAverage, Above, target=90, recovery=70): avg=80 < 90 AND 80 > 70 → IsRecovering=true + // Warning (Last, Above, target=95, recovery=85): last=100 > 95 → Firing + // Info (AllTheTimes, Above, target=50, recovery=40): all > 50 → Firing (min=60) + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: Mixed MatchTypes - OnAverage recovering, Last firing, AllTheTimes firing", + values: v3.Series{ + Points: []v3.Point{ + {Value: 60.0}, + {Value: 80.0}, + {Value: 100.0}, + }, + Labels: map[string]string{"service": "mixed"}, + }, + compareOp: "1", // Above + matchType: "3", // OnAverage + target: 90.0, + recoveryTarget: func() *float64 { v := 70.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 95.0, + recoveryTarget: func() *float64 { v := 85.0; return &v }(), + matchType: "5", // Last + compareOp: "1", // Above + }, + { + name: "info", + target: 50.0, + recoveryTarget: func() *float64 { v := 40.0; return &v }(), + matchType: "2", // AllTheTimes + compareOp: "1", // Above + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: true, // avg=80 < 90 AND 80 > 70 + SampleValue: 80.0, // Average + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 70.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: false, // last=100 > 95 (firing) + SampleValue: 100.0, // Last value + TargetValue: 95.0, + RecoveryValue: func() *float64 { v := 85.0; return &v }(), + }, + "info": { + ShouldReturnSample: true, + IsRecovering: false, // all > 50 (firing) + SampleValue: 60.0, // Min value for AllTheTimes + TargetValue: 50.0, + RecoveryValue: func() *float64 { v := 40.0; return &v }(), + }, + }, + ExpectedSampleCount: 3, + ExpectedSampleOrder: []string{"warning", "critical", "info"}, // Sorted descending: 95, 90, 50 + }, + + // ============================================================ + // Test 17: ValueIsEq operator with different MatchTypes + // ============================================================ + // Values: [90, 90, 85] + // Critical (AtleastOnce, Eq, target=90, recovery=85): At least one == 90 → Firing + // Warning (AllTheTimes, Eq, target=90, recovery=85): Not all == 90 → Not firing + // - Recovery check: Not all == 85 either → Fully resolved (no sample) + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: ValueIsEq operator - Critical(Once) firing, Warning(Always) resolved", + values: v3.Series{ + Points: []v3.Point{ + {Value: 90.0}, + {Value: 90.0}, + {Value: 85.0}, + }, + Labels: map[string]string{"service": "equality"}, + }, + compareOp: "3", // CompareOp: ValueIsEq (3) + matchType: "1", // AtleastOnce + target: 90.0, + recoveryTarget: func() *float64 { v := 85.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 90.0, + recoveryTarget: func() *float64 { v := 85.0; return &v }(), + matchType: "2", // AllTheTimes + compareOp: "3", // Eq + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: false, // At least one == 90 (firing) + SampleValue: 90.0, + TargetValue: 90.0, + RecoveryValue: func() *float64 { v := 85.0; return &v }(), + }, + "warning": { + ShouldReturnSample: false, // Not all == 90 (not firing), not all == 85 (not recovering) → Resolved + IsRecovering: false, + SampleValue: 0, + TargetValue: 0, + RecoveryValue: nil, + }, + }, + ExpectedSampleCount: 1, + ExpectedSampleOrder: []string{"critical"}, // Only critical firing + }, + + // ============================================================ + // Test 18: ValueIsNotEq operator + // ============================================================ + // Value: 85.0 + // Critical (NotEq, target=100, recovery=90): 85 != 100 → Firing + // Warning (NotEq, target=85, recovery=80): + // - shouldAlert: 85 != 85? No (85 == 85) → Not firing + // - matchesRecovery: 85 != 80? Yes → IsRecovering=true + { + recoveryTestCase: recoveryTestCase{ + description: "MultiThreshold: ValueIsNotEq operator - Critical firing, Warning recovering", + values: v3.Series{ + Points: []v3.Point{{Value: 85.0}}, + Labels: map[string]string{"service": "inequality"}, + }, + compareOp: "4", // CompareOp: ValueIsNotEq (4) + matchType: "1", // AtleastOnce + target: 100.0, + recoveryTarget: func() *float64 { v := 90.0; return &v }(), + thresholdName: "critical", + additionalThresholds: []struct { + name string + target float64 + recoveryTarget *float64 + matchType string + compareOp string + }{ + { + name: "warning", + target: 85.0, + recoveryTarget: func() *float64 { v := 80.0; return &v }(), + matchType: "1", // AtleastOnce + compareOp: "4", // NotEq + }, + }, + activeAlerts: nil, + }, + ExpectedResults: map[string]thresholdExpectation{ + "critical": { + ShouldReturnSample: true, + IsRecovering: false, // 85 != 100 (firing) + SampleValue: 85.0, + TargetValue: 100.0, + RecoveryValue: func() *float64 { v := 90.0; return &v }(), + }, + "warning": { + ShouldReturnSample: true, + IsRecovering: true, // 85 == 85 (not firing), but 85 != 80 (recovering) + SampleValue: 85.0, + TargetValue: 85.0, + RecoveryValue: func() *float64 { v := 80.0; return &v }(), + }, + }, + ExpectedSampleCount: 2, + ExpectedSampleOrder: []string{"critical", "warning"}, // Sorted descending: 100, 85 + }, + } ) diff --git a/pkg/types/ruletypes/alerting.go b/pkg/types/ruletypes/alerting.go index 52f56c4799..49f9bfd5c8 100644 --- a/pkg/types/ruletypes/alerting.go +++ b/pkg/types/ruletypes/alerting.go @@ -59,7 +59,8 @@ type Alert struct { LastSentAt time.Time ValidUntil time.Time - Missing bool + Missing bool + IsRecovering bool } func (a *Alert) NeedsSending(ts time.Time, resendDelay time.Duration) bool { diff --git a/pkg/types/ruletypes/api_params_test.go b/pkg/types/ruletypes/api_params_test.go index ee3b263675..2a5df0fa44 100644 --- a/pkg/types/ruletypes/api_params_test.go +++ b/pkg/types/ruletypes/api_params_test.go @@ -621,10 +621,10 @@ func TestParseIntoRuleThresholdGeneration(t *testing.T) { } // Test that threshold can evaluate properly - vector, err := threshold.ShouldAlert(v3.Series{ + vector, err := threshold.Eval(v3.Series{ Points: []v3.Point{{Value: 0.15, Timestamp: 1000}}, // 150ms in seconds Labels: map[string]string{"test": "label"}, - }, "") + }, "", EvalData{}) if err != nil { t.Fatalf("Unexpected error in shouldAlert: %v", err) } @@ -698,20 +698,20 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) { } // Test with a value that should trigger both WARNING and CRITICAL thresholds - vector, err := threshold.ShouldAlert(v3.Series{ + vector, err := threshold.Eval(v3.Series{ Points: []v3.Point{{Value: 95.0, Timestamp: 1000}}, // 95% CPU usage Labels: map[string]string{"service": "test"}, - }, "") + }, "", EvalData{}) if err != nil { t.Fatalf("Unexpected error in shouldAlert: %v", err) } assert.Equal(t, 2, len(vector)) - vector, err = threshold.ShouldAlert(v3.Series{ + vector, err = threshold.Eval(v3.Series{ Points: []v3.Point{{Value: 75.0, Timestamp: 1000}}, // 75% CPU usage Labels: map[string]string{"service": "test"}, - }, "") + }, "", EvalData{}) if err != nil { t.Fatalf("Unexpected error in shouldAlert: %v", err) } @@ -719,7 +719,7 @@ func TestParseIntoRuleMultipleThresholds(t *testing.T) { assert.Equal(t, 1, len(vector)) } -func TestAnomalyNegationShouldAlert(t *testing.T) { +func TestAnomalyNegationEval(t *testing.T) { tests := []struct { name string ruleJSON []byte @@ -1046,9 +1046,9 @@ func TestAnomalyNegationShouldAlert(t *testing.T) { t.Fatalf("unexpected error from GetRuleThreshold: %v", err) } - resultVector, err := ruleThreshold.ShouldAlert(tt.series, "") + resultVector, err := ruleThreshold.Eval(tt.series, "", EvalData{}) if err != nil { - t.Fatalf("unexpected error from ShouldAlert: %v", err) + t.Fatalf("unexpected error from Eval: %v", err) } shouldAlert := len(resultVector) > 0 diff --git a/pkg/types/ruletypes/result_types.go b/pkg/types/ruletypes/result_types.go index 2460322a6d..8d20c08db2 100644 --- a/pkg/types/ruletypes/result_types.go +++ b/pkg/types/ruletypes/result_types.go @@ -19,7 +19,11 @@ type Sample struct { IsMissing bool - Target float64 + // IsRecovering is true if the sample is part of a recovering alert. + IsRecovering bool + + Target float64 + RecoveryTarget *float64 TargetUnit string } diff --git a/pkg/types/ruletypes/threshold.go b/pkg/types/ruletypes/threshold.go index 598ba9b866..4342ae9644 100644 --- a/pkg/types/ruletypes/threshold.go +++ b/pkg/types/ruletypes/threshold.go @@ -57,8 +57,28 @@ type RuleReceivers struct { Name string `json:"name"` } +// EvalData are other dependent values used to evaluate the threshold rules. +type EvalData struct { + // ActiveAlerts is a map of active alert fingerprints + // used to check if a sample is part of an active alert + // when evaluating the recovery threshold. + ActiveAlerts map[uint64]struct{} +} + +// HasActiveAlert checks if the given sample figerprint is active +// as an alert. +func (eval EvalData) HasActiveAlert(sampleLabelFp uint64) bool { + if len(eval.ActiveAlerts) == 0 { + return false + } + _, ok := eval.ActiveAlerts[sampleLabelFp] + return ok +} + type RuleThreshold interface { - ShouldAlert(series v3.Series, unit string) (Vector, error) + // Eval runs the given series through the threshold rules + // using the given EvalData and returns the matching series + Eval(series v3.Series, unit string, evalData EvalData) (Vector, error) GetRuleReceivers() []RuleReceivers } @@ -97,7 +117,7 @@ func (r BasicRuleThresholds) Validate() error { return errors.Join(errs...) } -func (r BasicRuleThresholds) ShouldAlert(series v3.Series, unit string) (Vector, error) { +func (r BasicRuleThresholds) Eval(series v3.Series, unit string, evalData EvalData) (Vector, error) { var resultVector Vector thresholds := []BasicRuleThreshold(r) sortThresholds(thresholds) @@ -105,8 +125,31 @@ func (r BasicRuleThresholds) ShouldAlert(series v3.Series, unit string) (Vector, smpl, shouldAlert := threshold.shouldAlert(series, unit) if shouldAlert { smpl.Target = *threshold.TargetValue + if threshold.RecoveryTarget != nil { + smpl.RecoveryTarget = threshold.RecoveryTarget + } smpl.TargetUnit = threshold.TargetUnit resultVector = append(resultVector, smpl) + continue + } + + // Prepare alert hash from series labels and threshold name if recovery target option was provided + if threshold.RecoveryTarget == nil { + continue + } + sampleLabels := PrepareSampleLabelsForRule(series.Labels, threshold.Name) + alertHash := sampleLabels.Hash() + // check if alert is active and then check if recovery threshold matches + if evalData.HasActiveAlert(alertHash) { + smpl, matchesRecoveryThrehold := threshold.matchesRecoveryThreshold(series, unit) + if matchesRecoveryThrehold { + smpl.Target = *threshold.TargetValue + smpl.RecoveryTarget = threshold.RecoveryTarget + smpl.TargetUnit = threshold.TargetUnit + // IsRecovering to notify that metrics is in recovery stage + smpl.IsRecovering = true + resultVector = append(resultVector, smpl) + } } } return resultVector, nil @@ -133,16 +176,27 @@ func sortThresholds(thresholds []BasicRuleThreshold) { }) } -func (b BasicRuleThreshold) target(ruleUnit string) float64 { +// convertToRuleUnit converts the given value from the target unit to the rule unit +func (b BasicRuleThreshold) convertToRuleUnit(val float64, ruleUnit string) float64 { unitConverter := converter.FromUnit(converter.Unit(b.TargetUnit)) // convert the target value to the y-axis unit value := unitConverter.Convert(converter.Value{ - F: *b.TargetValue, + F: val, U: converter.Unit(b.TargetUnit), }, converter.Unit(ruleUnit)) return value.F } +// target returns the target value in the rule unit +func (b BasicRuleThreshold) target(ruleUnit string) float64 { + return b.convertToRuleUnit(*b.TargetValue, ruleUnit) +} + +// recoveryTarget returns the recovery target value in the rule unit +func (b BasicRuleThreshold) recoveryTarget(ruleUnit string) float64 { + return b.convertToRuleUnit(*b.RecoveryTarget, ruleUnit) +} + func (b BasicRuleThreshold) getCompareOp() CompareOp { return b.CompareOp } @@ -178,6 +232,13 @@ func (b BasicRuleThreshold) Validate() error { return errors.Join(errs...) } +func (b BasicRuleThreshold) matchesRecoveryThreshold(series v3.Series, ruleUnit string) (Sample, bool) { + return b.shouldAlertWithTarget(series, b.recoveryTarget(ruleUnit)) +} +func (b BasicRuleThreshold) shouldAlert(series v3.Series, ruleUnit string) (Sample, bool) { + return b.shouldAlertWithTarget(series, b.target(ruleUnit)) +} + func removeGroupinSetPoints(series v3.Series) []v3.Point { var result []v3.Point for _, s := range series.Points { @@ -188,21 +249,22 @@ func removeGroupinSetPoints(series v3.Series) []v3.Point { return result } -func (b BasicRuleThreshold) shouldAlert(series v3.Series, ruleUnit string) (Sample, bool) { +// PrepareSampleLabelsForRule prepares the labels for the sample to be used in the alerting. +// It accepts seriesLabels and thresholdName as input and returns the labels with the threshold name label added. +func PrepareSampleLabelsForRule(seriesLabels map[string]string, thresholdName string) (lbls labels.Labels) { + lb := labels.NewBuilder(labels.Labels{}) + for name, value := range seriesLabels { + lb.Set(name, value) + } + lb.Set(LabelThresholdName, thresholdName) + lb.Set(LabelSeverityName, strings.ToLower(thresholdName)) + return lb.Labels() +} + +func (b BasicRuleThreshold) shouldAlertWithTarget(series v3.Series, target float64) (Sample, bool) { var shouldAlert bool var alertSmpl Sample - var lbls labels.Labels - - for name, value := range series.Labels { - lbls = append(lbls, labels.Label{Name: name, Value: value}) - } - - target := b.target(ruleUnit) - - // TODO(srikanthccv): is it better to move the logic to notifier instead of - // adding two labels? - lbls = append(lbls, labels.Label{Name: LabelThresholdName, Value: b.Name}) - lbls = append(lbls, labels.Label{Name: LabelSeverityName, Value: strings.ToLower(b.Name)}) + lbls := PrepareSampleLabelsForRule(series.Labels, b.Name) series.Points = removeGroupinSetPoints(series) diff --git a/pkg/types/ruletypes/threshold_test.go b/pkg/types/ruletypes/threshold_test.go index cfa81eea13..61a4a6d894 100644 --- a/pkg/types/ruletypes/threshold_test.go +++ b/pkg/types/ruletypes/threshold_test.go @@ -9,7 +9,7 @@ import ( v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3" ) -func TestBasicRuleThresholdShouldAlert_UnitConversion(t *testing.T) { +func TestBasicRuleThresholdEval_UnitConversion(t *testing.T) { target := 100.0 tests := []struct { @@ -270,7 +270,7 @@ func TestBasicRuleThresholdShouldAlert_UnitConversion(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { thresholds := BasicRuleThresholds{tt.threshold} - vector, err := thresholds.ShouldAlert(tt.series, tt.ruleUnit) + vector, err := thresholds.Eval(tt.series, tt.ruleUnit, EvalData{}) assert.NoError(t, err) alert := len(vector) > 0 @@ -301,3 +301,31 @@ func TestBasicRuleThresholdShouldAlert_UnitConversion(t *testing.T) { }) } } + +func TestPrepareSampleLabelsForRule(t *testing.T) { + alertAllHashes := make(map[uint64]struct{}) + thresholdName := "test" + for range 50_000 { + sampleLabels := map[string]string{ + "service": "test", + "env": "prod", + "tier": "backend", + "namespace": "default", + "pod": "test-pod", + "container": "test-container", + "node": "test-node", + "cluster": "test-cluster", + "region": "test-region", + "az": "test-az", + "hostname": "test-hostname", + "ip": "192.168.1.1", + "port": "8080", + } + lbls := PrepareSampleLabelsForRule(sampleLabels, thresholdName) + assert.True(t, lbls.Has(LabelThresholdName), "LabelThresholdName not found in labels") + alertAllHashes[lbls.Hash()] = struct{}{} + } + t.Logf("Total hashes: %d", len(alertAllHashes)) + // there should be only one hash for all the samples + assert.Equal(t, 1, len(alertAllHashes), "Expected only one hash for all the samples") +}