Merge branch 'main' into feat/setup_intrinsic_metric_fields

pkg/telemetrymetadata/metadata.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/factory"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/telemetrymetrics"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	"github.com/SigNoz/signoz/pkg/telemetrytraces"
 	"github.com/SigNoz/signoz/pkg/types/metrictypes"
@@ -796,6 +797,53 @@ func applyBackwardCompatibleKeys(mapOfKeys map[string][]*telemetrytypes.Telemetr
 	}
 }
 
+func enrichWithIntrinsicMetricKeys(keys map[string][]*telemetrytypes.TelemetryFieldKey, selectors []*telemetrytypes.FieldKeySelector) map[string][]*telemetrytypes.TelemetryFieldKey {
+	if len(selectors) == 0 {
+		return keys
+	}
+
+	for _, selector := range selectors {
+		if selector.Signal != telemetrytypes.SignalMetrics && selector.Signal != telemetrytypes.SignalUnspecified {
+			continue
+		}
+
+		for name, key := range telemetrymetrics.IntrinsicMetricFieldDefinitions {
+			if !selectorMatchesIntrinsicField(selector, key) {
+				continue
+			}
+			keyCopy := key
+			keys[name] = append(keys[name], &keyCopy)
+		}
+	}
+
+	return keys
+}
+
+func selectorMatchesIntrinsicField(selector *telemetrytypes.FieldKeySelector, definition telemetrytypes.TelemetryFieldKey) bool {
+	if selector.FieldContext != telemetrytypes.FieldContextUnspecified && selector.FieldContext != definition.FieldContext {
+		return false
+	}
+
+	if selector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && selector.FieldDataType != definition.FieldDataType {
+		return false
+	}
+
+	return matchesSelectorName(selector.Name, definition.Name, selector.SelectorMatchType)
+}
+
+func matchesSelectorName(selectorName, target string, matchType telemetrytypes.FieldSelectorMatchType) bool {
+	if selectorName == "" {
+		return true
+	}
+
+	switch matchType {
+	case telemetrytypes.FieldSelectorMatchTypeExact:
+		return strings.EqualFold(selectorName, target)
+	default:
+		return strings.Contains(strings.ToLower(target), strings.ToLower(selectorName))
+	}
+}
+
 func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *telemetrytypes.FieldKeySelector) (map[string][]*telemetrytypes.TelemetryFieldKey, bool, error) {
 	var keys []*telemetrytypes.TelemetryFieldKey
 	var complete bool = true
@@ -851,6 +899,7 @@ func (t *telemetryMetaStore) GetKeys(ctx context.Context, fieldKeySelector *tele
 	}
 
 	applyBackwardCompatibleKeys(mapOfKeys)
+	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, selectors)
 
 	return mapOfKeys, complete, nil
 }
@@ -916,6 +965,7 @@ func (t *telemetryMetaStore) GetKeysMulti(ctx context.Context, fieldKeySelectors
 	}
 
 	applyBackwardCompatibleKeys(mapOfKeys)
+	mapOfKeys = enrichWithIntrinsicMetricKeys(mapOfKeys, fieldKeySelectors)
 
 	return mapOfKeys, complete, nil
 }
@@ -978,7 +1028,7 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
 			FieldMapper:      t.fm,
 			ConditionBuilder: t.conditionBuilder,
 			FieldKeys:        keys,
-        }, 0, 0)
+		}, 0, 0)
 		if err == nil {
 			sb.AddWhereClause(whereClause.WhereClause)
 		} else {
@@ -1002,20 +1052,20 @@ func (t *telemetryMetaStore) getRelatedValues(ctx context.Context, fieldValueSel
 
 			// search on attributes
 			key.FieldContext = telemetrytypes.FieldContextAttribute
-            cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
+			cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
 			if err == nil {
 				conds = append(conds, cond)
 			}
 
 			// search on resource
 			key.FieldContext = telemetrytypes.FieldContextResource
-            cond, err = t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
+			cond, err = t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
 			if err == nil {
 				conds = append(conds, cond)
 			}
 			key.FieldContext = origContext
 		} else {
-            cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
+			cond, err := t.conditionBuilder.ConditionFor(ctx, key, qbtypes.FilterOperatorContains, fieldValueSelector.Value, sb, 0, 0)
 			if err == nil {
 				conds = append(conds, cond)
 			}
@@ -1241,6 +1291,10 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
 
 // getMetricFieldValues returns field values and whether the result is complete
 func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
+	if values, complete, handled, err := t.getIntrinsicMetricFieldValues(ctx, fieldValueSelector); handled {
+		return values, complete, err
+	}
+
 	sb := sqlbuilder.
 		Select("DISTINCT attr_string_value").
 		From(t.metricsDBName + "." + t.metricsFieldsTblName)
@@ -1314,6 +1368,85 @@ func (t *telemetryMetaStore) getMetricFieldValues(ctx context.Context, fieldValu
 	return values, complete, nil
 }
 
+func (t *telemetryMetaStore) getIntrinsicMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, bool, error) {
+	key, ok := telemetrymetrics.IntrinsicMetricFieldDefinitions[fieldValueSelector.Name]
+	if !ok {
+		return nil, false, false, nil
+	}
+
+	if fieldValueSelector.Signal != telemetrytypes.SignalMetrics && fieldValueSelector.Signal != telemetrytypes.SignalUnspecified {
+		return nil, false, false, nil
+	}
+
+	// TODO(nikhilmantri0902, srikanthccv): Please verify if handled to be returned as true n the following case.
+	// if we return handled as true, there is no further query in the getMetricFieldsValue function.
+	if fieldValueSelector.FieldContext != telemetrytypes.FieldContextUnspecified && fieldValueSelector.FieldContext != key.FieldContext {
+		return &telemetrytypes.TelemetryFieldValues{}, true, true, nil
+	}
+	if fieldValueSelector.FieldDataType != telemetrytypes.FieldDataTypeUnspecified && fieldValueSelector.FieldDataType != key.FieldDataType {
+		return &telemetrytypes.TelemetryFieldValues{}, true, true, nil
+	}
+
+	// no values are surfaced for intrinsic boolean fields.
+	if key.FieldDataType == telemetrytypes.FieldDataTypeBool {
+		return &telemetrytypes.TelemetryFieldValues{}, true, true, nil
+	}
+
+	sb := sqlbuilder.Select(fmt.Sprintf("DISTINCT %s", sqlbuilder.Escape(key.Name))).
+		From(t.metricsDBName + "." + telemetrymetrics.TimeseriesV4TableName)
+
+	if fieldValueSelector.MetricContext != nil && fieldValueSelector.MetricContext.MetricName != "" {
+		sb.Where(sb.E("metric_name", fieldValueSelector.MetricContext.MetricName))
+	}
+
+	if fieldValueSelector.StartUnixMilli > 0 {
+		sb.Where(sb.GE("unix_milli", fieldValueSelector.StartUnixMilli))
+	}
+
+	if fieldValueSelector.EndUnixMilli > 0 {
+		sb.Where(sb.LE("unix_milli", fieldValueSelector.EndUnixMilli))
+	}
+
+	if fieldValueSelector.Value != "" {
+		if fieldValueSelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
+			sb.Where(sb.E(key.Name, fieldValueSelector.Value))
+		} else {
+			sb.Where(sb.ILike(key.Name, "%"+escapeForLike(fieldValueSelector.Value)+"%"))
+		}
+	}
+
+	limit := fieldValueSelector.Limit
+	if limit == 0 {
+		limit = 50
+	}
+	sb.Limit(limit + 1)
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	rows, err := t.telemetrystore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, false, true, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error())
+	}
+	defer rows.Close()
+
+	values := &telemetrytypes.TelemetryFieldValues{}
+	rowCount := 0
+	for rows.Next() {
+		rowCount++
+		if rowCount > limit {
+			break
+		}
+
+		var str string
+		if err := rows.Scan(&str); err != nil {
+			return nil, false, true, errors.Wrap(err, errors.TypeInternal, errors.CodeInternal, ErrFailedToGetMetricsKeys.Error())
+		}
+		values.StringValues = append(values.StringValues, str)
+	}
+
+	complete := rowCount <= limit
+	return values, complete, true, nil
+}
+
 func (t *telemetryMetaStore) getMeterSourceMetricFieldValues(ctx context.Context, fieldValueSelector *telemetrytypes.FieldValueSelector) (*telemetrytypes.TelemetryFieldValues, bool, error) {
 	sb := sqlbuilder.Select("DISTINCT arrayJoin(JSONExtractKeysAndValues(labels, 'String')) AS attr").
 		From(t.meterDBName + "." + t.meterFieldsTblName)

pkg/telemetrymetadata/metadata_test.go

@@ -19,6 +19,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func newTestTelemetryMetaStoreTestHelper(store telemetrystore.TelemetryStore) telemetrytypes.MetadataStore {
+	return NewTelemetryMetaStore(
+		instrumentationtest.New().ToProviderSettings(),
+		store,
+		telemetrytraces.DBName,
+		telemetrytraces.TagAttributesV2TableName,
+		telemetrytraces.SpanAttributesKeysTblName,
+		telemetrytraces.SpanIndexV3TableName,
+		telemetrymetrics.DBName,
+		telemetrymetrics.AttributesMetadataTableName,
+		telemetrymeter.DBName,
+		telemetrymeter.SamplesAgg1dTableName,
+		telemetrylogs.DBName,
+		telemetrylogs.LogsV2TableName,
+		telemetrylogs.TagAttributesV2TableName,
+		telemetrylogs.LogAttributeKeysTblName,
+		telemetrylogs.LogResourceKeysTblName,
+		DBName,
+		AttributesMetadataLocalTableName,
+	)
+}
+
 type regexMatcher struct {
 }
 
@@ -37,25 +59,7 @@ func TestGetKeys(t *testing.T) {
 	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
 	mock := mockTelemetryStore.Mock()
 
-	metadata := NewTelemetryMetaStore(
-		instrumentationtest.New().ToProviderSettings(),
-		mockTelemetryStore,
-		telemetrytraces.DBName,
-		telemetrytraces.TagAttributesV2TableName,
-		telemetrytraces.SpanAttributesKeysTblName,
-		telemetrytraces.SpanIndexV3TableName,
-		telemetrymetrics.DBName,
-		telemetrymetrics.AttributesMetadataTableName,
-		telemetrymeter.DBName,
-		telemetrymeter.SamplesAgg1dTableName,
-		telemetrylogs.DBName,
-		telemetrylogs.LogsV2TableName,
-		telemetrylogs.TagAttributesV2TableName,
-		telemetrylogs.LogAttributeKeysTblName,
-		telemetrylogs.LogResourceKeysTblName,
-		DBName,
-		AttributesMetadataLocalTableName,
-	)
+	metadata := newTestTelemetryMetaStoreTestHelper(mockTelemetryStore)
 
 	rows := cmock.NewRows([]cmock.ColumnType{
 		{Name: "statement", Type: "String"},
@@ -165,25 +169,7 @@ func TestApplyBackwardCompatibleKeys(t *testing.T) {
 			mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
 			mock := mockTelemetryStore.Mock()
 
-			metadata := NewTelemetryMetaStore(
-				instrumentationtest.New().ToProviderSettings(),
-				mockTelemetryStore,
-				telemetrytraces.DBName,
-				telemetrytraces.TagAttributesV2TableName,
-				telemetrytraces.SpanAttributesKeysTblName,
-				telemetrytraces.SpanIndexV3TableName,
-				telemetrymetrics.DBName,
-				telemetrymetrics.AttributesMetadataTableName,
-				telemetrymeter.DBName,
-				telemetrymeter.SamplesAgg1dTableName,
-				telemetrylogs.DBName,
-				telemetrylogs.LogsV2TableName,
-				telemetrylogs.TagAttributesV2TableName,
-				telemetrylogs.LogAttributeKeysTblName,
-				telemetrylogs.LogResourceKeysTblName,
-				DBName,
-				AttributesMetadataLocalTableName,
-			)
+			metadata := newTestTelemetryMetaStoreTestHelper(mockTelemetryStore)
 
 			hasTraces := false
 			hasLogs := false
@@ -298,3 +284,85 @@ func TestApplyBackwardCompatibleKeys(t *testing.T) {
 		})
 	}
 }
+
+func TestEnrichWithIntrinsicMetricKeys(t *testing.T) {
+	result := enrichWithIntrinsicMetricKeys(
+		map[string][]*telemetrytypes.TelemetryFieldKey{},
+		[]*telemetrytypes.FieldKeySelector{
+			{
+				Signal:            telemetrytypes.SignalMetrics,
+				Name:              "metric",
+				SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+			},
+		},
+	)
+
+	require.Contains(t, result, "metric_name")
+	assert.Equal(t, telemetrytypes.FieldContextMetric, result["metric_name"][0].FieldContext)
+
+	result = enrichWithIntrinsicMetricKeys(
+		map[string][]*telemetrytypes.TelemetryFieldKey{},
+		[]*telemetrytypes.FieldKeySelector{
+			{
+				Signal:            telemetrytypes.SignalMetrics,
+				Name:              "metric",
+				FieldContext:      telemetrytypes.FieldContextAttribute,
+				SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+			},
+		},
+	)
+	assert.NotContains(t, result, "metric_name")
+}
+
+func TestGetMetricFieldValuesIntrinsicMetricName(t *testing.T) {
+	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
+	mock := mockTelemetryStore.Mock()
+
+	metadata := newTestTelemetryMetaStoreTestHelper(mockTelemetryStore)
+
+	valueRows := cmock.NewRows([]cmock.ColumnType{
+		{Name: "metric_name", Type: "String"},
+	}, [][]any{{"metric.a"}, {"metric.b"}})
+
+	query := `SELECT DISTINCT .*metric_name.*` + telemetrymetrics.TimeseriesV4TableName
+
+	mock.ExpectQuery(query).
+		WithArgs(51).
+		WillReturnRows(valueRows)
+
+	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
+		FieldKeySelector: &telemetrytypes.FieldKeySelector{
+			Signal:            telemetrytypes.SignalMetrics,
+			Name:              "metric_name",
+			Limit:             50,
+			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+		},
+		Limit: 50,
+	})
+	require.NoError(t, err)
+	assert.True(t, complete)
+	assert.ElementsMatch(t, []string{"metric.a", "metric.b"}, values.StringValues)
+	require.NoError(t, mock.ExpectationsWereMet())
+}
+
+func TestGetMetricFieldValuesIntrinsicBoolReturnsEmpty(t *testing.T) {
+	mockTelemetryStore := telemetrystoretest.New(telemetrystore.Config{}, &regexMatcher{})
+
+	metadata := newTestTelemetryMetaStoreTestHelper(mockTelemetryStore)
+
+	values, complete, err := metadata.(*telemetryMetaStore).getMetricFieldValues(context.Background(), &telemetrytypes.FieldValueSelector{
+		FieldKeySelector: &telemetrytypes.FieldKeySelector{
+			Signal:            telemetrytypes.SignalMetrics,
+			Name:              "is_monotonic",
+			FieldDataType:     telemetrytypes.FieldDataTypeBool,
+			Limit:             10,
+			SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+		},
+		Value: "true",
+		Limit: 10,
+	})
+	require.NoError(t, err)
+	assert.True(t, complete)
+	assert.Empty(t, values.StringValues)
+	assert.Empty(t, values.BoolValues)
+}

pkg/telemetrymetrics/const.go

@@ -1,5 +1,7 @@
 package telemetrymetrics
 
+import "github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+
 var IntrinsicFields = []string{
 	"__normalized",
 	"temporality",
@@ -7,3 +9,30 @@ var IntrinsicFields = []string{
 	"type",
 	"is_monotonic",
 }
+
+var IntrinsicMetricFieldDefinitions = map[string]telemetrytypes.TelemetryFieldKey{
+	"metric_name": {
+		Name:          "metric_name",
+		Signal:        telemetrytypes.SignalMetrics,
+		FieldContext:  telemetrytypes.FieldContextMetric,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+	},
+	"type": {
+		Name:          "type",
+		Signal:        telemetrytypes.SignalMetrics,
+		FieldContext:  telemetrytypes.FieldContextMetric,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+	},
+	"temporality": {
+		Name:          "temporality",
+		Signal:        telemetrytypes.SignalMetrics,
+		FieldContext:  telemetrytypes.FieldContextMetric,
+		FieldDataType: telemetrytypes.FieldDataTypeString,
+	},
+	"is_monotonic": {
+		Name:          "is_monotonic",
+		Signal:        telemetrytypes.SignalMetrics,
+		FieldContext:  telemetrytypes.FieldContextMetric,
+		FieldDataType: telemetrytypes.FieldDataTypeBool,
+	},
+}