Merge branch 'json-plan' into json

* chore: minor comment change

* chore: func rename, file rename

* chore: change table names

* feat: list values for simple paths

* chore: in progress

* feat: in progress TestStatementBuilderListQueryBody working

* feat: in progress TestStatementBuilderListQueryBodyPromoted

* feat: in progress TestStatementBuilderListQueryBodyMessage

* feat: in progress group by

* feat: all query tests passed new

.devenv/docker/clickhouse/compose.yaml

@@ -1,6 +1,6 @@
 services:
   clickhouse:
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.10.1
     container_name: clickhouse
     volumes:
       - ${PWD}/fs/etc/clickhouse-server/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml

.gitignore

@@ -49,6 +49,7 @@ ee/query-service/tests/test-deploy/data/
 # local data
 *.backup
 *.db
+**/db
 /deploy/docker/clickhouse-setup/data/
 /deploy/docker-swarm/clickhouse-setup/data/
 bin/

Makefile

@@ -72,6 +72,12 @@ devenv-up: devenv-clickhouse devenv-signoz-otel-collector ## Start both clickhou
 	@echo "   - ClickHouse: http://localhost:8123"
 	@echo "   - Signoz OTel Collector: grpc://localhost:4317, http://localhost:4318"
 
+.PHONY: devenv-clickhouse-clean
+devenv-clickhouse-clean: ## Clean all ClickHouse data from filesystem
+	@echo "Removing ClickHouse data..."
+	@rm -rf .devenv/docker/clickhouse/fs/tmp/*
+	@echo "ClickHouse data cleaned!"
+
 ##############################################################
 # go commands
 ##############################################################

deploy/docker-swarm/docker-compose.ha.yaml

@@ -11,7 +11,7 @@ x-common: &common
       max-file: "3"
 x-clickhouse-defaults: &clickhouse-defaults
   !!merge <<: *common
-  image: clickhouse/clickhouse-server:25.5.6
+  image: clickhouse/clickhouse-server:25.10.1
   tty: true
   deploy:
     labels:
@@ -65,7 +65,7 @@ x-db-depend: &db-depend
 services:
   init-clickhouse:
     !!merge <<: *common
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.10.1
     command:
       - bash
       - -c

deploy/docker-swarm/docker-compose.yaml

@@ -11,7 +11,7 @@ x-common: &common
       max-file: "3"
 x-clickhouse-defaults: &clickhouse-defaults
   !!merge <<: *common
-  image: clickhouse/clickhouse-server:25.5.6
+  image: clickhouse/clickhouse-server:25.10.1
   tty: true
   deploy:
     labels:
@@ -62,7 +62,7 @@ x-db-depend: &db-depend
 services:
   init-clickhouse:
     !!merge <<: *common
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.10.1
     command:
       - bash
       - -c

deploy/docker/docker-compose.ha.yaml

@@ -10,7 +10,7 @@ x-common: &common
 x-clickhouse-defaults: &clickhouse-defaults
   !!merge <<: *common
   # addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab
-  image: clickhouse/clickhouse-server:25.5.6
+  image: clickhouse/clickhouse-server:25.10.1
   tty: true
   labels:
     signoz.io/scrape: "true"
@@ -67,7 +67,7 @@ x-db-depend: &db-depend
 services:
   init-clickhouse:
     !!merge <<: *common
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.10.1
     container_name: signoz-init-clickhouse
     command:
       - bash

deploy/docker/docker-compose.yaml

@@ -9,7 +9,7 @@ x-common: &common
       max-file: "3"
 x-clickhouse-defaults: &clickhouse-defaults
   !!merge <<: *common
-  image: clickhouse/clickhouse-server:25.5.6
+  image: clickhouse/clickhouse-server:25.10.1
   tty: true
   labels:
     signoz.io/scrape: "true"
@@ -62,7 +62,7 @@ x-db-depend: &db-depend
 services:
   init-clickhouse:
     !!merge <<: *common
-    image: clickhouse/clickhouse-server:25.5.6
+    image: clickhouse/clickhouse-server:25.10.1
     container_name: signoz-init-clickhouse
     command:
       - bash

ee/query-service/constants/constants.go

@@ -9,6 +9,7 @@ var LicenseAPIKey = GetOrDefaultEnv("SIGNOZ_LICENSE_API_KEY", "")
 var SaasSegmentKey = GetOrDefaultEnv("SIGNOZ_SAAS_SEGMENT_KEY", "")
 var FetchFeatures = GetOrDefaultEnv("FETCH_FEATURES", "false")
 var ZeusFeaturesURL = GetOrDefaultEnv("ZEUS_FEATURES_URL", "ZeusFeaturesURL")
+var BodyJSONQueryEnabled = GetOrDefaultEnv("BODY_JSON_QUERY_ENABLED", "false") == "true"
 
 func GetOrDefaultEnv(key string, fallback string) string {
 	v := os.Getenv(key)

go.mod

@@ -8,9 +8,10 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.40.1
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.129.4
+	github.com/SigNoz/signoz-otel-collector v0.129.10-rc.7
 	github.com/antlr4-go/antlr/v4 v4.13.1
 	github.com/antonmedv/expr v1.15.3
+	github.com/bytedance/sonic v1.14.1
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/coreos/go-oidc/v3 v3.14.1
 	github.com/dgraph-io/ristretto/v2 v2.3.0
@@ -86,12 +87,18 @@ require (
 )
 
 require (
+	github.com/bytedance/gopkg v0.1.3 // indirect
+	github.com/bytedance/sonic/loader v0.3.0 // indirect
+	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/redis/go-redis/extra/rediscmd/v9 v9.15.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect
+	go.opentelemetry.io/collector/config/configretry v1.34.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
 	modernc.org/libc v1.66.10 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.11.0 // indirect

go.sum

@@ -106,8 +106,8 @@ github.com/SigNoz/expr v1.17.7-beta h1:FyZkleM5dTQ0O6muQfwGpoH5A2ohmN/XTasRCO72g
 github.com/SigNoz/expr v1.17.7-beta/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
-github.com/SigNoz/signoz-otel-collector v0.129.4 h1:DGDu9y1I1FU+HX4eECPGmfhnXE4ys4yr7LL6znbf6to=
-github.com/SigNoz/signoz-otel-collector v0.129.4/go.mod h1:xyR+coBzzO04p6Eu+ql2RVYUl/jFD+8hD9lArcc9U7g=
+github.com/SigNoz/signoz-otel-collector v0.129.10-rc.7 h1:r8/+t3ARWek9+X5aH05qavdA9ATbkssfssHh/zjzsEM=
+github.com/SigNoz/signoz-otel-collector v0.129.10-rc.7/go.mod h1:4eJCRUd/P4OiCHXvGYZK8q6oyBVGJFVj/G6qKSoN/TQ=
 github.com/Yiling-J/theine-go v0.6.2 h1:1GeoXeQ0O0AUkiwj2S9Jc0Mzx+hpqzmqsJ4kIC4M9AY=
 github.com/Yiling-J/theine-go v0.6.2/go.mod h1:08QpMa5JZ2pKN+UJCRrCasWYO1IKCdl54Xa836rpmDU=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
@@ -162,6 +162,12 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
+github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
+github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
+github.com/bytedance/sonic v1.14.1 h1:FBMC0zVz5XUmE4z9wF4Jey0An5FueFvOsTKKKtwIl7w=
+github.com/bytedance/sonic v1.14.1/go.mod h1:gi6uhQLMbTdeP0muCnrjHLeCUPyb70ujhnNlhOylAFc=
+github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
+github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
 github.com/cactus/go-statsd-client/statsd v0.0.0-20200423205355-cb0885a1018c/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@@ -178,6 +184,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
 github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
+github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -991,6 +999,8 @@ github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GH
 github.com/trivago/tgo v1.0.7 h1:uaWH/XIy9aWYWpjm2CU3RpcqZXmX2ysQ9/Go+d9gyrM=
 github.com/trivago/tgo v1.0.7/go.mod h1:w4dpD+3tzNIIiIfkWWa85w5/B77tlvdZckQ+6PkFnhc=
 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/uptrace/bun v1.2.9 h1:OOt2DlIcRUMSZPr6iXDFg/LaQd59kOxbAjpIVHddKRs=
 github.com/uptrace/bun v1.2.9/go.mod h1:r2ZaaGs9Ru5bpGTr8GQfp8jp+TlCav9grYCPOu2CJSg=
 github.com/uptrace/bun/dialect/pgdialect v1.2.9 h1:caf5uFbOGiXvadV6pA5gn87k0awFFxL1kuuY3SpxnWk=
@@ -1235,6 +1245,8 @@ go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
 go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670 h1:18EFjUmQOcUvxNYSkA6jO9VAiXCnxFY6NyDX0bHDmkU=
+golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

pkg/errors/errors.go

@@ -208,3 +208,8 @@ func WrapUnexpectedf(cause error, code Code, format string, args ...any) *base {
 func NewUnexpectedf(code Code, format string, args ...any) *base {
 	return Newf(TypeInvalidInput, code, format, args...)
 }
+
+// NewMethodNotAllowedf is a wrapper around Newf with TypeMethodNotAllowed.
+func NewMethodNotAllowedf(code Code, format string, args ...any) *base {
+	return Newf(TypeMethodNotAllowed, code, format, args...)
+}

pkg/errors/http.go

@@ -18,13 +18,18 @@ type responseerroradditional struct {
 
 func AsJSON(cause error) *JSON {
 	// See if this is an instance of the base error or not
-	_, c, m, _, u, a := Unwrapb(cause)
+	_, c, m, cause, u, a := Unwrapb(cause)
 
 	rea := make([]responseerroradditional, len(a))
 	for k, v := range a {
 		rea[k] = responseerroradditional{v}
 	}
 
+	// add the underlying error message
+	if cause != nil {
+		rea = append(rea, responseerroradditional{Message: cause.Error()})
+	}
+
 	return &JSON{
 		Code:    c.String(),
 		Message: m,

pkg/modules/promote/implpromote/handler.go

@@ -0,0 +1,142 @@
+package implpromote
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/http/render"
+	"github.com/SigNoz/signoz/pkg/modules/promote"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/types/authtypes"
+	"github.com/SigNoz/signoz/pkg/types/promotetypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+type handler struct {
+	module promote.Module
+}
+
+func NewHandler(module promote.Module) promote.Handler {
+	return &handler{module: module}
+}
+
+func (h *handler) HandlePromote(w http.ResponseWriter, r *http.Request) {
+	_, err := authtypes.ClaimsFromContext(r.Context())
+	if err != nil {
+		render.Error(w, errors.NewInternalf(errors.CodeInternal, "failed to get org id from context"))
+		return
+	}
+
+	switch r.Method {
+	case http.MethodGet:
+		h.GetPromotedAndIndexedPaths(w, r)
+		return
+	case http.MethodPost:
+		h.PromotePaths(w, r)
+		return
+	case http.MethodDelete:
+		h.DropIndex(w, r)
+		return
+	default:
+		render.Error(w, errors.NewMethodNotAllowedf(errors.CodeMethodNotAllowed, "method not allowed"))
+		return
+	}
+}
+
+func (h *handler) DropIndex(w http.ResponseWriter, r *http.Request) {
+	var req promotetypes.PromotePath
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		render.Error(w, errors.NewInvalidInputf(errors.CodeInvalidInput, "Invalid data"))
+		return
+	}
+
+	err := h.module.DropIndex(r.Context(), req)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	render.Success(w, http.StatusOK, nil)
+}
+
+func (h *handler) PromotePaths(w http.ResponseWriter, r *http.Request) {
+	var req []promotetypes.PromotePath
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		render.Error(w, errors.NewInvalidInputf(errors.CodeInvalidInput, "Invalid data"))
+		return
+	}
+
+	// Delegate all processing to the reader
+	err := h.module.PromoteAndIndexPaths(r.Context(), req...)
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	render.Success(w, http.StatusOK, nil)
+}
+
+func (h *handler) GetPromotedAndIndexedPaths(w http.ResponseWriter, r *http.Request) {
+	response, err := func() ([]promotetypes.PromotePath, error) {
+		indexes, err := h.module.ListBodySkipIndexes(r.Context())
+		if err != nil {
+			return nil, err
+		}
+		aggr := map[string][]promotetypes.WrappedIndex{}
+		for _, index := range indexes {
+			path, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
+			if err != nil {
+				return nil, err
+			}
+
+			// clean backticks from the path
+			path = strings.ReplaceAll(path, "`", "")
+
+			aggr[path] = append(aggr[path], promotetypes.WrappedIndex{
+				ColumnType:  columnType,
+				Type:        index.Type,
+				Granularity: index.Granularity,
+			})
+		}
+		promotedPaths, err := h.module.ListPromotedPaths(r.Context())
+		if err != nil {
+			return nil, err
+		}
+
+		response := []promotetypes.PromotePath{}
+		for _, path := range promotedPaths {
+			fullPath := telemetrylogs.BodyPromotedColumnPrefix + path
+			path = telemetrytypes.BodyJSONStringSearchPrefix + path
+			item := promotetypes.PromotePath{
+				Path:    path,
+				Promote: true,
+			}
+			indexes, ok := aggr[fullPath]
+			if ok {
+				item.Indexes = indexes
+				delete(aggr, fullPath)
+			}
+			response = append(response, item)
+		}
+
+		// add the paths that are not promoted but have indexes
+		for path, indexes := range aggr {
+			path := strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
+			path = telemetrytypes.BodyJSONStringSearchPrefix + path
+			response = append(response, promotetypes.PromotePath{
+				Path:    path,
+				Indexes: indexes,
+			})
+		}
+		return response, nil
+	}()
+	if err != nil {
+		render.Error(w, err)
+		return
+	}
+
+	render.Success(w, http.StatusOK, response)
+}

pkg/modules/promote/implpromote/module.go

@@ -0,0 +1,238 @@
+package implpromote
+
+import (
+	"context"
+	"fmt"
+	"maps"
+	"slices"
+	"strings"
+	"time"
+
+	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/modules/promote"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/telemetrymetadata"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/types/promotetypes"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	CodeFailedToPrepareBatch       = errors.MustNewCode("failed_to_prepare_batch_promoted_paths")
+	CodeFailedToSendBatch          = errors.MustNewCode("failed_to_send_batch_promoted_paths")
+	CodeFailedToAppendPath         = errors.MustNewCode("failed_to_append_path_promoted_paths")
+	CodeFailedToCreateIndex        = errors.MustNewCode("failed_to_create_index_promoted_paths")
+	CodeFailedToDropIndex          = errors.MustNewCode("failed_to_drop_index_promoted_paths")
+	CodeFailedToQueryPromotedPaths = errors.MustNewCode("failed_to_query_promoted_paths")
+)
+
+type module struct {
+	store telemetrystore.TelemetryStore
+}
+
+func NewModule(store telemetrystore.TelemetryStore) promote.Module {
+	return &module{store: store}
+}
+
+func (m *module) ListBodySkipIndexes(ctx context.Context) ([]schemamigrator.Index, error) {
+	return telemetrymetadata.ListLogsJSONIndexes(ctx, m.store.Cluster(), m.store.ClickhouseDB())
+}
+
+func (m *module) ListPromotedPaths(ctx context.Context) ([]string, error) {
+	paths, err := telemetrymetadata.ListPromotedPaths(ctx, m.store.ClickhouseDB())
+	if err != nil {
+		return nil, err
+	}
+	return slices.Collect(maps.Keys(paths)), nil
+}
+
+// PromotePaths inserts provided JSON paths into the promoted paths table for logs queries.
+func (m *module) PromotePaths(ctx context.Context, paths []string) error {
+	if len(paths) == 0 {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
+	}
+
+	batch, err := m.store.ClickhouseDB().PrepareBatch(ctx,
+		fmt.Sprintf("INSERT INTO %s.%s (path, created_at) VALUES", telemetrymetadata.DBName,
+			telemetrymetadata.PromotedPathsTableName))
+	if err != nil {
+		return errors.WrapInternalf(err, CodeFailedToPrepareBatch, "failed to prepare batch")
+	}
+
+	nowMs := uint64(time.Now().UnixMilli())
+	for _, p := range paths {
+		trimmed := strings.TrimSpace(p)
+		if trimmed == "" {
+			continue
+		}
+		if err := batch.Append(trimmed, nowMs); err != nil {
+			_ = batch.Abort()
+			return errors.WrapInternalf(err, CodeFailedToAppendPath, "failed to append path")
+		}
+	}
+
+	if err := batch.Send(); err != nil {
+		return errors.WrapInternalf(err, CodeFailedToSendBatch, "failed to send batch")
+	}
+	return nil
+}
+
+// createIndexes creates string ngram + token filter indexes on JSON path subcolumns for LIKE queries.
+func (m *module) createIndexes(ctx context.Context, indexes []schemamigrator.Index) error {
+	if len(indexes) == 0 {
+		return nil
+	}
+
+	for _, index := range indexes {
+		alterStmt := schemamigrator.AlterTableAddIndex{
+			Database: telemetrylogs.DBName,
+			Table:    telemetrylogs.LogsV2LocalTableName,
+			Index:    index,
+		}
+		op := alterStmt.OnCluster(m.store.Cluster())
+		if err := m.store.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
+			return errors.WrapInternalf(err, CodeFailedToCreateIndex, "failed to create index")
+		}
+	}
+
+	return nil
+}
+
+func (m *module) DropIndex(ctx context.Context, path promotetypes.PromotePath) error {
+	// validate the paths
+	if err := path.Validate(); err != nil {
+		return err
+	}
+
+	promoted, err := telemetrymetadata.IsPathPromoted(ctx, m.store.ClickhouseDB(), path.Path)
+	if err != nil {
+		return err
+	}
+	parentColumn := telemetrylogs.LogsV2BodyJSONColumn
+	if promoted {
+		parentColumn = telemetrylogs.LogsV2BodyPromotedColumn
+	}
+
+	for _, index := range path.Indexes {
+		typeIndex := schemamigrator.IndexTypeTokenBF
+		switch {
+		case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeNGramBF)):
+			typeIndex = schemamigrator.IndexTypeNGramBF
+		case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeTokenBF)):
+			typeIndex = schemamigrator.IndexTypeTokenBF
+		case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeMinMax)):
+			typeIndex = schemamigrator.IndexTypeMinMax
+		default:
+			return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid index type: %s", index.Type)
+		}
+
+		alterStmt := schemamigrator.AlterTableDropIndex{
+			Database: telemetrylogs.DBName,
+			Table:    telemetrylogs.LogsV2LocalTableName,
+			Index: schemamigrator.Index{
+				Name:        schemamigrator.JSONSubColumnIndexName(parentColumn, path.Path, index.JSONDataType.StringValue(), typeIndex),
+				Expression:  schemamigrator.JSONSubColumnIndexExpr(parentColumn, path.Path, index.JSONDataType.StringValue()),
+				Type:        index.Type,
+				Granularity: index.Granularity,
+			},
+		}
+		op := alterStmt.OnCluster(m.store.Cluster())
+		if err := m.store.ClickhouseDB().Exec(ctx, op.ToSQL()); err != nil {
+			return errors.WrapInternalf(err, CodeFailedToDropIndex, "failed to drop index")
+		}
+	}
+
+	return nil
+}
+
+// PromoteAndIndexPaths handles promoting paths and creating indexes in one call.
+func (m *module) PromoteAndIndexPaths(
+	ctx context.Context,
+	paths ...promotetypes.PromotePath,
+) error {
+	if len(paths) == 0 {
+		return errors.NewInvalidInputf(errors.CodeInvalidInput, "paths cannot be empty")
+	}
+
+	// validate the paths
+	for _, path := range paths {
+		if err := path.Validate(); err != nil {
+			return err
+		}
+	}
+
+	sb := sqlbuilder.NewSelectBuilder().From(fmt.Sprintf("%s.%s", telemetrymetadata.DBName, telemetrymetadata.PromotedPathsTableName)).Select("path")
+	cond := []string{}
+	for _, path := range paths {
+		cond = append(cond, sb.Equal("path", path.Path))
+	}
+	sb.Where(sb.Or(cond...))
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	rows, err := m.store.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return errors.WrapInternalf(err, CodeFailedToQueryPromotedPaths, "failed to query promoted paths")
+	}
+	defer rows.Close()
+
+	// Load existing promoted paths once
+	existingPromotedPaths := make(map[string]struct{})
+	for rows.Next() {
+		var p string
+		if err := rows.Scan(&p); err == nil {
+			existingPromotedPaths[p] = struct{}{}
+		}
+	}
+
+	var toInsert []string
+	indexes := []schemamigrator.Index{}
+	for _, it := range paths {
+		if it.Promote {
+			if _, promoted := existingPromotedPaths[it.Path]; !promoted {
+				toInsert = append(toInsert, it.Path)
+			}
+		}
+		if len(it.Indexes) > 0 {
+			parentColumn := telemetrylogs.LogsV2BodyJSONColumn
+			// if the path is already promoted or is being promoted, add it to the promoted column
+			if _, promoted := existingPromotedPaths[it.Path]; promoted || it.Promote {
+				parentColumn = telemetrylogs.LogsV2BodyPromotedColumn
+			}
+
+			for _, index := range it.Indexes {
+				typeIndex := schemamigrator.IndexTypeTokenBF
+				switch {
+				case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeNGramBF)):
+					typeIndex = schemamigrator.IndexTypeNGramBF
+				case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeTokenBF)):
+					typeIndex = schemamigrator.IndexTypeTokenBF
+				case strings.HasPrefix(index.Type, string(schemamigrator.IndexTypeMinMax)):
+					typeIndex = schemamigrator.IndexTypeMinMax
+				default:
+					return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid index type: %s", index.Type)
+				}
+				indexes = append(indexes, schemamigrator.Index{
+					Name:        schemamigrator.JSONSubColumnIndexName(parentColumn, it.Path, index.JSONDataType.StringValue(), typeIndex),
+					Expression:  schemamigrator.JSONSubColumnIndexExpr(parentColumn, it.Path, index.JSONDataType.StringValue()),
+					Type:        index.Type,
+					Granularity: index.Granularity,
+				})
+			}
+		}
+	}
+
+	if len(toInsert) > 0 {
+		err := m.PromotePaths(ctx, toInsert)
+		if err != nil {
+			return err
+		}
+	}
+
+	if len(indexes) > 0 {
+		if err := m.createIndexes(ctx, indexes); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/modules/promote/promote.go

@@ -0,0 +1,20 @@
+package promote
+
+import (
+	"context"
+	"net/http"
+
+	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/pkg/types/promotetypes"
+)
+
+type Module interface {
+	ListBodySkipIndexes(ctx context.Context) ([]schemamigrator.Index, error)
+	ListPromotedPaths(ctx context.Context) ([]string, error)
+	PromoteAndIndexPaths(ctx context.Context, paths ...promotetypes.PromotePath) error
+	DropIndex(ctx context.Context, path promotetypes.PromotePath) error
+}
+
+type Handler interface {
+	HandlePromote(w http.ResponseWriter, r *http.Request)
+}

pkg/querier/builder_query.go

@@ -10,9 +10,11 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/bytedance/sonic"
 )
 
 type builderQuery[T any] struct {
@@ -248,6 +250,40 @@ func (q *builderQuery[T]) executeWithContext(ctx context.Context, query string,
 		return nil, err
 	}
 
+	// merge body_json and promoted into body
+	if q.spec.Signal == telemetrytypes.SignalLogs {
+		switch typedPayload := payload.(type) {
+		case *qbtypes.RawData:
+			for _, rr := range typedPayload.Rows {
+				seeder := func() error {
+					body, ok := rr.Data[telemetrylogs.LogsV2BodyJSONColumn].(map[string]any)
+					if !ok {
+						return nil
+					}
+					promoted, ok := rr.Data[telemetrylogs.LogsV2BodyPromotedColumn].(map[string]any)
+					if !ok {
+						return nil
+					}
+					seed(promoted, body)
+					str, err := sonic.MarshalString(body)
+					if err != nil {
+						return errors.Wrapf(err, errors.TypeInternal, errors.CodeInternal, "failed to marshal body")
+					}
+					rr.Data["body"] = str
+					return nil
+				}
+				err := seeder()
+				if err != nil {
+					return nil, err
+				}
+
+				delete(rr.Data, telemetrylogs.LogsV2BodyJSONColumn)
+				delete(rr.Data, telemetrylogs.LogsV2BodyPromotedColumn)
+			}
+			payload = typedPayload
+		}
+	}
+
 	return &qbtypes.Result{
 		Type:  q.kind,
 		Value: payload,
@@ -375,3 +411,18 @@ func decodeCursor(cur string) (int64, error) {
 	}
 	return strconv.ParseInt(string(b), 10, 64)
 }
+
+func seed(promoted map[string]any, body map[string]any) {
+	for key, fromValue := range promoted {
+		if toValue, ok := body[key]; !ok {
+			body[key] = fromValue
+		} else {
+			if fromValue, ok := fromValue.(map[string]any); ok {
+				if toValue, ok := toValue.(map[string]any); ok {
+					seed(fromValue, toValue)
+					body[key] = toValue
+				}
+			}
+		}
+	}
+}

pkg/querier/consume.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/bytedance/sonic"
 )
 
 var (
@@ -51,7 +52,6 @@ func consume(rows driver.Rows, kind qbtypes.RequestType, queryWindow *qbtypes.Ti
 }
 
 func readAsTimeSeries(rows driver.Rows, queryWindow *qbtypes.TimeRange, step qbtypes.Step, queryName string) (*qbtypes.TimeSeriesData, error) {
-
 	colTypes := rows.ColumnTypes()
 	colNames := rows.Columns()
 
@@ -354,10 +354,22 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
 	colTypes := rows.ColumnTypes()
 	colCnt := len(colNames)
 
+	// Helper that decides scan target per column based on DB type
+	makeScanTarget := func(i int) any {
+		dbt := strings.ToUpper(colTypes[i].DatabaseTypeName())
+		if strings.HasPrefix(dbt, "JSON") {
+			// Since the driver fails to decode JSON/Dynamic into native Go values, we read it as raw bytes
+			// TODO: check in future if fixed in the driver
+			var v []byte
+			return &v
+		}
+		return reflect.New(colTypes[i].ScanType()).Interface()
+	}
+
 	// Build a template slice of correctly-typed pointers once
 	scanTpl := make([]any, colCnt)
-	for i, ct := range colTypes {
-		scanTpl[i] = reflect.New(ct.ScanType()).Interface()
+	for i := range colTypes {
+		scanTpl[i] = makeScanTarget(i)
 	}
 
 	var outRows []*qbtypes.RawRow
@@ -366,7 +378,7 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
 		// fresh copy of the scan slice (otherwise the driver reuses pointers)
 		scan := make([]any, colCnt)
 		for i := range scanTpl {
-			scan[i] = reflect.New(colTypes[i].ScanType()).Interface()
+			scan[i] = makeScanTarget(i)
 		}
 
 		if err := rows.Scan(scan...); err != nil {
@@ -383,6 +395,28 @@ func readAsRaw(rows driver.Rows, queryName string) (*qbtypes.RawData, error) {
 			// de-reference the typed pointer to any
 			val := reflect.ValueOf(cellPtr).Elem().Interface()
 
+			// Post-process JSON columns: normalize into structured values
+			if strings.HasPrefix(strings.ToUpper(colTypes[i].DatabaseTypeName()), "JSON") {
+				switch x := val.(type) {
+				case []byte:
+					if len(x) > 0 {
+						var v any
+						if err := sonic.Unmarshal(x, &v); err == nil {
+							val = v
+						}
+					}
+				case string:
+					if x != "" {
+						var v any
+						if err := sonic.Unmarshal([]byte(x), &v); err == nil {
+							val = v
+						}
+					}
+				default:
+					// already a structured type (map[string]any, []any, etc.)
+				}
+			}
+
 			// special-case: timestamp column
 			if name == "timestamp" || name == "timestamp_datetime" {
 				switch t := val.(type) {

pkg/querier/signozquerier/provider.go

@@ -36,7 +36,7 @@ func NewFactory(
 }
 
 func newProvider(
-	_ context.Context,
+	ctx context.Context,
 	settings factory.ProviderSettings,
 	cfg querier.Config,
 	telemetryStore telemetrystore.TelemetryStore,
@@ -78,7 +78,7 @@ func newProvider(
 		telemetryMetadataStore,
 	)
 
-	traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, "", nil)
+	traceAggExprRewriter := querybuilder.NewAggExprRewriter(settings, nil, traceFieldMapper, traceConditionBuilder, nil)
 	traceStmtBuilder := telemetrytraces.NewTraceQueryStatementBuilder(
 		settings,
 		telemetryMetadataStore,
@@ -100,16 +100,15 @@ func newProvider(
 		traceAggExprRewriter,
 	)
 
-	// Create log statement builder
+	// Create field mapper and condition builder for body JSON queries
 	logFieldMapper := telemetrylogs.NewFieldMapper()
-	logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper)
+	logConditionBuilder := telemetrylogs.NewConditionBuilder(logFieldMapper, telemetryMetadataStore)
 	logResourceFilterStmtBuilder := resourcefilter.NewLogResourceFilterStatementBuilder(
 		settings,
 		resourceFilterFieldMapper,
 		resourceFilterConditionBuilder,
 		telemetryMetadataStore,
 		telemetrylogs.DefaultFullTextColumn,
-		telemetrylogs.BodyJSONStringSearchPrefix,
 		telemetrylogs.GetBodyJSONKey,
 	)
 	logAggExprRewriter := querybuilder.NewAggExprRewriter(
@@ -117,7 +116,6 @@ func newProvider(
 		telemetrylogs.DefaultFullTextColumn,
 		logFieldMapper,
 		logConditionBuilder,
-		telemetrylogs.BodyJSONStringSearchPrefix,
 		telemetrylogs.GetBodyJSONKey,
 	)
 	logStmtBuilder := telemetrylogs.NewLogQueryStatementBuilder(
@@ -128,7 +126,6 @@ func newProvider(
 		logResourceFilterStmtBuilder,
 		logAggExprRewriter,
 		telemetrylogs.DefaultFullTextColumn,
-		telemetrylogs.BodyJSONStringSearchPrefix,
 		telemetrylogs.GetBodyJSONKey,
 	)
 

pkg/query-service/app/http_handler.go

@@ -549,6 +549,7 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *middleware.AuthZ) {
 	router.HandleFunc("/api/v1/settings/ttl", am.ViewAccess(aH.getTTL)).Methods(http.MethodGet)
 	router.HandleFunc("/api/v2/settings/ttl", am.AdminAccess(aH.setCustomRetentionTTL)).Methods(http.MethodPost)
 	router.HandleFunc("/api/v2/settings/ttl", am.ViewAccess(aH.getCustomRetentionTTL)).Methods(http.MethodGet)
+
 	router.HandleFunc("/api/v1/settings/apdex", am.AdminAccess(aH.Signoz.Handlers.Apdex.Set)).Methods(http.MethodPost)
 	router.HandleFunc("/api/v1/settings/apdex", am.ViewAccess(aH.Signoz.Handlers.Apdex.Get)).Methods(http.MethodGet)
 
@@ -4020,6 +4021,9 @@ func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router, am *middleware.Auth
 	subRouter.HandleFunc("/pipelines/preview", am.ViewAccess(aH.PreviewLogsPipelinesHandler)).Methods(http.MethodPost)
 	subRouter.HandleFunc("/pipelines/{version}", am.ViewAccess(aH.ListLogsPipelinesHandler)).Methods(http.MethodGet)
 	subRouter.HandleFunc("/pipelines", am.EditAccess(aH.CreateLogsPipeline)).Methods(http.MethodPost)
+
+	// Promote and index JSON paths used in logs
+	subRouter.HandleFunc("/promote_paths", am.AdminAccess(aH.Signoz.Handlers.Promote.HandlePromote)).Methods(http.MethodGet, http.MethodPost, http.MethodDelete)
 }
 
 func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) {

pkg/query-service/app/logparsingpipeline/controller.go

@@ -11,6 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/query-service/agentConf"
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/query-service/model"
+	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
 	"github.com/SigNoz/signoz/pkg/query-service/utils"
 	"github.com/SigNoz/signoz/pkg/sqlstore"
 	"github.com/SigNoz/signoz/pkg/types"
@@ -18,6 +19,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/types/pipelinetypes"
 	"github.com/SigNoz/signoz/pkg/valuer"
 	"github.com/google/uuid"
+
 	"go.uber.org/zap"
 )
 
@@ -128,6 +130,40 @@ func (ic *LogParsingPipelineController) ValidatePipelines(ctx context.Context,
 	return err
 }
 
+func (ic *LogParsingPipelineController) getDefaultPipelines() ([]pipelinetypes.GettablePipeline, error) {
+	defaultPipelines := []pipelinetypes.GettablePipeline{}
+	if constants.BodyJSONQueryEnabled {
+		preprocessingPipeline := pipelinetypes.GettablePipeline{
+			StoreablePipeline: pipelinetypes.StoreablePipeline{
+				Name:    "Default Pipeline - PreProcessing Body",
+				Alias:   "NormalizeBodyDefault",
+				Enabled: true,
+			},
+			Filter: &v3.FilterSet{
+				Items: []v3.FilterItem{
+					{
+						Key: v3.AttributeKey{
+							Key: "body",
+						},
+						Operator: v3.FilterOperatorExists,
+					},
+				},
+			},
+			Config: []pipelinetypes.PipelineOperator{
+				{
+					ID:      uuid.NewString(),
+					Type:    "normalize",
+					Enabled: true,
+					If:      "body != nil",
+				},
+			},
+		}
+
+		defaultPipelines = append(defaultPipelines, preprocessingPipeline)
+	}
+	return defaultPipelines, nil
+}
+
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(
@@ -258,6 +294,13 @@ func (pc *LogParsingPipelineController) RecommendAgentConfig(
 		return nil, "", err
 	}
 
+	// recommend default pipelines along with user created pipelines
+	defaultPipelines, err := pc.getDefaultPipelines()
+	if err != nil {
+		return nil, "", model.InternalError(fmt.Errorf("failed to get default pipelines: %w", err))
+	}
+	pipelinesResp.Pipelines = append(pipelinesResp.Pipelines, defaultPipelines...)
+
 	updatedConf, err := GenerateCollectorConfigWithPipelines(currentConfYaml, pipelinesResp.Pipelines)
 	if err != nil {
 		return nil, "", err

pkg/query-service/app/logparsingpipeline/preview.go

@@ -132,7 +132,7 @@ func SignozLogsToPLogs(logs []model.SignozLog) []plog.Logs {
 		slRecord.SetSeverityText(log.SeverityText)
 		slRecord.SetSeverityNumber(plog.SeverityNumber(log.SeverityNumber))
 
-		slRecord.Body().SetStr(log.Body)
+		slRecord.Body().FromRaw(log.Body)
 
 		slAttribs := slRecord.Attributes()
 		for k, v := range log.Attributes_int64 {

pkg/query-service/app/logs/v4/query_builder.go

@@ -404,7 +404,7 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build
 	// if noop create the query and return
 	if mq.AggregateOperator == v3.AggregateOperatorNoOp {
 		// with noop any filter or different order by other than ts will use new table
-		sqlSelect := constants.LogsSQLSelectV2
+		sqlSelect := constants.LogsSQLSelectV2()
 		queryTmpl := sqlSelect + "from signoz_logs.%s where %s%s order by %s"
 		query := fmt.Sprintf(queryTmpl, DISTRIBUTED_LOGS_V2, timeFilter, filterSubQuery, orderBy)
 		return query, nil
@@ -488,7 +488,7 @@ func buildLogsLiveTailQuery(mq *v3.BuilderQuery) (string, error) {
 	// the reader will add the timestamp and id filters
 	switch mq.AggregateOperator {
 	case v3.AggregateOperatorNoOp:
-		query := constants.LogsSQLSelectV2 + "from signoz_logs." + DISTRIBUTED_LOGS_V2 + " where "
+		query := constants.LogsSQLSelectV2() + "from signoz_logs." + DISTRIBUTED_LOGS_V2 + " where "
 		if len(filterSubQuery) > 0 {
 			query = query + filterSubQuery + " AND "
 		}

pkg/query-service/app/logs/v4/query_builder_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
+	"github.com/stretchr/testify/assert"
 )
 
 func Test_getClickhouseKey(t *testing.T) {
@@ -1210,9 +1211,8 @@ func TestPrepareLogsQuery(t *testing.T) {
 				t.Errorf("PrepareLogsQuery() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if got != tt.want {
-				t.Errorf("PrepareLogsQuery() = %v, want %v", got, tt.want)
-			}
+
+			assert.Equal(t, tt.want, got)
 		})
 	}
 }

pkg/query-service/constants/constants.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/query-service/model"
 	v3 "github.com/SigNoz/signoz/pkg/query-service/model/v3"
+	"github.com/huandu/go-sqlbuilder"
 )
 
 const (
@@ -216,13 +217,6 @@ const (
 		"CAST((attributes_bool_key, attributes_bool_value), 'Map(String, Bool)') as  attributes_bool," +
 		"CAST((resources_string_key, resources_string_value), 'Map(String, String)') as resources_string," +
 		"CAST((scope_string_key, scope_string_value), 'Map(String, String)') as scope "
-	LogsSQLSelectV2 = "SELECT " +
-		"timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, " +
-		"attributes_string, " +
-		"attributes_number, " +
-		"attributes_bool, " +
-		"resources_string, " +
-		"scope_string "
 	TracesExplorerViewSQLSelectWithSubQuery = "(SELECT traceID, durationNano, " +
 		"serviceName, name FROM %s.%s WHERE parentSpanID = '' AND %s ORDER BY durationNano DESC LIMIT 1 BY traceID"
 	TracesExplorerViewSQLSelectBeforeSubQuery = "SELECT subQuery.serviceName as `subQuery.serviceName`, subQuery.name as `subQuery.name`, count() AS " +
@@ -692,6 +686,7 @@ var StaticFieldsTraces = map[string]v3.AttributeKey{}
 var IsDotMetricsEnabled = false
 var PreferSpanMetrics = false
 var MaxJSONFlatteningDepth = 1
+var BodyJSONQueryEnabled = GetOrDefaultEnv("BODY_JSON_QUERY_ENABLED", "false") == "true"
 
 func init() {
 	StaticFieldsTraces = maps.Clone(NewStaticFieldsTraces)
@@ -732,3 +727,15 @@ const InspectMetricsMaxTimeDiff = 1800000
 
 const DotMetricsEnabled = "DOT_METRICS_ENABLED"
 const maxJSONFlatteningDepth = "MAX_JSON_FLATTENING_DEPTH"
+
+func LogsSQLSelectV2() string {
+	sb := sqlbuilder.NewSelectBuilder()
+	columns := []string{"timestamp", "id", "trace_id", "span_id", "trace_flags", "severity_text", "severity_number", "scope_name", "scope_version", "body"}
+	if BodyJSONQueryEnabled {
+		columns = append(columns, "body_json", "body_json_promoted")
+	}
+	columns = append(columns, "attributes_string", "attributes_number", "attributes_bool", "resources_string", "scope_string")
+	sb.Select(columns...)
+	query, _ := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return query + " " // add space to avoid concatenation issues
+}

pkg/query-service/model/response.go

@@ -598,13 +598,16 @@ type SignozLog struct {
 	TraceFlags         uint32             `json:"trace_flags" ch:"trace_flags"`
 	SeverityText       string             `json:"severity_text" ch:"severity_text"`
 	SeverityNumber     uint8              `json:"severity_number" ch:"severity_number"`
-	Body               string             `json:"body" ch:"body"`
+	Body               any                `json:"body" ch:"body"`
+	BodyJSON           map[string]any     `json:"-" ch:"body_json"`
+	BodyPromoted       map[string]any     `json:"-" ch:"body_json_promoted"`
 	Resources_string   map[string]string  `json:"resources_string" ch:"resources_string"`
 	Attributes_string  map[string]string  `json:"attributes_string" ch:"attributes_string"`
 	Attributes_int64   map[string]int64   `json:"attributes_int" ch:"attributes_int64"`
 	Attributes_float64 map[string]float64 `json:"attributes_float" ch:"attributes_float64"`
 	Attributes_bool    map[string]bool    `json:"attributes_bool" ch:"attributes_bool"`
 }
+
 type GetLogsAggregatesResponse struct {
 	Items map[int64]LogsAggregatesResponseItem `json:"items"`
 }

pkg/querybuilder/agg_rewrite.go

@@ -20,7 +20,6 @@ type aggExprRewriter struct {
 	fullTextColumn   *telemetrytypes.TelemetryFieldKey
 	fieldMapper      qbtypes.FieldMapper
 	conditionBuilder qbtypes.ConditionBuilder
-	jsonBodyPrefix   string
 	jsonKeyToKey     qbtypes.JsonKeyToFieldFunc
 }
 
@@ -31,7 +30,6 @@ func NewAggExprRewriter(
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
 	fieldMapper qbtypes.FieldMapper,
 	conditionBuilder qbtypes.ConditionBuilder,
-	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *aggExprRewriter {
 	set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/agg_rewrite")
@@ -41,7 +39,6 @@ func NewAggExprRewriter(
 		fullTextColumn:   fullTextColumn,
 		fieldMapper:      fieldMapper,
 		conditionBuilder: conditionBuilder,
-		jsonBodyPrefix:   jsonBodyPrefix,
 		jsonKeyToKey:     jsonKeyToKey,
 	}
 }
@@ -81,7 +78,6 @@ func (r *aggExprRewriter) Rewrite(
 		r.fullTextColumn,
 		r.fieldMapper,
 		r.conditionBuilder,
-		r.jsonBodyPrefix,
 		r.jsonKeyToKey,
 	)
 	// Rewrite the first select item (our expression)
@@ -129,7 +125,6 @@ type exprVisitor struct {
 	fullTextColumn   *telemetrytypes.TelemetryFieldKey
 	fieldMapper      qbtypes.FieldMapper
 	conditionBuilder qbtypes.ConditionBuilder
-	jsonBodyPrefix   string
 	jsonKeyToKey     qbtypes.JsonKeyToFieldFunc
 	Modified         bool
 	chArgs           []any
@@ -142,7 +137,6 @@ func newExprVisitor(
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
 	fieldMapper qbtypes.FieldMapper,
 	conditionBuilder qbtypes.ConditionBuilder,
-	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *exprVisitor {
 	return &exprVisitor{
@@ -151,7 +145,6 @@ func newExprVisitor(
 		fullTextColumn:   fullTextColumn,
 		fieldMapper:      fieldMapper,
 		conditionBuilder: conditionBuilder,
-		jsonBodyPrefix:   jsonBodyPrefix,
 		jsonKeyToKey:     jsonKeyToKey,
 	}
 }
@@ -198,7 +191,6 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 				FieldMapper:      v.fieldMapper,
 				ConditionBuilder: v.conditionBuilder,
 				FullTextColumn:   v.fullTextColumn,
-				JsonBodyPrefix:   v.jsonBodyPrefix,
 				JsonKeyToKey:     v.jsonKeyToKey,
 			}, 0, 0,
 		)
@@ -220,7 +212,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 		for i := 0; i < len(args)-1; i++ {
 			origVal := args[i].String()
 			fieldKey := telemetrytypes.GetFieldKeyFromKeyText(origVal)
-			expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonBodyPrefix, v.jsonKeyToKey)
+			expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
 			if err != nil {
 				return errors.WrapInvalidInputf(err, errors.CodeInvalidInput, "failed to get table field name for %q", origVal)
 			}
@@ -238,7 +230,7 @@ func (v *exprVisitor) VisitFunctionExpr(fn *chparser.FunctionExpr) error {
 		for i, arg := range args {
 			orig := arg.String()
 			fieldKey := telemetrytypes.GetFieldKeyFromKeyText(orig)
-			expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonBodyPrefix, v.jsonKeyToKey)
+			expr, exprArgs, err := CollisionHandledFinalExpr(context.Background(), &fieldKey, v.fieldMapper, v.conditionBuilder, v.fieldKeys, dataType, v.jsonKeyToKey)
 			if err != nil {
 				return err
 			}

pkg/querybuilder/fallback_expr.go

@@ -24,7 +24,6 @@ func CollisionHandledFinalExpr(
 	cb qbtypes.ConditionBuilder,
 	keys map[string][]*telemetrytypes.TelemetryFieldKey,
 	requiredDataType telemetrytypes.FieldDataType,
-	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) (string, []any, error) {
 
@@ -58,8 +57,8 @@ func CollisionHandledFinalExpr(
 		return nil
 	}
 
-	colName, err := fm.FieldFor(ctx, field)
-	if errors.Is(err, qbtypes.ErrColumnNotFound) {
+	colName, fieldForErr := fm.FieldFor(ctx, field)
+	if errors.Is(fieldForErr, qbtypes.ErrColumnNotFound) {
 		// the key didn't have the right context to be added to the query
 		// we try to use the context we know of
 		keysForField := keys[field.Name]
@@ -82,10 +81,10 @@ func CollisionHandledFinalExpr(
 			correction, found := telemetrytypes.SuggestCorrection(field.Name, maps.Keys(keys))
 			if found {
 				// we found a close match, in the error message send the suggestion
-				return "", nil, errors.Wrap(err, errors.TypeInvalidInput, errors.CodeInvalidInput, correction)
+				return "", nil, errors.WithAdditionalf(fieldForErr, "%s", correction)
 			} else {
 				// not even a close match, return an error
-				return "", nil, errors.Wrapf(err, errors.TypeInvalidInput, errors.CodeInvalidInput, "field `%s` not found", field.Name)
+				return "", nil, errors.WithAdditionalf(fieldForErr, "field `%s` not found", field.Name)
 			}
 		} else {
 			for _, key := range keysForField {
@@ -104,10 +103,8 @@ func CollisionHandledFinalExpr(
 			return "", nil, err
 		}
 
-		if strings.HasPrefix(field.Name, jsonBodyPrefix) && jsonBodyPrefix != "" && jsonKeyToKey != nil {
-			// TODO(nitya): enable group by on body column?
-			return "", nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the body column")
-			// colName, _ = jsonKeyToKey(context.Background(), field, qbtypes.FilterOperatorUnknown, dummyValue)
+		if field.FieldContext == telemetrytypes.FieldContextBody && jsonKeyToKey != nil {
+			return "", nil, fieldForErr
 		} else {
 			colName, _ = DataTypeCollisionHandledFieldName(field, dummyValue, colName, qbtypes.FilterOperatorUnknown)
 		}
@@ -204,7 +201,7 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
 	// While we expect user not to send the mixed data types, it inevitably happens
 	// So we handle the data type collisions here
 	switch key.FieldDataType {
-	case telemetrytypes.FieldDataTypeString:
+	case telemetrytypes.FieldDataTypeString, telemetrytypes.FieldDataTypeArrayString:
 		switch v := value.(type) {
 		case float64:
 			// try to convert the string value to to number
@@ -219,8 +216,40 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
 			// we don't have a toBoolOrNull in ClickHouse, so we need to convert the bool to a string
 			value = fmt.Sprintf("%t", v)
 		}
+	case telemetrytypes.FieldDataTypeFloat64,
+		telemetrytypes.FieldDataTypeArrayFloat64:
+		switch v := value.(type) {
+		case string:
+			// check if it's a number inside a string
+			isNumber := false
+			if _, err := strconv.ParseFloat(v, 64); err == nil {
+				isNumber = true
+			}
 
-	case telemetrytypes.FieldDataTypeFloat64, telemetrytypes.FieldDataTypeInt64, telemetrytypes.FieldDataTypeNumber:
+			if !operator.IsComparisonOperator() || !isNumber {
+				// try to convert the number attribute to string
+				tblFieldName = castString(tblFieldName) // numeric col vs string literal
+			} else {
+				tblFieldName = castFloatHack(tblFieldName)
+			}
+		case []any:
+			if allFloats(v) {
+				tblFieldName = castFloatHack(tblFieldName)
+			} else if hasString(v) {
+				tblFieldName, value = castString(tblFieldName), toStrings(v)
+			}
+		}
+
+	case telemetrytypes.FieldDataTypeInt64,
+		telemetrytypes.FieldDataTypeArrayInt64,
+		telemetrytypes.FieldDataTypeNumber,
+		telemetrytypes.FieldDataTypeArrayNumber:
+		// case telemetrytypes.FieldDataTypeFloat64,
+		// 	telemetrytypes.FieldDataTypeArrayFloat64,
+		// 	telemetrytypes.FieldDataTypeInt64,
+		// 	telemetrytypes.FieldDataTypeArrayInt64,
+		// 	telemetrytypes.FieldDataTypeNumber,
+		// 	telemetrytypes.FieldDataTypeArrayNumber:
 		switch v := value.(type) {
 		// why? ; CH returns an error for a simple check
 		// attributes_number['http.status_code'] = 200 but not for attributes_number['http.status_code'] >= 200
@@ -258,7 +287,8 @@ func DataTypeCollisionHandledFieldName(key *telemetrytypes.TelemetryFieldKey, va
 			}
 		}
 
-	case telemetrytypes.FieldDataTypeBool:
+	case telemetrytypes.FieldDataTypeBool,
+		telemetrytypes.FieldDataTypeArrayBool:
 		switch v := value.(type) {
 		case string:
 			tblFieldName = castString(tblFieldName)

pkg/querybuilder/query_to_keys_test.go

@@ -7,7 +7,6 @@ import (
 )
 
 func TestQueryToKeys(t *testing.T) {
-
 	testCases := []struct {
 		query        string
 		expectedKeys []telemetrytypes.FieldKeySelector
@@ -66,9 +65,9 @@ func TestQueryToKeys(t *testing.T) {
 			query: `body.user_ids[*] = 123`,
 			expectedKeys: []telemetrytypes.FieldKeySelector{
 				{
-					Name:          "body.user_ids[*]",
+					Name:          "user_ids[*]",
 					Signal:        telemetrytypes.SignalUnspecified,
-					FieldContext:  telemetrytypes.FieldContextUnspecified,
+					FieldContext:  telemetrytypes.FieldContextBody,
 					FieldDataType: telemetrytypes.FieldDataTypeUnspecified,
 				},
 			},

pkg/querybuilder/resourcefilter/statement_builder.go

@@ -43,7 +43,6 @@ type resourceFilterStatementBuilder[T any] struct {
 	signal           telemetrytypes.Signal
 
 	fullTextColumn *telemetrytypes.TelemetryFieldKey
-	jsonBodyPrefix string
 	jsonKeyToKey   qbtypes.JsonKeyToFieldFunc
 }
 
@@ -76,7 +75,6 @@ func NewLogResourceFilterStatementBuilder(
 	conditionBuilder qbtypes.ConditionBuilder,
 	metadataStore telemetrytypes.MetadataStore,
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
-	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *resourceFilterStatementBuilder[qbtypes.LogAggregation] {
 	set := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/querybuilder/resourcefilter")
@@ -87,7 +85,6 @@ func NewLogResourceFilterStatementBuilder(
 		metadataStore:    metadataStore,
 		signal:           telemetrytypes.SignalLogs,
 		fullTextColumn:   fullTextColumn,
-		jsonBodyPrefix:   jsonBodyPrefix,
 		jsonKeyToKey:     jsonKeyToKey,
 	}
 }
@@ -100,12 +97,18 @@ func (b *resourceFilterStatementBuilder[T]) getKeySelectors(query qbtypes.QueryB
 		keySelectors = append(keySelectors, whereClauseSelectors...)
 	}
 
+	// exclude out the body related key selectors
+	filteredKeySelectors := []*telemetrytypes.FieldKeySelector{}
 	for idx := range keySelectors {
+		if keySelectors[idx].FieldContext == telemetrytypes.FieldContextBody {
+			continue
+		}
 		keySelectors[idx].Signal = b.signal
 		keySelectors[idx].SelectorMatchType = telemetrytypes.FieldSelectorMatchTypeExact
+		filteredKeySelectors = append(filteredKeySelectors, keySelectors[idx])
 	}
 
-	return keySelectors
+	return filteredKeySelectors
 }
 
 // Build builds a SQL query based on the given parameters
@@ -162,7 +165,6 @@ func (b *resourceFilterStatementBuilder[T]) addConditions(
 			ConditionBuilder:   b.conditionBuilder,
 			FieldKeys:          keys,
 			FullTextColumn:     b.fullTextColumn,
-			JsonBodyPrefix:     b.jsonBodyPrefix,
 			JsonKeyToKey:       b.jsonKeyToKey,
 			SkipFullTextFilter: true,
 			SkipFunctionCalls:  true,

pkg/querybuilder/where_clause_visitor.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	grammar "github.com/SigNoz/signoz/pkg/parser/grammar"
+	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 	"github.com/antlr4-go/antlr/v4"
@@ -33,7 +34,6 @@ type filterExpressionVisitor struct {
 	mainErrorURL       string
 	builder            *sqlbuilder.SelectBuilder
 	fullTextColumn     *telemetrytypes.TelemetryFieldKey
-	jsonBodyPrefix     string
 	jsonKeyToKey       qbtypes.JsonKeyToFieldFunc
 	skipResourceFilter bool
 	skipFullTextFilter bool
@@ -53,7 +53,6 @@ type FilterExprVisitorOpts struct {
 	FieldKeys          map[string][]*telemetrytypes.TelemetryFieldKey
 	Builder            *sqlbuilder.SelectBuilder
 	FullTextColumn     *telemetrytypes.TelemetryFieldKey
-	JsonBodyPrefix     string
 	JsonKeyToKey       qbtypes.JsonKeyToFieldFunc
 	SkipResourceFilter bool
 	SkipFullTextFilter bool
@@ -73,7 +72,6 @@ func newFilterExpressionVisitor(opts FilterExprVisitorOpts) *filterExpressionVis
 		fieldKeys:          opts.FieldKeys,
 		builder:            opts.Builder,
 		fullTextColumn:     opts.FullTextColumn,
-		jsonBodyPrefix:     opts.JsonBodyPrefix,
 		jsonKeyToKey:       opts.JsonKeyToKey,
 		skipResourceFilter: opts.SkipResourceFilter,
 		skipFullTextFilter: opts.SkipFullTextFilter,
@@ -172,7 +170,7 @@ func PrepareWhereClause(query string, opts FilterExprVisitorOpts, startNs uint64
 
 	whereClause := sqlbuilder.NewWhereClause().AddWhereExpr(visitor.builder.Args, cond)
 
-	return &PreparedWhereClause{whereClause, visitor.warnings, visitor.mainWarnURL}, nil
+	return &PreparedWhereClause{WhereClause: whereClause, Warnings: visitor.warnings, WarningsDocURL: visitor.mainWarnURL}, nil
 }
 
 // Visit dispatches to the specific visit method based on node type
@@ -717,7 +715,7 @@ func (v *filterExpressionVisitor) VisitFunctionCall(ctx *grammar.FunctionCallCon
 			conds = append(conds, fmt.Sprintf("hasToken(LOWER(%s), LOWER(%s))", key.Name, v.builder.Var(value[0])))
 		} else {
 			// this is that all other functions only support array fields
-			if strings.HasPrefix(key.Name, v.jsonBodyPrefix) {
+			if key.FieldContext == telemetrytypes.FieldContextBody {
 				fieldName, _ = v.jsonKeyToKey(context.Background(), key, qbtypes.FilterOperatorUnknown, value)
 			} else {
 				// TODO(add docs for json body search)
@@ -808,10 +806,8 @@ func (v *filterExpressionVisitor) VisitValue(ctx *grammar.ValueContext) any {
 
 // VisitKey handles field/column references
 func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
-
 	fieldKey := telemetrytypes.GetFieldKeyFromKeyText(ctx.GetText())
-
-	keyName := strings.TrimPrefix(fieldKey.Name, v.jsonBodyPrefix)
+	keyName := fieldKey.Name
 
 	fieldKeysForName := v.fieldKeys[keyName]
 
@@ -845,11 +841,12 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
 	// if there is a field with the same name as attribute/resource attribute
 	// Since it will ORed with the fieldKeysForName, it will not result empty
 	// when either of them have values
-	if strings.HasPrefix(fieldKey.Name, v.jsonBodyPrefix) && v.jsonBodyPrefix != "" {
-		if keyName != "" {
+	// Note: Skip this logic if body json query is enabled so we can look up the key inside fields
+	//
+	// TODO(Piyush): After entire migration this is supposed to be removed.
+	if !constants.BodyJSONQueryEnabled && fieldKey.FieldContext == telemetrytypes.FieldContextBody {
 		fieldKeysForName = append(fieldKeysForName, &fieldKey)
 	}
-	}
 
 	if len(fieldKeysForName) == 0 {
 		// check if the key exists with {fieldContext}.{key}
@@ -859,7 +856,7 @@ func (v *filterExpressionVisitor) VisitKey(ctx *grammar.KeyContext) any {
 			return v.fieldKeys[keyWithContext]
 		}
 
-		if strings.HasPrefix(fieldKey.Name, v.jsonBodyPrefix) && v.jsonBodyPrefix != "" && keyName == "" {
+		if fieldKey.FieldContext == telemetrytypes.FieldContextBody && keyName == "" {
 			v.errors = append(v.errors, "missing key for body json search - expected key of the form `body.key` (ex: `body.status`)")
 		} else if !v.ignoreNotFoundKeys {
 			// TODO(srikanthccv): do we want to return an error here?

pkg/signoz/handler.go

@@ -13,6 +13,8 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
 	"github.com/SigNoz/signoz/pkg/modules/preference"
 	"github.com/SigNoz/signoz/pkg/modules/preference/implpreference"
+	"github.com/SigNoz/signoz/pkg/modules/promote"
+	"github.com/SigNoz/signoz/pkg/modules/promote/implpromote"
 	"github.com/SigNoz/signoz/pkg/modules/quickfilter"
 	"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
 	"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
@@ -46,6 +48,7 @@ type Handlers struct {
 	Session        session.Handler
 	SpanPercentile spanpercentile.Handler
 	Services       services.Handler
+	Promote        promote.Handler
 }
 
 func NewHandlers(modules Modules, providerSettings factory.ProviderSettings, querier querier.Querier, licensing licensing.Licensing) Handlers {
@@ -63,5 +66,6 @@ func NewHandlers(modules Modules, providerSettings factory.ProviderSettings, que
 		Session:        implsession.NewHandler(modules.Session),
 		Services:       implservices.NewHandler(modules.Services),
 		SpanPercentile: implspanpercentile.NewHandler(modules.SpanPercentile),
+		Promote:        implpromote.NewHandler(modules.Promote),
 	}
 }

pkg/signoz/module.go

@@ -17,6 +17,8 @@ import (
 	"github.com/SigNoz/signoz/pkg/modules/organization/implorganization"
 	"github.com/SigNoz/signoz/pkg/modules/preference"
 	"github.com/SigNoz/signoz/pkg/modules/preference/implpreference"
+	"github.com/SigNoz/signoz/pkg/modules/promote"
+	"github.com/SigNoz/signoz/pkg/modules/promote/implpromote"
 	"github.com/SigNoz/signoz/pkg/modules/quickfilter"
 	"github.com/SigNoz/signoz/pkg/modules/quickfilter/implquickfilter"
 	"github.com/SigNoz/signoz/pkg/modules/rawdataexport"
@@ -58,6 +60,7 @@ type Modules struct {
 	Session        session.Module
 	Services       services.Module
 	SpanPercentile spanpercentile.Module
+	Promote        promote.Module
 }
 
 func NewModules(
@@ -94,5 +97,6 @@ func NewModules(
 		Session:        implsession.NewModule(providerSettings, authNs, user, userGetter, implauthdomain.NewModule(implauthdomain.NewStore(sqlstore)), tokenizer, orgGetter),
 		SpanPercentile: implspanpercentile.NewModule(querier, providerSettings),
 		Services:       implservices.NewModule(querier, telemetryStore),
+		Promote:        implpromote.NewModule(telemetryStore),
 	}
 }

pkg/telemetrylogs/condition_builder.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"fmt"
 	"slices"
-	"strings"
 
 	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz/ee/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
@@ -18,10 +18,11 @@ import (
 
 type conditionBuilder struct {
 	fm            qbtypes.FieldMapper
+	metadataStore telemetrytypes.MetadataStore
 }
 
-func NewConditionBuilder(fm qbtypes.FieldMapper) *conditionBuilder {
-	return &conditionBuilder{fm: fm}
+func NewConditionBuilder(fm qbtypes.FieldMapper, metadataStore telemetrytypes.MetadataStore) *conditionBuilder {
+	return &conditionBuilder{fm: fm, metadataStore: metadataStore}
 }
 
 func (c *conditionBuilder) conditionFor(
@@ -31,7 +32,15 @@ func (c *conditionBuilder) conditionFor(
 	value any,
 	sb *sqlbuilder.SelectBuilder,
 ) (string, error) {
+	column, err := c.fm.ColumnFor(ctx, key)
+	if err != nil {
+		return "", err
+	}
 
+	// For JSON columns, preserve the original value type (numeric, bool, etc.)
+	// Only format to string for non-JSON columns that need string formatting
+	isJSONColumn := column.IsJSONColumn() && constants.BodyJSONQueryEnabled && key.FieldContext == telemetrytypes.FieldContextBody
+	if !isJSONColumn {
 		switch operator {
 		case qbtypes.FilterOperatorContains,
 			qbtypes.FilterOperatorNotContains,
@@ -41,18 +50,23 @@ func (c *conditionBuilder) conditionFor(
 			qbtypes.FilterOperatorNotLike:
 			value = querybuilder.FormatValueForContains(value)
 		}
+	}
 
-	column, err := c.fm.ColumnFor(ctx, key)
+	if isJSONColumn {
+		cond, err := c.buildJSONCondition(ctx, key, operator, value, sb)
 		if err != nil {
 			return "", err
 		}
+		return cond, nil
+	}
 
 	tblFieldName, err := c.fm.FieldFor(ctx, key)
 	if err != nil {
 		return "", err
 	}
 
-	if strings.HasPrefix(key.Name, BodyJSONStringSearchPrefix) {
+	// Check if this is a body JSON search - either by FieldContext
+	if key.FieldContext == telemetrytypes.FieldContextBody {
 		tblFieldName, value = GetBodyJSONKey(ctx, key, operator, value)
 	}
 
@@ -155,8 +169,7 @@ func (c *conditionBuilder) conditionFor(
 	// in the UI based query builder, `exists` and `not exists` are used for
 	// key membership checks, so depending on the column type, the condition changes
 	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
-
-		if strings.HasPrefix(key.Name, BodyJSONStringSearchPrefix) {
+		if key.FieldContext == telemetrytypes.FieldContextBody && !constants.BodyJSONQueryEnabled {
 			if operator == qbtypes.FilterOperatorExists {
 				return GetBodyJSONKeyForExists(ctx, key, operator, value), nil
 			} else {
@@ -165,13 +178,15 @@ func (c *conditionBuilder) conditionFor(
 		}
 
 		var value any
-		switch column.Type {
-		case schema.JSONColumnType{}:
+		// schema.JSONColumnType{} now can not be used in switch cases, so we need to check if the column is a JSON column
+		if column.IsJSONColumn() {
 			if operator == qbtypes.FilterOperatorExists {
 				return sb.IsNotNull(tblFieldName), nil
 			} else {
 				return sb.IsNull(tblFieldName), nil
 			}
+		}
+		switch column.Type {
 		case schema.ColumnTypeString, schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}:
 			value = ""
 			if operator == qbtypes.FilterOperatorExists {
@@ -226,11 +241,11 @@ func (c *conditionBuilder) ConditionFor(
 		return "", err
 	}
 
-	if operator.AddDefaultExistsFilter() {
+	if !(key.FieldContext == telemetrytypes.FieldContextBody && constants.BodyJSONQueryEnabled) && operator.AddDefaultExistsFilter() {
 		// skip adding exists filter for intrinsic fields
 		// with an exception for body json search
 		field, _ := c.fm.FieldFor(ctx, key)
-		if slices.Contains(maps.Keys(IntrinsicFields), field) && !strings.HasPrefix(key.Name, BodyJSONStringSearchPrefix) {
+		if slices.Contains(maps.Keys(IntrinsicFields), field) && key.FieldContext != telemetrytypes.FieldContextBody {
 			return condition, nil
 		}
 

pkg/telemetrylogs/condition_builder_test.go

@@ -271,7 +271,8 @@ func TestConditionFor(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	conditionBuilder := NewConditionBuilder(fm)
+	mockMetadataStore := buildTestTelemetryMetadataStore()
+	conditionBuilder := NewConditionBuilder(fm, mockMetadataStore)
 
 	for _, tc := range testCases {
 		sb := sqlbuilder.NewSelectBuilder()
@@ -324,7 +325,8 @@ func TestConditionForMultipleKeys(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	conditionBuilder := NewConditionBuilder(fm)
+	mockMetadataStore := buildTestTelemetryMetadataStore()
+	conditionBuilder := NewConditionBuilder(fm, mockMetadataStore)
 
 	for _, tc := range testCases {
 		sb := sqlbuilder.NewSelectBuilder()
@@ -363,7 +365,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Equal operator - int64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorEqual,
 			value:         200,
@@ -373,7 +376,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Equal operator - float64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.duration_ms",
+				Name:         "duration_ms",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorEqual,
 			value:         405.5,
@@ -383,7 +387,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Equal operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.method",
+				Name:         "http.method",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorEqual,
 			value:         "GET",
@@ -393,7 +398,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Equal operator - bool",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.success",
+				Name:         "http.success",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorEqual,
 			value:         true,
@@ -403,7 +409,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Exists operator",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorExists,
 			value:         nil,
@@ -413,7 +420,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Not Exists operator",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorNotExists,
 			value:         nil,
@@ -423,7 +431,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Greater than operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorGreaterThan,
 			value:         "200",
@@ -433,7 +442,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Greater than operator - int64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorGreaterThan,
 			value:         200,
@@ -443,7 +453,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Less than operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorLessThan,
 			value:         "300",
@@ -453,7 +464,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Less than operator - int64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorLessThan,
 			value:         300,
@@ -463,7 +475,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Contains operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorContains,
 			value:         "200",
@@ -473,7 +486,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Not Contains operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorNotContains,
 			value:         "200",
@@ -483,7 +497,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Between operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorBetween,
 			value:         []any{"200", "300"},
@@ -493,7 +508,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "Between operator - int64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorBetween,
 			value:         []any{400, 500},
@@ -503,7 +519,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "In operator - string",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorIn,
 			value:         []any{"200", "300"},
@@ -513,7 +530,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 		{
 			name: "In operator - int64",
 			key: telemetrytypes.TelemetryFieldKey{
-				Name: "body.http.status_code",
+				Name:         "http.status_code",
+				FieldContext: telemetrytypes.FieldContextBody,
 			},
 			operator:      qbtypes.FilterOperatorIn,
 			value:         []any{401, 404, 500},
@@ -523,7 +541,8 @@ func TestConditionForJSONBodySearch(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	conditionBuilder := NewConditionBuilder(fm)
+	mockMetadataStore := buildTestTelemetryMetadataStore()
+	conditionBuilder := NewConditionBuilder(fm, mockMetadataStore)
 
 	for _, tc := range testCases {
 		sb := sqlbuilder.NewSelectBuilder()

pkg/telemetrylogs/const.go

@@ -1,6 +1,8 @@
 package telemetrylogs
 
 import (
+	"github.com/SigNoz/signoz-otel-collector/constants"
+	"github.com/SigNoz/signoz-otel-collector/exporter/jsontypeexporter"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
 )
@@ -16,6 +18,8 @@ const (
 	LogsV2TimestampColumn         = "timestamp"
 	LogsV2ObservedTimestampColumn = "observed_timestamp"
 	LogsV2BodyColumn              = "body"
+	LogsV2BodyJSONColumn          = constants.BodyJSONColumn
+	LogsV2BodyPromotedColumn      = constants.BodyPromotedColumn
 	LogsV2TraceIDColumn           = "trace_id"
 	LogsV2SpanIDColumn            = "span_id"
 	LogsV2TraceFlagsColumn        = "trace_flags"
@@ -30,6 +34,11 @@ const (
 	LogsV2AttributesBoolColumn   = "attributes_bool"
 	LogsV2ResourcesStringColumn  = "resources_string"
 	LogsV2ScopeStringColumn      = "scope_string"
+
+	BodyJSONColumnPrefix     = constants.BodyJSONColumnPrefix
+	BodyPromotedColumnPrefix = constants.BodyPromotedColumnPrefix
+	ArraySep                 = jsontypeexporter.ArraySeparator
+	ArrayAnyIndex            = "[*]."
 )
 
 var (
@@ -39,7 +48,6 @@ var (
 		FieldContext:  telemetrytypes.FieldContextLog,
 		FieldDataType: telemetrytypes.FieldDataTypeString,
 	}
-	BodyJSONStringSearchPrefix = `body.`
 	IntrinsicFields = map[string]telemetrytypes.TelemetryFieldKey{
 		"body": {
 			Name:          "body",

pkg/telemetrylogs/field_mapper.go

@@ -6,6 +6,8 @@ import (
 	"strings"
 
 	schema "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz-otel-collector/utils"
+	"github.com/SigNoz/signoz/ee/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/errors"
 	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
@@ -28,6 +30,11 @@ var (
 		"severity_text":      {Name: "severity_text", Type: schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString}},
 		"severity_number":    {Name: "severity_number", Type: schema.ColumnTypeUInt8},
 		"body":               {Name: "body", Type: schema.ColumnTypeString},
+		LogsV2BodyJSONColumn: {Name: LogsV2BodyJSONColumn, Type: schema.JSONColumnType{
+			MaxDynamicTypes: utils.ToPointer(uint(32)),
+			MaxDynamicPaths: utils.ToPointer(uint(0)),
+		}},
+		LogsV2BodyPromotedColumn: {Name: LogsV2BodyPromotedColumn, Type: schema.JSONColumnType{}},
 		"attributes_string": {Name: "attributes_string", Type: schema.MapColumnType{
 			KeyType:   schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 			ValueType: schema.ColumnTypeString,
@@ -61,7 +68,7 @@ func NewFieldMapper() qbtypes.FieldMapper {
 	return &fieldMapper{}
 }
 
-func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
+func (m *fieldMapper) getColumn(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*schema.Column, error) {
 	switch key.FieldContext {
 	case telemetrytypes.FieldContextResource:
 		return logsV2Columns["resource"], nil
@@ -82,11 +89,24 @@ func (m *fieldMapper) getColumn(_ context.Context, key *telemetrytypes.Telemetry
 		case telemetrytypes.FieldDataTypeBool:
 			return logsV2Columns["attributes_bool"], nil
 		}
+	case telemetrytypes.FieldContextBody:
+		// Body context is for JSON body fields
+		// Use body_json if feature flag is enabled
+		if constants.BodyJSONQueryEnabled {
+			return logsV2Columns[LogsV2BodyJSONColumn], nil
+		}
+		// Fall back to legacy body column
+		return logsV2Columns["body"], nil
 	case telemetrytypes.FieldContextLog, telemetrytypes.FieldContextUnspecified:
 		col, ok := logsV2Columns[key.Name]
 		if !ok {
 			// check if the key has body JSON search
-			if strings.HasPrefix(key.Name, BodyJSONStringSearchPrefix) {
+			if strings.HasPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
+				// Use body_json if feature flag is enabled and we have a body condition builder
+				if constants.BodyJSONQueryEnabled {
+					return logsV2Columns[LogsV2BodyJSONColumn], nil
+				}
+				// Fall back to legacy body column
 				return logsV2Columns["body"], nil
 			}
 			return nil, qbtypes.ErrColumnNotFound
@@ -103,12 +123,15 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr
 		return "", err
 	}
 
-	switch column.Type {
-	case schema.JSONColumnType{}:
-		// json is only supported for resource context as of now
-		if key.FieldContext != telemetrytypes.FieldContextResource {
-			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
+	if !constants.BodyJSONQueryEnabled && key.FieldContext == telemetrytypes.FieldContextBody {
+		return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the body column")
 	}
+
+	// schema.JSONColumnType{} now can not be used in switch cases, so we need to check if the column is a JSON column
+	if column.IsJSONColumn() {
+		// json is only supported for resource context as of now
+		switch key.FieldContext {
+		case telemetrytypes.FieldContextResource:
 			oldColumn := logsV2Columns["resources_string"]
 			oldKeyName := fmt.Sprintf("%s['%s']", oldColumn.Name, key.Name)
 
@@ -118,10 +141,25 @@ func (m *fieldMapper) FieldFor(ctx context.Context, key *telemetrytypes.Telemetr
 				oldKeyName = telemetrytypes.FieldKeyToMaterializedColumnName(key)
 				oldKeyNameExists := telemetrytypes.FieldKeyToMaterializedColumnNameForExists(key)
 				return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, %s==true, %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldKeyNameExists, oldKeyName), nil
-		} else {
+			}
 			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
+		case telemetrytypes.FieldContextBody:
+			if strings.Contains(key.Name, ArraySep) || strings.Contains(key.Name, ArrayAnyIndex) {
+				return "", errors.NewInvalidInputf(errors.CodeInvalidInput, "Group by/Aggregation isn't available for the Array Paths: %s", key.Name)
+			}
+			expr := fmt.Sprintf("dynamicElement(%s, '%s')", BodyJSONColumnPrefix+key.Name, key.JSONDataType.StringValue())
+			if key.Materialized {
+				expr = fmt.Sprintf("coalesce(%s, %s)", expr, fmt.Sprintf("dynamicElement(%s, '%s')", BodyPromotedColumnPrefix+key.Name, key.JSONDataType.StringValue()))
+			}
+			// returning qbtypes.ErrColumnNotFound is a hack that will trigger the fallback expr logic to include all the types for the key
+			return expr, qbtypes.ErrColumnNotFound
+		default:
+			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
 		}
 
+	}
+
+	switch column.Type {
 	case schema.ColumnTypeString,
 		schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 		schema.ColumnTypeUInt64,

pkg/telemetrylogs/filter_expr_like_warning_test.go

@@ -11,7 +11,7 @@ import (
 // TestLikeAndILikeWithoutWildcards_Warns Tests that LIKE/ILIKE without wildcards add warnings and include docs URL
 func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
+	cb := NewConditionBuilder(fm, nil)
 
 	keys := buildCompleteFieldKeyMap()
 
@@ -21,7 +21,6 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
 		ConditionBuilder: cb,
 		FieldKeys:        keys,
 		FullTextColumn:   DefaultFullTextColumn,
-		JsonBodyPrefix:   BodyJSONStringSearchPrefix,
 		JsonKeyToKey:     GetBodyJSONKey,
 	}
 
@@ -48,7 +47,7 @@ func TestLikeAndILikeWithoutWildcards_Warns(t *testing.T) {
 // TestLikeAndILikeWithWildcards_NoWarn Tests that LIKE/ILIKE with wildcards do not add warnings
 func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
+	cb := NewConditionBuilder(fm, nil)
 
 	keys := buildCompleteFieldKeyMap()
 
@@ -58,7 +57,6 @@ func TestLikeAndILikeWithWildcards_NoWarn(t *testing.T) {
 		ConditionBuilder: cb,
 		FieldKeys:        keys,
 		FullTextColumn:   DefaultFullTextColumn,
-		JsonBodyPrefix:   BodyJSONStringSearchPrefix,
 		JsonKeyToKey:     GetBodyJSONKey,
 	}
 

pkg/telemetrylogs/filter_expr_logs_body_json_test.go

@@ -7,6 +7,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/instrumentation/instrumentationtest"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes/telemetrytypestest"
 	"github.com/huandu/go-sqlbuilder"
 	"github.com/stretchr/testify/require"
 )
@@ -14,8 +15,7 @@ import (
 // TestFilterExprLogsBodyJSON tests a comprehensive set of query patterns for body JSON search
 func TestFilterExprLogsBodyJSON(t *testing.T) {
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
-
+	cb := NewConditionBuilder(fm, telemetrytypestest.NewMockMetadataStore())
 	// Define a comprehensive set of field keys to support all test cases
 	keys := buildCompleteFieldKeyMap()
 
@@ -27,7 +27,6 @@ func TestFilterExprLogsBodyJSON(t *testing.T) {
 		FullTextColumn: &telemetrytypes.TelemetryFieldKey{
 			Name: "body",
 		},
-		JsonBodyPrefix: "body",
 		JsonKeyToKey: GetBodyJSONKey,
 	}
 

pkg/telemetrylogs/filter_expr_logs_test.go

@@ -16,7 +16,7 @@ import (
 // TestFilterExprLogs tests a comprehensive set of query patterns for logs search
 func TestFilterExprLogs(t *testing.T) {
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
+	cb := NewConditionBuilder(fm, nil)
 
 	// Define a comprehensive set of field keys to support all test cases
 	keys := buildCompleteFieldKeyMap()
@@ -27,7 +27,6 @@ func TestFilterExprLogs(t *testing.T) {
 		ConditionBuilder: cb,
 		FieldKeys:        keys,
 		FullTextColumn:   DefaultFullTextColumn,
-		JsonBodyPrefix:   BodyJSONStringSearchPrefix,
 		JsonKeyToKey:     GetBodyJSONKey,
 	}
 
@@ -2424,7 +2423,7 @@ func TestFilterExprLogs(t *testing.T) {
 // TestFilterExprLogs tests a comprehensive set of query patterns for logs search
 func TestFilterExprLogsConflictNegation(t *testing.T) {
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
+	cb := NewConditionBuilder(fm, nil)
 
 	// Define a comprehensive set of field keys to support all test cases
 	keys := buildCompleteFieldKeyMap()
@@ -2448,7 +2447,6 @@ func TestFilterExprLogsConflictNegation(t *testing.T) {
 		ConditionBuilder: cb,
 		FieldKeys:        keys,
 		FullTextColumn:   DefaultFullTextColumn,
-		JsonBodyPrefix:   BodyJSONStringSearchPrefix,
 		JsonKeyToKey:     GetBodyJSONKey,
 	}
 

pkg/telemetrylogs/json_access_pb.go

@@ -0,0 +1,149 @@
+package telemetrylogs
+
+import (
+	"context"
+	"slices"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+var (
+	CodePlanIndexOutOfBounds = errors.MustNewCode("plan_index_out_of_bounds")
+)
+
+type JSONAccessPlanBuilder struct {
+	key        *telemetrytypes.TelemetryFieldKey
+	value      any
+	op         qbtypes.FilterOperator
+	parts      []string
+	getTypes   func(ctx context.Context, path string) ([]telemetrytypes.JSONDataType, error)
+	isPromoted bool
+}
+
+// buildPlan recursively builds the path plan tree
+func (pb *JSONAccessPlanBuilder) buildPlan(ctx context.Context, index int, parent *telemetrytypes.JSONAccessNode, isDynArrChild bool) (*telemetrytypes.JSONAccessNode, error) {
+	if index >= len(pb.parts) {
+		return nil, errors.NewInvalidInputf(CodePlanIndexOutOfBounds, "index is out of bounds")
+	}
+
+	part := pb.parts[index]
+	pathSoFar := strings.Join(pb.parts[:index+1], ArraySep)
+	isTerminal := index == len(pb.parts)-1
+
+	// Calculate progression parameters based on parent's values
+	var maxTypes, maxPaths int
+	if isDynArrChild {
+		// Child of Dynamic array - reset progression to base values (16, 256)
+		// This happens when we switch from Array(Dynamic) to Array(JSON)
+		maxTypes = 16
+		maxPaths = 256
+	} else if parent != nil {
+		// Child of JSON array - use parent's progression divided by 2 and 4
+		maxTypes = parent.MaxDynamicTypes / 2
+		maxPaths = parent.MaxDynamicPaths / 4
+		if maxTypes < 0 {
+			maxTypes = 0
+		}
+		if maxPaths < 0 {
+			maxPaths = 0
+		}
+	}
+
+	types, err := pb.getTypes(ctx, pathSoFar)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create node for this path segment
+	node := &telemetrytypes.JSONAccessNode{
+		Name:            part,
+		IsTerminal:      isTerminal,
+		AvailableTypes:  types,
+		Branches:        make(map[telemetrytypes.JSONAccessBranchType]*telemetrytypes.JSONAccessNode),
+		Parent:          parent,
+		MaxDynamicTypes: maxTypes,
+		MaxDynamicPaths: maxPaths,
+	}
+
+	hasJSON := slices.Contains(node.AvailableTypes, telemetrytypes.ArrayJSON)
+	hasDynamic := slices.Contains(node.AvailableTypes, telemetrytypes.ArrayDynamic)
+
+	// Configure terminal if this is the last part
+	if isTerminal {
+		valueType, _ := inferDataType(pb.value, pb.op, pb.key)
+		node.TerminalConfig = &telemetrytypes.TerminalConfig{
+			Key:       pb.key,
+			ElemType:  *pb.key.JSONDataType,
+			ValueType: telemetrytypes.MappingFieldDataTypeToJSONDataType[valueType],
+		}
+	} else {
+		if hasJSON {
+			node.Branches[telemetrytypes.BranchJSON], err = pb.buildPlan(ctx, index+1, node, false)
+			if err != nil {
+				return nil, err
+			}
+		}
+		if hasDynamic {
+			node.Branches[telemetrytypes.BranchDynamic], err = pb.buildPlan(ctx, index+1, node, true)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return node, nil
+}
+
+// PlanJSON builds a tree structure representing the complete JSON path traversal
+// that precomputes all possible branches and their types
+func PlanJSON(ctx context.Context, key *telemetrytypes.TelemetryFieldKey, op qbtypes.FilterOperator,
+	value any,
+	getTypes func(ctx context.Context, path string) ([]telemetrytypes.JSONDataType, error),
+) (telemetrytypes.JSONAccessPlan, error) {
+	// if path is empty, return nil
+	if key.Name == "" {
+		return nil, errors.NewInvalidInputf(errors.CodeInvalidInput, "path is empty")
+	}
+
+	// TODO: PlanJSON requires the Start and End of the Query to select correct column between promoted and body_json using
+	// creation time in distributed_promoted_paths
+	path := strings.ReplaceAll(key.Name, ArrayAnyIndex, ArraySep)
+	parts := strings.Split(path, ArraySep)
+
+	pb := &JSONAccessPlanBuilder{
+		key:        key,
+		op:         op,
+		value:      value,
+		parts:      parts,
+		getTypes:   getTypes,
+		isPromoted: key.Materialized,
+	}
+	plans := telemetrytypes.JSONAccessPlan{}
+
+	node, err := pb.buildPlan(ctx, 0,
+		telemetrytypes.NewRootJSONAccessNode(LogsV2BodyJSONColumn,
+			32, 0),
+		false,
+	)
+	if err != nil {
+		return nil, err
+	}
+	plans = append(plans, node)
+
+	if pb.isPromoted {
+		node, err := pb.buildPlan(ctx, 0,
+			telemetrytypes.NewRootJSONAccessNode(LogsV2BodyPromotedColumn,
+				32, 1024),
+			true,
+		)
+		if err != nil {
+			return nil, err
+		}
+		plans = append(plans, node)
+	}
+
+	return plans, nil
+}

pkg/telemetrylogs/json_condition_builder.go

@@ -0,0 +1,454 @@
+package telemetrylogs
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	CodeCurrentNodeNil           = errors.MustNewCode("current_node_nil")
+	CodeNextNodeNil              = errors.MustNewCode("next_node_nil")
+	CodeNestedExpressionsEmpty   = errors.MustNewCode("nested_expressions_empty")
+	CodeGroupByPlanEmpty         = errors.MustNewCode("group_by_plan_empty")
+	CodeArrayMapExpressionsEmpty = errors.MustNewCode("array_map_expressions_empty")
+	CodePromotedPlanMissing      = errors.MustNewCode("promoted_plan_missing")
+)
+
+func (c *conditionBuilder) getTypes(ctx context.Context, path string) ([]telemetrytypes.JSONDataType, error) {
+	keys, _, err := c.metadataStore.GetKeys(ctx, &telemetrytypes.FieldKeySelector{
+		Name:              path,
+		SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+		Signal:            telemetrytypes.SignalLogs,
+		Limit:             1,
+	})
+	if err != nil {
+		return nil, err
+	}
+	types := []telemetrytypes.JSONDataType{}
+	for _, key := range keys[path] {
+		if key.JSONDataType != nil {
+			types = append(types, *key.JSONDataType)
+		}
+	}
+	return types, nil
+}
+
+// BuildCondition builds the full WHERE condition for body_json JSON paths
+func (c *conditionBuilder) buildJSONCondition(ctx context.Context, key *telemetrytypes.TelemetryFieldKey,
+	operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
+
+	plan, err := PlanJSON(ctx, key, operator, value, c.getTypes)
+	if err != nil {
+		return "", err
+	}
+
+	conditions := []string{}
+	for _, plan := range plan {
+		condition, err := c.emitPlannedCondition(plan, operator, value, sb)
+		if err != nil {
+			return "", err
+		}
+		conditions = append(conditions, condition)
+	}
+	return sb.Or(conditions...), nil
+}
+
+// emitPlannedCondition handles paths with array traversal
+func (c *conditionBuilder) emitPlannedCondition(plan *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
+	// Build traversal + terminal recursively per-hop
+	compiled, err := c.recurseArrayHops(plan, operator, value, sb)
+	if err != nil {
+		return "", err
+	}
+
+	// sb.AddWhereClause(sqlbuilder.NewWhereClause().AddWhereExpr(sb.Args, compiled))
+	return compiled, nil
+}
+
+// buildTerminalCondition creates the innermost condition
+func (c *conditionBuilder) buildTerminalCondition(node *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
+	// Use the parent's alias + current field name for the full path
+	fieldPath := node.FieldPath()
+
+	if node.TerminalConfig.ElemType.IsArray {
+		// switch operator for array membership checks
+		switch operator {
+		case qbtypes.FilterOperatorContains, qbtypes.FilterOperatorIn:
+			operator = qbtypes.FilterOperatorEqual
+		case qbtypes.FilterOperatorNotContains, qbtypes.FilterOperatorNotIn:
+			operator = qbtypes.FilterOperatorNotEqual
+		}
+		arrayCond, err := c.buildArrayMembershipCondition(node, operator, value, sb)
+		if err != nil {
+			return "", err
+		}
+		return arrayCond, nil
+	}
+	conditions := []string{}
+
+	elemType := node.TerminalConfig.ElemType
+	fieldExpr := fmt.Sprintf("dynamicElement(%s, '%s')", fieldPath, elemType.StringValue())
+	fieldExpr, value = querybuilder.DataTypeCollisionHandledFieldName(node.TerminalConfig.Key, value, fieldExpr, operator)
+
+	indexed := slices.ContainsFunc(node.TerminalConfig.Key.Indexes, func(index telemetrytypes.JSONDataTypeIndex) bool {
+		return index.Type == elemType && index.ColumnExpression == fieldPath
+	})
+	if elemType.IndexSupported && indexed {
+		indexedExpr := assumeNotNull(fieldPath, elemType)
+		emptyValue := func() any {
+			switch elemType {
+			case telemetrytypes.String:
+				return ""
+			case telemetrytypes.Int64, telemetrytypes.Float64, telemetrytypes.Bool:
+				return 0
+			default:
+				return nil
+			}
+		}()
+
+		// switch the operator and value for exists and not exists
+		switch operator {
+		case qbtypes.FilterOperatorExists:
+			operator = qbtypes.FilterOperatorNotEqual
+			value = emptyValue
+		case qbtypes.FilterOperatorNotExists:
+			operator = qbtypes.FilterOperatorEqual
+			value = emptyValue
+		default:
+			// do nothing
+		}
+
+		cond, err := c.applyOperator(sb, indexedExpr, operator, value)
+		if err != nil {
+			return "", err
+		}
+		conditions = append(conditions, cond)
+		// Switch operator to EXISTS
+		operator = qbtypes.FilterOperatorExists
+	}
+
+	cond, err := c.applyOperator(sb, fieldExpr, operator, value)
+	if err != nil {
+		return "", err
+	}
+	conditions = append(conditions, cond)
+	if len(conditions) > 1 {
+		return sb.And(conditions...), nil
+	}
+	return cond, nil
+}
+
+// buildArrayMembershipCondition handles array membership checks
+func (c *conditionBuilder) buildArrayMembershipCondition(node *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
+	arrayPath := node.FieldPath()
+
+	// create typed array out of a dynamic array
+	filteredDynamicExpr := func() string {
+		baseArrayDynamicExpr := fmt.Sprintf("dynamicElement(%s, 'Array(Dynamic)')", arrayPath)
+		return fmt.Sprintf("arrayMap(x->dynamicElement(x, '%s'), arrayFilter(x->(dynamicType(x) = '%s'), %s))",
+			node.TerminalConfig.ValueType.StringValue(),
+			node.TerminalConfig.ValueType.StringValue(),
+			baseArrayDynamicExpr)
+	}
+	typedArrayExpr := func() string {
+		return fmt.Sprintf("dynamicElement(%s, '%s')", arrayPath, node.TerminalConfig.ElemType.StringValue())
+	}
+
+	var arrayExpr string
+	if node.TerminalConfig.ElemType == telemetrytypes.ArrayDynamic {
+		arrayExpr = filteredDynamicExpr()
+	} else {
+		arrayExpr = typedArrayExpr()
+	}
+
+	fieldExpr, value := querybuilder.DataTypeCollisionHandledFieldName(node.TerminalConfig.Key, value, "x", operator)
+	op, err := c.applyOperator(sb, fieldExpr, operator, value)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("arrayExists(%s -> %s, %s)", fieldExpr, op, arrayExpr), nil
+}
+
+// recurseArrayHops recursively builds array traversal conditions
+func (c *conditionBuilder) recurseArrayHops(current *telemetrytypes.JSONAccessNode, operator qbtypes.FilterOperator, value any, sb *sqlbuilder.SelectBuilder) (string, error) {
+	if current == nil {
+		return "", fmt.Errorf("navigation failed, current node is nil")
+	}
+
+	if current.IsTerminal {
+		terminalCond, err := c.buildTerminalCondition(current, operator, value, sb)
+		if err != nil {
+			return "", err
+		}
+		return terminalCond, nil
+	}
+
+	currAlias := current.Alias()
+	fieldPath := current.FieldPath()
+	// Determine availability of Array(JSON) and Array(Dynamic) at this hop
+	hasArrayJSON := current.Branches[telemetrytypes.BranchJSON] != nil
+	hasArrayDynamic := current.Branches[telemetrytypes.BranchDynamic] != nil
+
+	// Then, at this hop, compute child per branch and wrap
+	branches := make([]string, 0, 2)
+	if hasArrayJSON {
+		jsonArrayExpr := fmt.Sprintf("dynamicElement(%s, 'Array(JSON(max_dynamic_types=%d, max_dynamic_paths=%d))')", fieldPath, current.MaxDynamicTypes, current.MaxDynamicPaths)
+		childGroupJSON, err := c.recurseArrayHops(current.Branches[telemetrytypes.BranchJSON], operator, value, sb)
+		if err != nil {
+			return "", err
+		}
+		branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", currAlias, childGroupJSON, jsonArrayExpr))
+	}
+	if hasArrayDynamic {
+		dynBaseExpr := fmt.Sprintf("dynamicElement(%s, 'Array(Dynamic)')", fieldPath)
+		dynFilteredExpr := fmt.Sprintf("arrayMap(x->dynamicElement(x, 'JSON'), arrayFilter(x->(dynamicType(x) = 'JSON'), %s))", dynBaseExpr)
+
+		// Create the Query for Dynamic array
+		childGroupDyn, err := c.recurseArrayHops(current.Branches[telemetrytypes.BranchDynamic], operator, value, sb)
+		if err != nil {
+			return "", err
+		}
+		branches = append(branches, fmt.Sprintf("arrayExists(%s-> %s, %s)", currAlias, childGroupDyn, dynFilteredExpr))
+	}
+
+	if len(branches) == 1 {
+		return branches[0], nil
+	}
+	return fmt.Sprintf("(%s)", strings.Join(branches, " OR ")), nil
+}
+
+func (c *conditionBuilder) applyOperator(sb *sqlbuilder.SelectBuilder, fieldExpr string, operator qbtypes.FilterOperator, value any) (string, error) {
+	switch operator {
+	case qbtypes.FilterOperatorEqual:
+		return sb.E(fieldExpr, value), nil
+	case qbtypes.FilterOperatorNotEqual:
+		return sb.NE(fieldExpr, value), nil
+	case qbtypes.FilterOperatorGreaterThan:
+		return sb.G(fieldExpr, value), nil
+	case qbtypes.FilterOperatorGreaterThanOrEq:
+		return sb.GE(fieldExpr, value), nil
+	case qbtypes.FilterOperatorLessThan:
+		return sb.LT(fieldExpr, value), nil
+	case qbtypes.FilterOperatorLessThanOrEq:
+		return sb.LE(fieldExpr, value), nil
+	case qbtypes.FilterOperatorLike:
+		return sb.Like(fieldExpr, value), nil
+	case qbtypes.FilterOperatorNotLike:
+		return sb.NotLike(fieldExpr, value), nil
+	case qbtypes.FilterOperatorILike:
+		return sb.ILike(fieldExpr, value), nil
+	case qbtypes.FilterOperatorNotILike:
+		return sb.NotILike(fieldExpr, value), nil
+	case qbtypes.FilterOperatorRegexp:
+		return fmt.Sprintf("match(%s, %s)", fieldExpr, sb.Var(value)), nil
+	case qbtypes.FilterOperatorNotRegexp:
+		return fmt.Sprintf("NOT match(%s, %s)", fieldExpr, sb.Var(value)), nil
+	case qbtypes.FilterOperatorContains:
+		return sb.ILike(fieldExpr, fmt.Sprintf("%%%v%%", value)), nil
+	case qbtypes.FilterOperatorNotContains:
+		return sb.NotILike(fieldExpr, fmt.Sprintf("%%%v%%", value)), nil
+	case qbtypes.FilterOperatorIn, qbtypes.FilterOperatorNotIn:
+		// emulate IN/NOT IN using OR/AND over equals to leverage indexes consistently
+		values, ok := value.([]any)
+		if !ok {
+			values = []any{value}
+		}
+		conds := []string{}
+		for _, v := range values {
+			if operator == qbtypes.FilterOperatorIn {
+				conds = append(conds, sb.E(fieldExpr, v))
+			} else {
+				conds = append(conds, sb.NE(fieldExpr, v))
+			}
+		}
+		if operator == qbtypes.FilterOperatorIn {
+			return sb.Or(conds...), nil
+		}
+		return sb.And(conds...), nil
+	case qbtypes.FilterOperatorExists:
+		return fmt.Sprintf("%s IS NOT NULL", fieldExpr), nil
+	case qbtypes.FilterOperatorNotExists:
+		return fmt.Sprintf("%s IS NULL", fieldExpr), nil
+	default:
+		return "", qbtypes.ErrUnsupportedOperator
+	}
+}
+
+// GroupByArrayJoinInfo contains information about array joins needed for GroupBy
+type GroupByArrayJoinInfo struct {
+	ArrayJoinClauses []string // ARRAY JOIN clauses to add to FROM clause
+	TerminalExpr     string   // Terminal field expression for SELECT/GROUP BY
+}
+
+// BuildGroupBy builds GroupBy information for body JSON fields using arrayConcat pattern
+func (c *conditionBuilder) BuildGroupBy(ctx context.Context, key *telemetrytypes.TelemetryFieldKey) (*GroupByArrayJoinInfo, error) {
+	path := strings.TrimPrefix(key.Name, telemetrytypes.BodyJSONStringSearchPrefix)
+
+	plan, err := PlanJSON(ctx, key, qbtypes.FilterOperatorExists, nil, c.getTypes)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(plan) == 0 {
+		return nil, errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput,
+			"Could not find any valid paths for: %s", path)
+	}
+
+	if plan[0].IsTerminal {
+		node := plan[0]
+
+		expr := fmt.Sprintf("dynamicElement(%s, '%s')", node.FieldPath(), node.TerminalConfig.ElemType.StringValue())
+		if key.Materialized {
+			if len(plan) < 2 {
+				return nil, errors.Newf(errors.TypeUnexpected, CodePromotedPlanMissing,
+					"plan length is less than 2 for promoted path: %s", path)
+			}
+
+			// promoted column first then body_json column
+			// TODO(Piyush): Change this in future for better performance
+			expr = fmt.Sprintf("coalesce(%s, %s)",
+				fmt.Sprintf("dynamicElement(%s, '%s')", plan[1].FieldPath(), plan[1].TerminalConfig.ElemType.StringValue()),
+				expr,
+			)
+		}
+
+		return &GroupByArrayJoinInfo{
+			ArrayJoinClauses: []string{},
+			TerminalExpr:     expr,
+		}, nil
+	}
+
+	// Build arrayConcat pattern directly from the tree structure
+	arrayConcatExpr, err := c.buildArrayConcat(plan)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create single ARRAY JOIN clause with arrayFlatten
+	arrayJoinClause := fmt.Sprintf("ARRAY JOIN %s AS `%s`", arrayConcatExpr, key.Name)
+
+	return &GroupByArrayJoinInfo{
+		ArrayJoinClauses: []string{arrayJoinClause},
+		TerminalExpr:     fmt.Sprintf("`%s`", key.Name),
+	}, nil
+}
+
+// buildArrayConcat builds the arrayConcat pattern directly from the tree structure
+func (c *conditionBuilder) buildArrayConcat(plan telemetrytypes.JSONAccessPlan) (string, error) {
+	if len(plan) == 0 {
+		return "", errors.Newf(errors.TypeInternal, CodeGroupByPlanEmpty, "group by plan is empty while building arrayConcat")
+	}
+
+	// Build arrayMap expressions for ALL available branches at the root level
+	var arrayMapExpressions []string
+	for _, node := range plan {
+		hasJSON := node.Branches[telemetrytypes.BranchJSON] != nil
+		hasDynamic := node.Branches[telemetrytypes.BranchDynamic] != nil
+
+		if hasJSON {
+			jsonExpr, err := c.buildArrayMap(node, telemetrytypes.BranchJSON)
+			if err != nil {
+				return "", err
+			}
+			arrayMapExpressions = append(arrayMapExpressions, jsonExpr)
+		}
+
+		if hasDynamic {
+			dynamicExpr, err := c.buildArrayMap(node, telemetrytypes.BranchDynamic)
+			if err != nil {
+				return "", err
+			}
+			arrayMapExpressions = append(arrayMapExpressions, dynamicExpr)
+		}
+	}
+	if len(arrayMapExpressions) == 0 {
+		return "", errors.Newf(errors.TypeInternal, CodeArrayMapExpressionsEmpty, "array map expressions are empty while building arrayConcat")
+	}
+
+	// Build the arrayConcat expression
+	arrayConcatExpr := fmt.Sprintf("arrayConcat(%s)", strings.Join(arrayMapExpressions, ", "))
+
+	// Wrap with arrayFlatten
+	arrayFlattenExpr := fmt.Sprintf("arrayFlatten(%s)", arrayConcatExpr)
+
+	return arrayFlattenExpr, nil
+}
+
+// buildArrayMap builds the arrayMap expression for a specific branch, handling all sub-branches
+func (c *conditionBuilder) buildArrayMap(currentNode *telemetrytypes.JSONAccessNode, branchType telemetrytypes.JSONAccessBranchType) (string, error) {
+	if currentNode == nil {
+		return "", errors.Newf(errors.TypeInternal, CodeCurrentNodeNil, "current node is nil while building arrayMap")
+	}
+
+	nextNode := currentNode.Branches[branchType]
+	if nextNode == nil {
+		return "", errors.Newf(errors.TypeInternal, CodeNextNodeNil, "next node is nil while building arrayMap")
+	}
+
+	// Build the array expression for this level
+	var arrayExpr string
+	if branchType == telemetrytypes.BranchJSON {
+		// Array(JSON) branch
+		arrayExpr = fmt.Sprintf("dynamicElement(%s, 'Array(JSON(max_dynamic_types=%d, max_dynamic_paths=%d))')",
+			currentNode.FieldPath(), currentNode.MaxDynamicTypes, currentNode.MaxDynamicPaths)
+	} else {
+		// Array(Dynamic) branch - filter for JSON objects
+		dynBaseExpr := fmt.Sprintf("dynamicElement(%s, 'Array(Dynamic)')", currentNode.FieldPath())
+		arrayExpr = fmt.Sprintf("arrayMap(x->assumeNotNull(dynamicElement(x, 'JSON')), arrayFilter(x->(dynamicType(x) = 'JSON'), %s))", dynBaseExpr)
+	}
+
+	// If this is the terminal level, return the simple arrayMap
+	if nextNode.IsTerminal {
+		dynamicElementExpr := fmt.Sprintf("dynamicElement(%s, '%s')", nextNode.FieldPath(),
+			nextNode.TerminalConfig.ElemType.StringValue(),
+		)
+		return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), dynamicElementExpr, arrayExpr), nil
+	}
+
+	// For non-terminal nodes, we need to handle ALL possible branches at the next level
+	var nestedExpressions []string
+	hasJSON := nextNode.Branches[telemetrytypes.BranchJSON] != nil
+	hasDynamic := nextNode.Branches[telemetrytypes.BranchDynamic] != nil
+
+	if hasJSON {
+		jsonNested, err := c.buildArrayMap(nextNode, telemetrytypes.BranchJSON)
+		if err != nil {
+			return "", err
+		}
+		nestedExpressions = append(nestedExpressions, jsonNested)
+	}
+
+	if hasDynamic {
+		dynamicNested, err := c.buildArrayMap(nextNode, telemetrytypes.BranchDynamic)
+		if err != nil {
+			return "", err
+		}
+		nestedExpressions = append(nestedExpressions, dynamicNested)
+	}
+
+	// If we have multiple nested expressions, we need to concat them
+	var nestedExpr string
+	if len(nestedExpressions) == 1 {
+		nestedExpr = nestedExpressions[0]
+	} else if len(nestedExpressions) > 1 {
+		// This shouldn't happen in our current tree structure, but handle it just in case
+		nestedExpr = fmt.Sprintf("arrayConcat(%s)", strings.Join(nestedExpressions, ", "))
+	} else {
+		return "", errors.Newf(errors.TypeInternal, CodeNestedExpressionsEmpty, "nested expressions are empty while building arrayMap")
+	}
+
+	return fmt.Sprintf("arrayMap(%s->%s, %s)", currentNode.Alias(), nestedExpr, arrayExpr), nil
+}
+
+func assumeNotNull(column string, elemType telemetrytypes.JSONDataType) string {
+	return fmt.Sprintf("assumeNotNull(dynamicElement(%s, '%s'))", column, elemType.StringValue())
+}

pkg/telemetrylogs/json_string.go

@@ -11,7 +11,7 @@ import (
 	"github.com/SigNoz/signoz/pkg/valuer"
 )
 
-func parseStrValue(valueStr string, operator qbtypes.FilterOperator) (telemetrytypes.FieldDataType, any) {
+func ParseStrValue(valueStr string, operator qbtypes.FilterOperator) (telemetrytypes.FieldDataType, any) {
 
 	valueType := telemetrytypes.FieldDataTypeString
 
@@ -55,7 +55,7 @@ func inferDataType(value any, operator qbtypes.FilterOperator, key *telemetrytyp
 	case float32, float64:
 		valueType = telemetrytypes.FieldDataTypeFloat64
 	case string:
-		valueType, value = parseStrValue(v, operator)
+		valueType, value = ParseStrValue(v, operator)
 	case bool:
 		valueType = telemetrytypes.FieldDataTypeBool
 	}
@@ -69,7 +69,7 @@ func inferDataType(value any, operator qbtypes.FilterOperator, key *telemetrytyp
 }
 
 func getBodyJSONPath(key *telemetrytypes.TelemetryFieldKey) string {
-	parts := strings.Split(key.Name, ".")[1:]
+	parts := strings.Split(key.Name, ".")
 	newParts := []string{}
 	for _, part := range parts {
 		if strings.HasSuffix(part, "[*]") {
@@ -84,7 +84,6 @@ func getBodyJSONPath(key *telemetrytypes.TelemetryFieldKey) string {
 }
 
 func GetBodyJSONKey(_ context.Context, key *telemetrytypes.TelemetryFieldKey, operator qbtypes.FilterOperator, value any) (string, any) {
-
 	dataType, value := inferDataType(value, operator, key)
 
 	// for array types, we need to extract the value from the JSON_QUERY

pkg/telemetrylogs/queries

@@ -0,0 +1,186 @@
+SELECT timestamp,
+    id,
+    body_json,
+    body_json_promoted
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.customer.profile`->(
+                dynamicElement(`body_json.customer.profile`.header, 'String') != ''
+                OR dynamicElement(
+                    `body_json.customer.profile`.header,
+                    'Array(Nullable(String))'
+                ) != []
+            ),
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );
+SELECT timestamp,
+    id,
+    body_json,
+    body_json_promoted
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.customer.profile`->(
+                length(
+                    dynamicElement(
+                        `body_json.customer.profile`.header,
+                        'Array(Nullable(Float64))'
+                    )
+                ) > 0
+            ),
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );
+SELECT timestamp,
+    id,
+    trace_id,
+    span_id,
+    trace_flags,
+    severity_text,
+    severity_number,
+    scope_name,
+    scope_version,
+    body,
+    body_json,
+    body_json_promoted,
+    attributes_string,
+    attributes_number,
+    attributes_bool,
+    resources_string,
+    scope_string
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.customer.profile`->arrayExists(
+                x->x = '200',
+                dynamicElement(
+                    `body_json.customer.profile`.header,
+                    'Array(Nullable(String))'
+                )
+            ),
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );
+SELECT timestamp,
+    id,
+    trace_id,
+    span_id,
+    trace_flags,
+    severity_text,
+    severity_number,
+    scope_name,
+    scope_version,
+    body,
+    body_json,
+    body_json_promoted,
+    attributes_string,
+    attributes_number,
+    attributes_bool,
+    resources_string,
+    scope_string
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.customer.profile`->arrayExists(
+                x->x = '200',
+                dynamicElement(
+                    `body_json.customer.profile`.header,
+                    'Array(Nullable(String))'
+                )
+            ),
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    )
+    OR (
+        arrayExists(
+            `body_json.customer.profile`->dynamicElement(
+                `body_json.customer.profile`.header,
+                'String'
+            ) = '403',
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );
+Explain SELECT timestamp,
+    id,
+    trace_id,
+    span_id,
+    trace_flags,
+    severity_text,
+    severity_number,
+    scope_name,
+    scope_version,
+    body,
+    body_json,
+    body_json_promoted,
+    attributes_string,
+    attributes_number,
+    attributes_bool,
+    resources_string,
+    scope_string
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.customer.profile`->arrayExists(
+                x->x = '200',
+                dynamicElement(
+                    `body_json.customer.profile`.header,
+                    'Array(Nullable(String))'
+                )
+            )
+            OR dynamicElement(
+                `body_json.customer.profile`.header,
+                'String'
+            ) = '403',
+            dynamicElement(
+                body_json.customer.profile,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );
+SELECT timestamp,
+    id,
+    trace_id,
+    span_id,
+    trace_flags,
+    severity_text,
+    severity_number,
+    scope_name,
+    scope_version,
+    body,
+    body_json,
+    body_json_promoted,
+    attributes_string,
+    attributes_number,
+    attributes_bool,
+    resources_string,
+    scope_string
+FROM signoz_logs.distributed_logs_v2
+WHERE (
+        arrayExists(
+            `body_json.cache.persistence.retrieve`->dynamicElement(
+                `body_json.cache.persistence.retrieve`.settings,
+                'String'
+            ) = '503',
+            dynamicElement(
+                body_json.cache.persistence.retrieve,
+                'Array(JSON(max_dynamic_types=16, max_dynamic_paths=0))'
+            )
+        )
+    );

pkg/telemetrylogs/statement_builder.go

@@ -23,7 +23,6 @@ type logQueryStatementBuilder struct {
 	aggExprRewriter           qbtypes.AggExprRewriter
 
 	fullTextColumn *telemetrytypes.TelemetryFieldKey
-	jsonBodyPrefix string
 	jsonKeyToKey   qbtypes.JsonKeyToFieldFunc
 }
 
@@ -37,7 +36,6 @@ func NewLogQueryStatementBuilder(
 	resourceFilterStmtBuilder qbtypes.StatementBuilder[qbtypes.LogAggregation],
 	aggExprRewriter qbtypes.AggExprRewriter,
 	fullTextColumn *telemetrytypes.TelemetryFieldKey,
-	jsonBodyPrefix string,
 	jsonKeyToKey qbtypes.JsonKeyToFieldFunc,
 ) *logQueryStatementBuilder {
 	logsSettings := factory.NewScopedProviderSettings(settings, "github.com/SigNoz/signoz/pkg/telemetrylogs")
@@ -50,7 +48,6 @@ func NewLogQueryStatementBuilder(
 		resourceFilterStmtBuilder: resourceFilterStmtBuilder,
 		aggExprRewriter:           aggExprRewriter,
 		fullTextColumn:            fullTextColumn,
-		jsonBodyPrefix:            jsonBodyPrefix,
 		jsonKeyToKey:              jsonKeyToKey,
 	}
 }
@@ -171,6 +168,25 @@ func (b *logQueryStatementBuilder) adjustKeys(ctx context.Context, keys map[stri
 			overallMatch = overallMatch || findMatch(IntrinsicFields)
 		}
 
+		if strings.Contains(k.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
+			k.Name = strings.TrimPrefix(k.Name, telemetrytypes.BodyJSONStringSearchPrefix)
+			fieldKeys, found := keys[k.Name]
+			if found && len(fieldKeys) > 0 {
+				k.FieldContext = fieldKeys[0].FieldContext
+				k.FieldDataType = fieldKeys[0].FieldDataType
+				k.Materialized = fieldKeys[0].Materialized
+				k.JSONDataType = fieldKeys[0].JSONDataType
+				k.Indexes = fieldKeys[0].Indexes
+
+				overallMatch = true // because we found a match
+			} else {
+				b.logger.InfoContext(ctx, "overriding the field context and data type", "key", k.Name)
+				k.FieldContext = telemetrytypes.FieldContextBody
+				k.FieldDataType = telemetrytypes.FieldDataTypeString
+				k.JSONDataType = &telemetrytypes.String
+			}
+		}
+
 		if !overallMatch {
 			// check if all the key for the given field have been materialized, if so
 			// set the key to materialized
@@ -221,6 +237,9 @@ func (b *logQueryStatementBuilder) buildListQuery(
 		cteArgs = append(cteArgs, args)
 	}
 
+	// Collect array join info for body JSON fields
+	var arrayJoinClauses []string
+
 	// Select timestamp and id by default
 	sb.Select(LogsV2TimestampColumn)
 	sb.SelectMore(LogsV2IDColumn)
@@ -234,6 +253,8 @@ func (b *logQueryStatementBuilder) buildListQuery(
 		sb.SelectMore(LogsV2ScopeNameColumn)
 		sb.SelectMore(LogsV2ScopeVersionColumn)
 		sb.SelectMore(LogsV2BodyColumn)
+		sb.SelectMore(LogsV2BodyJSONColumn)
+		sb.SelectMore(LogsV2BodyPromotedColumn)
 		sb.SelectMore(LogsV2AttributesStringColumn)
 		sb.SelectMore(LogsV2AttributesNumberColumn)
 		sb.SelectMore(LogsV2AttributesBoolColumn)
@@ -246,6 +267,7 @@ func (b *logQueryStatementBuilder) buildListQuery(
 			if query.SelectFields[index].Name == LogsV2TimestampColumn || query.SelectFields[index].Name == LogsV2IDColumn {
 				continue
 			}
+
 			// get column expression for the field - use array index directly to avoid pointer to loop variable
 			colExpr, err := b.fm.ColumnExpressionFor(ctx, &query.SelectFields[index], keys)
 			if err != nil {
@@ -255,8 +277,12 @@ func (b *logQueryStatementBuilder) buildListQuery(
 		}
 	}
 
-	// From table
-	sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
+	// From table (inject ARRAY JOINs if collected)
+	fromBase := fmt.Sprintf("%s.%s", DBName, LogsV2TableName)
+	if len(arrayJoinClauses) > 0 {
+		fromBase = fromBase + " " + strings.Join(arrayJoinClauses, " ")
+	}
+	sb.From(fromBase)
 
 	// Add filter conditions
 	preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
@@ -330,13 +356,17 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
 
 	var allGroupByArgs []any
 
+	// Collect array join info for body JSON fields
+	var arrayJoinClauses []string
+
 	// Keep original column expressions so we can build the tuple
 	fieldNames := make([]string, 0, len(query.GroupBy))
 	for _, gb := range query.GroupBy {
-		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonBodyPrefix, b.jsonKeyToKey)
+		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
 		if err != nil {
 			return nil, err
 		}
+
 		colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name)
 		allGroupByArgs = append(allGroupByArgs, args...)
 		sb.SelectMore(colExpr)
@@ -358,7 +388,13 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
 		sb.SelectMore(fmt.Sprintf("%s AS __result_%d", rewritten, i))
 	}
 
-	sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
+	// Add FROM clause
+	fromBase := fmt.Sprintf("%s.%s", DBName, LogsV2TableName)
+	if len(arrayJoinClauses) > 0 {
+		fromBase = fromBase + " " + strings.Join(arrayJoinClauses, " ")
+	}
+	sb.From(fromBase)
+
 	preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
 
 	if err != nil {
@@ -404,7 +440,6 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
 		}
 
 		combinedArgs := append(allGroupByArgs, allAggChArgs...)
-
 		mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...)
 
 		// Stitch it all together:  WITH … SELECT …
@@ -431,7 +466,6 @@ func (b *logQueryStatementBuilder) buildTimeSeriesQuery(
 		}
 
 		combinedArgs := append(allGroupByArgs, allAggChArgs...)
-
 		mainSQL, mainArgs := sb.BuildWithFlavor(sqlbuilder.ClickHouse, combinedArgs...)
 
 		// Stitch it all together:  WITH … SELECT …
@@ -478,11 +512,15 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
 
 	var allGroupByArgs []any
 
+	// Collect array join info for body JSON fields
+	var arrayJoinClauses []string
+
 	for _, gb := range query.GroupBy {
-		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonBodyPrefix, b.jsonKeyToKey)
+		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, b.jsonKeyToKey)
 		if err != nil {
 			return nil, err
 		}
+
 		colExpr := fmt.Sprintf("toString(%s) AS `%s`", expr, gb.TelemetryFieldKey.Name)
 		allGroupByArgs = append(allGroupByArgs, args...)
 		sb.SelectMore(colExpr)
@@ -508,8 +546,12 @@ func (b *logQueryStatementBuilder) buildScalarQuery(
 		}
 	}
 
-	// From table
-	sb.From(fmt.Sprintf("%s.%s", DBName, LogsV2TableName))
+	// From table (inject ARRAY JOINs if collected)
+	fromBase := fmt.Sprintf("%s.%s", DBName, LogsV2TableName)
+	if len(arrayJoinClauses) > 0 {
+		fromBase = fromBase + " " + strings.Join(arrayJoinClauses, " ")
+	}
+	sb.From(fromBase)
 
 	// Add filter conditions
 	preparedWhereClause, err := b.addFilterCondition(ctx, sb, start, end, query, keys, variables)
@@ -589,7 +631,6 @@ func (b *logQueryStatementBuilder) addFilterCondition(
 			FieldKeys:          keys,
 			SkipResourceFilter: true,
 			FullTextColumn:     b.fullTextColumn,
-			JsonBodyPrefix:     b.jsonBodyPrefix,
 			JsonKeyToKey:       b.jsonKeyToKey,
 			Variables:          variables,
 		}, start, end)
@@ -655,7 +696,6 @@ func (b *logQueryStatementBuilder) buildResourceFilterCTE(
 	start, end uint64,
 	variables map[string]qbtypes.VariableItem,
 ) (*qbtypes.Statement, error) {
-
 	return b.resourceFilterStmtBuilder.Build(
 		ctx,
 		start,

pkg/telemetrylogs/stmt_builder_test.go

@@ -32,7 +32,6 @@ func resourceFilterStmtBuilder() qbtypes.StatementBuilder[qbtypes.LogAggregation
 		cb,
 		mockMetadataStore,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 }
@@ -175,11 +174,11 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	cb := NewConditionBuilder(fm, mockMetadataStore)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -191,7 +190,6 @@ func TestStatementBuilderTimeSeries(t *testing.T) {
 		resourceFilterStmtBuilder,
 		aggExprRewriter,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 
@@ -232,7 +230,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 				Limit: 10,
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
 				Args:  []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -260,7 +258,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND true AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
 				Args:  []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -268,11 +266,11 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	cb := NewConditionBuilder(fm, mockMetadataStore)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -284,7 +282,6 @@ func TestStatementBuilderListQuery(t *testing.T) {
 		resourceFilterStmtBuilder,
 		aggExprRewriter,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 
@@ -325,7 +322,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 				Limit: 10,
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND match(LOWER(body), LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
 				Args:  []any{uint64(1747945619), uint64(1747983448), "hello", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -353,7 +350,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (match(LOWER(body), LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (match(LOWER(body), LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
 				Args:  []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "hello", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -369,7 +366,7 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 				Limit: 10,
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (JSON_VALUE(body, '$.\"status\"') = ? AND JSON_EXISTS(body, '$.\"status\"')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE true AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (JSON_VALUE(body, '$.\"status\"') = ? AND JSON_EXISTS(body, '$.\"status\"')) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
 				Args:  []any{uint64(1747945619), uint64(1747983448), "success", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -377,11 +374,11 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	cb := NewConditionBuilder(fm, mockMetadataStore)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -393,10 +390,11 @@ func TestStatementBuilderListQueryResourceTests(t *testing.T) {
 		resourceFilterStmtBuilder,
 		aggExprRewriter,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 
+	//
+
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 
@@ -451,11 +449,11 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
+	cb := NewConditionBuilder(fm, mockMetadataStore)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -467,7 +465,6 @@ func TestStatementBuilderTimeSeriesBodyGroupBy(t *testing.T) {
 		resourceFilterStmtBuilder,
 		aggExprRewriter,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 
@@ -509,7 +506,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
 				Limit: 10,
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND true)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((LOWER(body) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE (((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND true)) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND ((LOWER(body) LIKE LOWER(?))) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? LIMIT ?",
 				Args:  []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -538,7 +535,7 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
 				},
 			},
 			expected: qbtypes.Statement{
-				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(body) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
+				Query: "WITH __resource_filter AS (SELECT fingerprint FROM signoz_logs.distributed_logs_v2_resource WHERE ((simpleJSONExtractString(labels, 'service.name') = ? AND labels LIKE ? AND labels LIKE ?) AND true) AND seen_at_ts_bucket_start >= ? AND seen_at_ts_bucket_start <= ?) SELECT timestamp, id, trace_id, span_id, trace_flags, severity_text, severity_number, scope_name, scope_version, body, body_json, body_json_promoted, attributes_string, attributes_number, attributes_bool, resources_string, scope_string FROM signoz_logs.distributed_logs_v2 WHERE resource_fingerprint GLOBAL IN (SELECT fingerprint FROM __resource_filter) AND (LOWER(body) LIKE LOWER(?)) AND timestamp >= ? AND ts_bucket_start >= ? AND timestamp < ? AND ts_bucket_start <= ? ORDER BY `attribute_string_materialized$$key$$name` AS `materialized.key.name` desc LIMIT ?",
 				Args:  []any{"cartservice", "%service.name%", "%service.name\":\"cartservice%", uint64(1747945619), uint64(1747983448), "%error%", "1747947419000000000", uint64(1747945619), "1747983448000000000", uint64(1747983448), 10},
 			},
 			expectedErr: nil,
@@ -547,11 +544,11 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
 	}
 
 	fm := NewFieldMapper()
-	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMapCollision()
+	cb := NewConditionBuilder(fm, mockMetadataStore)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -563,7 +560,6 @@ func TestStatementBuilderListQueryServiceCollision(t *testing.T) {
 		resourceFilterStmtBuilder,
 		aggExprRewriter,
 		DefaultFullTextColumn,
-		BodyJSONStringSearchPrefix,
 		GetBodyJSONKey,
 	)
 

pkg/telemetrymetadata/body_json_metadata.go

@@ -0,0 +1,434 @@
+package telemetrymetadata
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/chcol"
+	schemamigrator "github.com/SigNoz/signoz-otel-collector/cmd/signozschemamigrator/schema_migrator"
+	"github.com/SigNoz/signoz-otel-collector/constants"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/querybuilder"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/telemetrystore"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/huandu/go-sqlbuilder"
+)
+
+var (
+	defaultPathLimit = 100 // Default limit to prevent full table scans
+
+	CodeUnknownJSONDataType     = errors.MustNewCode("unknown_json_data_type")
+	CodeFailLoadPromotedPaths   = errors.MustNewCode("fail_load_promoted_paths")
+	CodeFailCheckPathPromoted   = errors.MustNewCode("fail_check_path_promoted")
+	CodeFailIterateBodyJSONKeys = errors.MustNewCode("fail_iterate_body_json_keys")
+	CodeFailExtractBodyJSONKeys = errors.MustNewCode("fail_extract_body_json_keys")
+	CodeFailLoadLogsJSONIndexes = errors.MustNewCode("fail_load_logs_json_indexes")
+	CodeFailListJSONValues      = errors.MustNewCode("fail_list_json_values")
+	CodeFailScanJSONValue       = errors.MustNewCode("fail_scan_json_value")
+	CodeFailScanVariant         = errors.MustNewCode("fail_scan_variant")
+	CodeFailBuildJSONPathsQuery = errors.MustNewCode("fail_build_json_paths_query")
+)
+
+// GetBodyJSONPaths extracts body JSON paths from the path_types table
+// This function can be used by both JSONQueryBuilder and metadata extraction
+// uniquePathLimit: 0 for no limit, >0 for maximum number of unique paths to return
+//   - For startup load: set to 10000 to get top 10k unique paths
+//   - For lookup: set to 0 (no limit needed for single path)
+//   - For metadata API: set to desired pagination limit
+//
+// searchOperator: LIKE for pattern matching, EQUAL for exact match
+// Returns: (paths, error)
+func getBodyJSONPaths(ctx context.Context, telemetryStore telemetrystore.TelemetryStore,
+	fieldKeySelectors []*telemetrytypes.FieldKeySelector) ([]*telemetrytypes.TelemetryFieldKey, bool, error) {
+
+	query, args, limit, err := buildGetBodyJSONPathsQuery(fieldKeySelectors)
+	if err != nil {
+		return nil, false, err
+	}
+
+	rows, err := telemetryStore.ClickhouseDB().Query(ctx, query, args...)
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, CodeFailExtractBodyJSONKeys, "failed to extract body JSON keys")
+	}
+	defer rows.Close()
+
+	paths := []*telemetrytypes.TelemetryFieldKey{}
+	rowCount := 0
+	for rows.Next() {
+		var path string
+		var typesArray []string // ClickHouse returns array as []string
+		var lastSeen uint64
+
+		err = rows.Scan(&path, &typesArray, &lastSeen)
+		if err != nil {
+			return nil, false, errors.WrapInternalf(err, CodeFailExtractBodyJSONKeys, "failed to scan body JSON key row")
+		}
+
+		promoted, err := IsPathPromoted(ctx, telemetryStore.ClickhouseDB(), path)
+		if err != nil {
+			return nil, false, err
+		}
+
+		indexes, err := listLogsIndexesClean(ctx, telemetryStore.ClickhouseDB(), telemetryStore.Cluster(), path)
+		if err != nil {
+			return nil, false, err
+		}
+
+		for _, typ := range typesArray {
+			mapping, found := telemetrytypes.MappingStringToJSONDataType[typ]
+			if !found {
+				return nil, false, errors.NewInternalf(CodeUnknownJSONDataType, "failed to map type string to JSON data type: %s", typ)
+			}
+			paths = append(paths, &telemetrytypes.TelemetryFieldKey{
+				Name:          path,
+				Signal:        telemetrytypes.SignalLogs,
+				FieldContext:  telemetrytypes.FieldContextBody,
+				FieldDataType: telemetrytypes.MappingJSONDataTypeToFieldDataType[mapping],
+				JSONDataType:  &mapping,
+				Indexes:       indexes,
+				Materialized:  promoted,
+			})
+		}
+
+		rowCount++
+	}
+
+	if rows.Err() != nil {
+		return nil, false, errors.WrapInternalf(rows.Err(), CodeFailIterateBodyJSONKeys, "error iterating body JSON keys")
+	}
+
+	return paths, rowCount <= limit, nil
+}
+
+func buildGetBodyJSONPathsQuery(fieldKeySelectors []*telemetrytypes.FieldKeySelector) (string, []any, int, error) {
+	if len(fieldKeySelectors) == 0 {
+		return "", nil, defaultPathLimit, errors.NewInternalf(CodeFailBuildJSONPathsQuery, "no field key selectors provided")
+	}
+	from := fmt.Sprintf("%s.%s", DBName, PathTypesTableName)
+
+	// Build a better query using GROUP BY to deduplicate at database level
+	// This aggregates all types per path and gets the max last_seen, then applies LIMIT
+	sb := sqlbuilder.Select(
+		"path",
+		"groupArray(DISTINCT type) AS types",
+		"max(last_seen) AS last_seen",
+	).From(from)
+
+	limit := 0
+	// Add search filter if provided
+	orClauses := []string{}
+	for _, fieldKeySelector := range fieldKeySelectors {
+		// replace [*] with []
+		fieldKeySelector.Name = strings.ReplaceAll(fieldKeySelector.Name, telemetrylogs.ArrayAnyIndex, telemetrylogs.ArraySep)
+		// Extract search text for body JSON keys
+		keyName := CleanPathPrefixes(fieldKeySelector.Name)
+		if fieldKeySelector.SelectorMatchType == telemetrytypes.FieldSelectorMatchTypeExact {
+			orClauses = append(orClauses, sb.Equal("path", keyName))
+		} else {
+			// Pattern matching for metadata API (defaults to LIKE behavior for other operators)
+			orClauses = append(orClauses, sb.Like("path", querybuilder.FormatValueForContains(keyName)))
+			limit += fieldKeySelector.Limit
+		}
+	}
+	sb.Where(sb.Or(orClauses...))
+
+	// Group by path to get unique paths with aggregated types
+	sb.GroupBy("path")
+
+	// Order by max last_seen to get most recent paths first
+	sb.OrderBy("last_seen DESC")
+	if limit == 0 {
+		limit = defaultPathLimit
+	}
+	sb.Limit(limit)
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	return query, args, limit, nil
+}
+
+func listLogsIndexesClean(ctx context.Context, conn clickhouse.Conn, cluster, path string) ([]telemetrytypes.JSONDataTypeIndex, error) {
+	// return empty slice if path is an array
+	if strings.Contains(path, telemetrylogs.ArraySep) || strings.Contains(path, telemetrylogs.ArrayAnyIndex) {
+		return nil, nil
+	}
+
+	// list indexes for the path
+	indexes, err := ListLogsJSONIndexes(ctx, cluster, conn, path)
+	if err != nil {
+		return nil, err
+	}
+
+	// build a set of indexes
+	cleanIndexes := []telemetrytypes.JSONDataTypeIndex{}
+	for _, index := range indexes {
+		columnExpr, columnType, err := schemamigrator.UnfoldJSONSubColumnIndexExpr(index.Expression)
+		if err != nil {
+			return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to unfold JSON sub column index expression: %s", index.Expression)
+		}
+
+		jsonDataType, found := telemetrytypes.MappingStringToJSONDataType[columnType]
+		if !found {
+			return nil, errors.NewInternalf(CodeUnknownJSONDataType, "failed to map column type to JSON data type: %s", columnType)
+		}
+
+		if jsonDataType == telemetrytypes.String {
+			cleanIndexes = append(cleanIndexes, telemetrytypes.JSONDataTypeIndex{
+				Type:             telemetrytypes.String,
+				ColumnExpression: columnExpr,
+				IndexExpression:  index.Expression,
+			})
+		} else if strings.HasPrefix(index.Type, "minmax") {
+			cleanIndexes = append(cleanIndexes, telemetrytypes.JSONDataTypeIndex{
+				Type:             jsonDataType,
+				ColumnExpression: columnExpr,
+				IndexExpression:  index.Expression,
+			})
+		}
+	}
+
+	return cleanIndexes, nil
+}
+
+func buildListLogsJSONIndexesQuery(cluster string, filters ...string) (string, []any) {
+	// Build a better query using GROUP BY to deduplicate at database level
+	// This aggregates all types per path and gets the max last_seen, then applies LIMIT
+	sb := sqlbuilder.Select(
+		"name", "type_full", "expr", "granularity",
+	).From(fmt.Sprintf("clusterAllReplicas('%s', %s)", cluster, SkipIndexTableName))
+
+	sb.Where(sb.Equal("database", telemetrylogs.DBName))
+	sb.Where(sb.Equal("table", telemetrylogs.LogsV2LocalTableName))
+	sb.Where(sb.Or(
+		sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyJSONColumnPrefix))),
+		sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(constants.BodyPromotedColumnPrefix))),
+	))
+
+	filterExprs := []string{}
+	for _, filter := range filters {
+		filterExprs = append(filterExprs, sb.ILike("expr", fmt.Sprintf("%%%s%%", querybuilder.FormatValueForContains(filter))))
+	}
+	sb.Where(sb.Or(filterExprs...))
+
+	return sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+}
+
+func ListLogsJSONIndexes(ctx context.Context, cluster string, conn clickhouse.Conn, filters ...string) ([]schemamigrator.Index, error) {
+	query, args := buildListLogsJSONIndexesQuery(cluster, filters...)
+	rows, err := conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to load string indexed columns")
+	}
+	defer rows.Close()
+
+	indexes := []schemamigrator.Index{}
+	for rows.Next() {
+		var name string
+		var typeFull string
+		var expr string
+		var granularity uint64
+		if err := rows.Scan(&name, &typeFull, &expr, &granularity); err != nil {
+			return nil, errors.WrapInternalf(err, CodeFailLoadLogsJSONIndexes, "failed to scan string indexed column")
+		}
+		indexes = append(indexes, schemamigrator.Index{
+			Name:        name,
+			Type:        typeFull,
+			Expression:  expr,
+			Granularity: int(granularity),
+		})
+	}
+
+	return indexes, nil
+}
+
+func ListPromotedPaths(ctx context.Context, conn clickhouse.Conn) (map[string]struct{}, error) {
+	query := fmt.Sprintf("SELECT path FROM %s.%s", DBName, PromotedPathsTableName)
+	rows, err := conn.Query(ctx, query)
+	if err != nil {
+		return nil, errors.WrapInternalf(err, CodeFailLoadPromotedPaths, "failed to load promoted paths")
+	}
+	defer rows.Close()
+
+	next := make(map[string]struct{})
+	for rows.Next() {
+		var path string
+		if err := rows.Scan(&path); err != nil {
+			return nil, errors.WrapInternalf(err, CodeFailLoadPromotedPaths, "failed to scan promoted path")
+		}
+		next[path] = struct{}{}
+	}
+
+	return next, nil
+}
+
+func ListJSONValues(ctx context.Context, conn clickhouse.Conn, path string, limit int) (*telemetrytypes.TelemetryFieldValues, bool, error) {
+	path = CleanPathPrefixes(path)
+
+	if strings.Contains(path, telemetrylogs.ArraySep) || strings.Contains(path, telemetrylogs.ArrayAnyIndex) {
+		return nil, false, errors.NewInvalidInputf(errors.CodeInvalidInput, "array paths are not supported")
+	}
+
+	promoted, err := IsPathPromoted(ctx, conn, path)
+	if err != nil {
+		return nil, false, err
+	}
+
+	if promoted {
+		path = telemetrylogs.BodyPromotedColumnPrefix + path
+	} else {
+		path = telemetrylogs.BodyJSONColumnPrefix + path
+	}
+
+	from := fmt.Sprintf("%s.%s", telemetrylogs.DBName, telemetrylogs.LogsV2TableName)
+	colExpr := func(typ telemetrytypes.JSONDataType) string {
+		return fmt.Sprintf("dynamicElement(%s, '%s')", path, typ.StringValue())
+	}
+
+	sb := sqlbuilder.Select(
+		colExpr(telemetrytypes.String),
+		colExpr(telemetrytypes.Int64),
+		colExpr(telemetrytypes.Float64),
+		colExpr(telemetrytypes.Bool),
+		colExpr(telemetrytypes.ArrayString),
+		colExpr(telemetrytypes.ArrayInt64),
+		colExpr(telemetrytypes.ArrayFloat64),
+		colExpr(telemetrytypes.ArrayBool),
+		colExpr(telemetrytypes.ArrayDynamic),
+	).From(from)
+	sb.Where(fmt.Sprintf("%s IS NOT NULL", path))
+	sb.Limit(limit)
+
+	query, args := sb.BuildWithFlavor(sqlbuilder.ClickHouse)
+	rows, err := conn.Query(ctx, query, args...)
+	if err != nil {
+		return nil, false, errors.WrapInternalf(err, CodeFailListJSONValues, "failed to list JSON values")
+	}
+	defer rows.Close()
+
+	// Get column types to determine proper scan types
+	colTypes := rows.ColumnTypes()
+	scanTargets := make([]any, len(colTypes))
+	for i := range colTypes {
+		scanTargets[i] = reflect.New(colTypes[i].ScanType()).Interface()
+	}
+
+	values := &telemetrytypes.TelemetryFieldValues{}
+	for rows.Next() {
+		// Create fresh scan targets for each row
+		scan := make([]any, len(colTypes))
+		for i := range colTypes {
+			scan[i] = reflect.New(colTypes[i].ScanType()).Interface()
+		}
+
+		if err := rows.Scan(scan...); err != nil {
+			return nil, false, errors.WrapInternalf(err, CodeFailListJSONValues, "failed to scan JSON value row")
+		}
+
+		// Extract values from scan targets and process them
+		// Column order: String, Int64, Float64, Bool, ArrayString, ArrayInt64, ArrayFloat64, ArrayBool, ArrayDynamic
+		var consume func(scan []any) error
+		consume = func(scan []any) error {
+			for _, value := range scan {
+				value := derefValue(value) // dereference the double pointer if it is a pointer
+				switch value := value.(type) {
+				case string:
+					values.StringValues = append(values.StringValues, value)
+				case int64:
+					values.NumberValues = append(values.NumberValues, float64(value))
+				case float64:
+					values.NumberValues = append(values.NumberValues, value)
+				case bool:
+					values.BoolValues = append(values.BoolValues, value)
+				case []*string:
+					for _, str := range value {
+						values.StringValues = append(values.StringValues, *str)
+					}
+				case []*int64:
+					for _, num := range value {
+						values.NumberValues = append(values.NumberValues, float64(*num))
+					}
+				case []*float64:
+					for _, num := range value {
+						values.NumberValues = append(values.NumberValues, float64(*num))
+					}
+				case []*bool:
+					for _, boolVal := range value {
+						values.BoolValues = append(values.BoolValues, *boolVal)
+					}
+				case chcol.Variant:
+					if !value.Nil() {
+						if err := consume([]any{value.Any()}); err != nil {
+							return err
+						}
+					}
+				case []chcol.Variant:
+					extractedValues := make([]any, len(value))
+					for _, variant := range value {
+						if !variant.Nil() && variant.Type() != "JSON" { // skip JSON values cuz they're relevant for nested keys
+							extractedValues = append(extractedValues, variant.Any())
+						}
+					}
+					if err := consume(extractedValues); err != nil {
+						return err
+					}
+				default:
+					if value == nil {
+						continue
+					}
+					return errors.NewInternalf(CodeFailScanJSONValue, "unknown JSON value type: %T", value)
+				}
+			}
+
+			return nil
+		}
+		if err := consume(scan); err != nil {
+			return nil, false, err
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, false, errors.WrapInternalf(err, CodeFailListJSONValues, "error iterating JSON values")
+	}
+
+	return values, true, nil
+}
+
+func derefValue(v any) any {
+	if v == nil {
+		return nil
+	}
+
+	val := reflect.ValueOf(v)
+	for val.Kind() == reflect.Ptr {
+		if val.IsNil() {
+			return nil
+		}
+		val = val.Elem()
+	}
+
+	return val.Interface()
+}
+
+// IsPathPromoted checks if a specific path is promoted
+func IsPathPromoted(ctx context.Context, conn clickhouse.Conn, path string) (bool, error) {
+	split := strings.Split(path, telemetrylogs.ArraySep)
+	query := fmt.Sprintf("SELECT 1 FROM %s.%s WHERE path = ? LIMIT 1", DBName, PromotedPathsTableName)
+	rows, err := conn.Query(ctx, query, split[0])
+	if err != nil {
+		return false, errors.WrapInternalf(err, CodeFailCheckPathPromoted, "failed to check if path %s is promoted", path)
+	}
+	defer rows.Close()
+
+	return rows.Next(), nil
+}
+
+// TODO(Piyush): Remove this function
+func CleanPathPrefixes(path string) string {
+	path = strings.TrimPrefix(path, telemetrytypes.BodyJSONStringSearchPrefix)
+	path = strings.TrimPrefix(path, telemetrylogs.BodyJSONColumnPrefix)
+	path = strings.TrimPrefix(path, telemetrylogs.BodyPromotedColumnPrefix)
+	return path
+}

pkg/telemetrymetadata/body_json_metadata_test.go

@@ -0,0 +1,209 @@
+package telemetrymetadata
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestBuildGetBodyJSONPathsQuery(t *testing.T) {
+	testCases := []struct {
+		name              string
+		fieldKeySelectors []*telemetrytypes.FieldKeySelector
+		expectedSQL       []string // SQL should contain these strings
+		notExpectedSQL    []string // SQL should NOT contain these strings
+		expectedArgs      []any    // Args include search texts (if any) + limit value
+		expectedLimit     int
+	}{
+
+		{
+			name: "Single search text with EQUAL operator",
+			fieldKeySelectors: []*telemetrytypes.FieldKeySelector{
+				{
+					Name:              "user.name",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+				},
+			},
+			expectedSQL: []string{
+				"SELECT",
+				"path",
+				"groupArray(DISTINCT type) AS types",
+				"max(last_seen) AS last_seen",
+				fmt.Sprintf("FROM %s.%s", DBName, PathTypesTableName),
+				"WHERE",
+				"path = ?",
+				"GROUP BY path",
+				"ORDER BY last_seen DESC",
+				"LIMIT ?",
+			},
+			notExpectedSQL: []string{
+				"ILIKE",
+				"LIKE",
+			},
+			expectedArgs:  []any{"user.name", defaultPathLimit},
+			expectedLimit: defaultPathLimit,
+		},
+		{
+			name: "Single search text with LIKE operator",
+			fieldKeySelectors: []*telemetrytypes.FieldKeySelector{
+				{
+					Name:              "user",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+				},
+			},
+			expectedSQL: []string{
+				"SELECT",
+				"path",
+				"groupArray(DISTINCT type) AS types",
+				"max(last_seen) AS last_seen",
+				fmt.Sprintf("FROM %s.%s", DBName, PathTypesTableName),
+				"WHERE",
+				"path LIKE ?",
+				"GROUP BY path",
+				"ORDER BY last_seen DESC",
+				"LIMIT ?",
+			},
+			notExpectedSQL: []string{
+				"path = ?",
+			},
+			expectedArgs:  []any{"user", 100},
+			expectedLimit: 100,
+		},
+		{
+			name: "Multiple search texts with EQUAL operator",
+			fieldKeySelectors: []*telemetrytypes.FieldKeySelector{
+				{
+					Name:              "user.name",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+				},
+				{
+					Name:              "user.age",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeExact,
+				},
+			},
+			expectedSQL: []string{
+				"SELECT",
+				"path",
+				"groupArray(DISTINCT type) AS types",
+				"max(last_seen) AS last_seen",
+				fmt.Sprintf("FROM %s.%s", DBName, PathTypesTableName),
+				"WHERE",
+				"OR",
+				"path = ?",
+				"GROUP BY path",
+				"ORDER BY last_seen DESC",
+				"LIMIT ?",
+			},
+			notExpectedSQL: []string{
+				"ILIKE",
+				"LIKE",
+			},
+			expectedArgs:  []any{"user.name", "user.age", defaultPathLimit},
+			expectedLimit: defaultPathLimit,
+		},
+		{
+			name: "Multiple search texts with LIKE operator",
+			fieldKeySelectors: []*telemetrytypes.FieldKeySelector{
+				{
+					Name:              "user",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+				},
+				{
+					Name:              "admin",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+				},
+			},
+			expectedSQL: []string{
+				"SELECT",
+				"path",
+				"groupArray(DISTINCT type) AS types",
+				"max(last_seen) AS last_seen",
+				fmt.Sprintf("FROM %s.%s", DBName, PathTypesTableName),
+				"WHERE",
+				"OR",
+				"path LIKE ?",
+				"GROUP BY path",
+				"ORDER BY last_seen DESC",
+				"LIMIT ?",
+			},
+			notExpectedSQL: []string{
+				"path = ?",
+			},
+			expectedArgs:  []any{"user", "admin", defaultPathLimit},
+			expectedLimit: defaultPathLimit,
+		},
+		{
+			name: "Search with Contains operator (should default to LIKE)",
+			fieldKeySelectors: []*telemetrytypes.FieldKeySelector{
+				{
+					Name:              "test",
+					SelectorMatchType: telemetrytypes.FieldSelectorMatchTypeFuzzy,
+				},
+			},
+			expectedSQL: []string{
+				"WHERE",
+				"path LIKE ?",
+			},
+			notExpectedSQL: []string{
+				"path = ?",
+			},
+			expectedArgs:  []any{"test", defaultPathLimit},
+			expectedLimit: defaultPathLimit,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			query, args, limit, err := buildGetBodyJSONPathsQuery(tc.fieldKeySelectors)
+			require.NoError(t, err, "Error building query: %v", err)
+
+			// Verify query is not empty
+			require.NotEmpty(t, query, "Query should not be empty")
+
+			// Verify expected SQL patterns are present
+			queryUpper := strings.ToUpper(query)
+			for _, expected := range tc.expectedSQL {
+				assert.Contains(t, queryUpper, strings.ToUpper(expected),
+					"Query should contain: %s\nActual query: %s", expected, query)
+			}
+
+			// Verify SQL patterns that should NOT be present
+			for _, notExpected := range tc.notExpectedSQL {
+				assert.NotContains(t, queryUpper, strings.ToUpper(notExpected),
+					"Query should NOT contain: %s\nActual query: %s", notExpected, query)
+			}
+
+			// Verify limit matches expected
+			assert.Equal(t, tc.expectedLimit, limit, "Limit should match expected value")
+
+			// Verify arguments match (including limit as last arg)
+			assert.Equal(t, tc.expectedArgs, args, "Arguments should match. Expected: %v, Got: %v", tc.expectedArgs, args)
+
+			// Verify limit value is in args (should be last element)
+			if len(args) > 0 {
+				actualLimit := args[len(args)-1]
+				assert.Equal(t, tc.expectedLimit, actualLimit, "Last argument should be the limit value")
+			}
+
+			// Verify LIMIT placeholder is in query
+			assert.Contains(t, query, "LIMIT ?", "Query should contain LIMIT ? placeholder")
+
+			// Verify GROUP BY is present
+			assert.Contains(t, queryUpper, "GROUP BY", "Query should contain GROUP BY")
+			assert.Contains(t, queryUpper, "GROUP BY PATH", "Query should group by path")
+
+			// Verify ORDER BY is present
+			assert.Contains(t, queryUpper, "ORDER BY", "Query should contain ORDER BY")
+			assert.Contains(t, queryUpper, "LAST_SEEN DESC", "Query should ORDER BY last_seen DESC")
+
+			// Verify SELECT columns
+			assert.Contains(t, queryUpper, "PATH", "Query should select path")
+			assert.Contains(t, queryUpper, "GROUPARRAY(DISTINCT TYPE)", "Query should select groupArray(DISTINCT type)")
+			assert.Contains(t, queryUpper, "MAX(LAST_SEEN)", "Query should select max(last_seen)")
+		})
+	}
+}

pkg/telemetrymetadata/metadata.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/SigNoz/signoz/pkg/errors"
 	"github.com/SigNoz/signoz/pkg/factory"
+	"github.com/SigNoz/signoz/pkg/query-service/constants"
 	"github.com/SigNoz/signoz/pkg/querybuilder"
 	"github.com/SigNoz/signoz/pkg/telemetrylogs"
 	"github.com/SigNoz/signoz/pkg/telemetrystore"
@@ -572,6 +573,14 @@ func (t *telemetryMetaStore) getLogsKeys(ctx context.Context, fieldKeySelectors
 		}
 	}
 
+	if constants.BodyJSONQueryEnabled {
+		bodyJSONPaths, finished, err := getBodyJSONPaths(ctx, t.telemetrystore, fieldKeySelectors) // LIKE for pattern matching
+		if err != nil {
+			t.logger.Error("failed to extract body JSON paths", "error", err)
+		}
+		keys = append(keys, bodyJSONPaths...)
+		complete = complete && finished
+	}
 	return keys, complete, nil
 }
 
@@ -1164,6 +1173,10 @@ func (t *telemetryMetaStore) getLogFieldValues(ctx context.Context, fieldValueSe
 		limit = 50
 	}
 
+	if strings.HasPrefix(fieldValueSelector.Name, telemetrytypes.BodyJSONStringSearchPrefix) {
+		return ListJSONValues(ctx, t.telemetrystore.ClickhouseDB(), fieldValueSelector.Name, limit)
+	}
+
 	sb := sqlbuilder.Select("DISTINCT string_value, number_value").From(t.logsDBName + "." + t.logsFieldsTblName)
 
 	if fieldValueSelector.Name != "" {

pkg/telemetrymetadata/tables.go

@@ -1,7 +1,12 @@
 package telemetrymetadata
 
+import "github.com/SigNoz/signoz-otel-collector/constants"
+
 const (
 	DBName                           = "signoz_metadata"
 	AttributesMetadataTableName      = "distributed_attributes_metadata"
 	AttributesMetadataLocalTableName = "attributes_metadata"
+	PathTypesTableName               = constants.DistributedPathTypesTable
+	PromotedPathsTableName           = constants.DistributedPromotedPathsTable
+	SkipIndexTableName               = "system.data_skipping_indices"
 )

pkg/telemetrystore/clickhousetelemetrystore/provider.go

@@ -32,6 +32,8 @@ func New(ctx context.Context, providerSettings factory.ProviderSettings, config
 	options.MaxIdleConns = config.Connection.MaxIdleConns
 	options.MaxOpenConns = config.Connection.MaxOpenConns
 	options.DialTimeout = config.Connection.DialTimeout
+	// This is to avoid the driver decoding issues with JSON columns
+	options.Settings["output_format_native_write_json_as_string"] = 1
 
 	chConn, err := clickhouse.Open(options)
 	if err != nil {

pkg/telemetrytraces/condition_builder.go

@@ -162,13 +162,15 @@ func (c *conditionBuilder) conditionFor(
 	case qbtypes.FilterOperatorExists, qbtypes.FilterOperatorNotExists:
 
 		var value any
-		switch column.Type {
-		case schema.JSONColumnType{}:
+		// schema.JSONColumnType{} now can not be used in switch cases, so we need to check if the column is a JSON column
+		if column.IsJSONColumn() {
 			if operator == qbtypes.FilterOperatorExists {
 				return sb.IsNotNull(tblFieldName), nil
 			} else {
 				return sb.IsNull(tblFieldName), nil
 			}
+		}
+		switch column.Type {
 		case schema.ColumnTypeString,
 			schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 			schema.FixedStringColumnType{Length: 32},

pkg/telemetrytraces/field_mapper.go

@@ -236,8 +236,8 @@ func (m *defaultFieldMapper) FieldFor(
 		return "", err
 	}
 
-	switch column.Type {
-	case schema.JSONColumnType{}:
+	// schema.JSONColumnType{} now can not be used in switch cases, so we need to check if the column is a JSON column
+	if column.IsJSONColumn() {
 		// json is only supported for resource context as of now
 		if key.FieldContext != telemetrytypes.FieldContextResource {
 			return "", errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "only resource context fields are supported for json columns, got %s", key.FieldContext.String)
@@ -253,7 +253,9 @@ func (m *defaultFieldMapper) FieldFor(
 		} else {
 			return fmt.Sprintf("multiIf(%s.`%s` IS NOT NULL, %s.`%s`::String, mapContains(%s, '%s'), %s, NULL)", column.Name, key.Name, column.Name, key.Name, oldColumn.Name, key.Name, oldKeyName), nil
 		}
+	}
 
+	switch column.Type {
 	case schema.ColumnTypeString,
 		schema.LowCardinalityColumnType{ElementType: schema.ColumnTypeString},
 		schema.ColumnTypeUInt64,

pkg/telemetrytraces/statement_builder.go

@@ -495,7 +495,7 @@ func (b *traceQueryStatementBuilder) buildTimeSeriesQuery(
 	// Keep original column expressions so we can build the tuple
 	fieldNames := make([]string, 0, len(query.GroupBy))
 	for _, gb := range query.GroupBy {
-		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, "", nil)
+		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -637,7 +637,7 @@ func (b *traceQueryStatementBuilder) buildScalarQuery(
 
 	var allGroupByArgs []any
 	for _, gb := range query.GroupBy {
-		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, "", nil)
+		expr, args, err := querybuilder.CollisionHandledFinalExpr(ctx, &gb.TelemetryFieldKey, b.fm, b.cb, keys, telemetrytypes.FieldDataTypeString, nil)
 		if err != nil {
 			return nil, err
 		}

pkg/telemetrytraces/stmt_builder_test.go

@@ -328,7 +328,7 @@ func TestStatementBuilder(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -496,7 +496,7 @@ func TestStatementBuilderListQuery(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 
@@ -572,7 +572,7 @@ func TestStatementBuilderTraceQuery(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 

pkg/telemetrytraces/trace_operator_cte_builder.go

@@ -552,7 +552,6 @@ func (b *traceOperatorCTEBuilder) buildTimeSeriesQuery(ctx context.Context, sele
 			b.stmtBuilder.cb,
 			keys,
 			telemetrytypes.FieldDataTypeString,
-			"",
 			nil,
 		)
 		if err != nil {
@@ -662,7 +661,6 @@ func (b *traceOperatorCTEBuilder) buildTraceQuery(ctx context.Context, selectFro
 			b.stmtBuilder.cb,
 			keys,
 			telemetrytypes.FieldDataTypeString,
-			"",
 			nil,
 		)
 		if err != nil {
@@ -802,7 +800,6 @@ func (b *traceOperatorCTEBuilder) buildScalarQuery(ctx context.Context, selectFr
 			b.stmtBuilder.cb,
 			keys,
 			telemetrytypes.FieldDataTypeString,
-			"",
 			nil,
 		)
 		if err != nil {

pkg/telemetrytraces/trace_operator_cte_builder_test.go

@@ -390,7 +390,7 @@ func TestTraceOperatorStatementBuilder(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 	traceStmtBuilder := NewTraceQueryStatementBuilder(
@@ -506,7 +506,7 @@ func TestTraceOperatorStatementBuilderErrors(t *testing.T) {
 	cb := NewConditionBuilder(fm)
 	mockMetadataStore := telemetrytypestest.NewMockMetadataStore()
 	mockMetadataStore.KeysMap = buildCompleteFieldKeyMap()
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	resourceFilterStmtBuilder := resourceFilterStmtBuilder()
 	traceStmtBuilder := NewTraceQueryStatementBuilder(

pkg/telemetrytraces/trace_time_range_test.go

@@ -44,7 +44,7 @@ func TestTraceTimeRangeOptimization(t *testing.T) {
 		mockMetadataStore,
 	)
 
-	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, "", nil)
+	aggExprRewriter := querybuilder.NewAggExprRewriter(instrumentationtest.New().ToProviderSettings(), nil, fm, cb, nil)
 
 	statementBuilder := NewTraceQueryStatementBuilder(
 		instrumentationtest.New().ToProviderSettings(),

pkg/types/promotetypes/types.go

@@ -0,0 +1,76 @@
+package promotetypes
+
+import (
+	"strings"
+
+	"github.com/SigNoz/signoz-otel-collector/constants"
+	"github.com/SigNoz/signoz-otel-collector/pkg/keycheck"
+	"github.com/SigNoz/signoz/pkg/errors"
+	"github.com/SigNoz/signoz/pkg/telemetrylogs"
+	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
+)
+
+type WrappedIndex struct {
+	JSONDataType telemetrytypes.JSONDataType `json:"-"`
+	ColumnType   string                      `json:"column_type"`
+	Type         string                      `json:"type"`
+	Granularity  int                         `json:"granularity"`
+}
+
+type PromotePath struct {
+	Path    string `json:"path"`
+	Promote bool   `json:"promote,omitempty"`
+
+	Indexes []WrappedIndex `json:"indexes,omitempty"`
+}
+
+func (i *PromotePath) Validate() error {
+	if i.Path == "" {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path is required")
+	}
+
+	if strings.Contains(i.Path, " ") {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path cannot contain spaces")
+	}
+
+	if strings.Contains(i.Path, telemetrylogs.ArraySep) || strings.Contains(i.Path, telemetrylogs.ArrayAnyIndex) {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "array paths can not be promoted or indexed")
+	}
+
+	if strings.HasPrefix(i.Path, constants.BodyJSONColumnPrefix) || strings.HasPrefix(i.Path, constants.BodyPromotedColumnPrefix) {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "`%s`, `%s` don't add these prefixes to the path", constants.BodyJSONColumnPrefix, constants.BodyPromotedColumnPrefix)
+	}
+
+	if !strings.HasPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix) {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "path must start with `body.`")
+	}
+
+	// remove the "body." prefix from the path
+	i.Path = strings.TrimPrefix(i.Path, telemetrytypes.BodyJSONStringSearchPrefix)
+
+	isCardinal := keycheck.IsCardinal(i.Path)
+	if isCardinal {
+		return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "cardinal paths can not be promoted or indexed")
+	}
+
+	for idx, index := range i.Indexes {
+		if index.Type == "" {
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index type is required")
+		}
+		if index.Granularity <= 0 {
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index granularity must be greater than 0")
+		}
+
+		jsonDataType, ok := telemetrytypes.MappingStringToJSONDataType[index.ColumnType]
+		if !ok {
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "invalid column type: %s", index.ColumnType)
+		}
+		if !jsonDataType.IndexSupported {
+			return errors.Newf(errors.TypeInvalidInput, errors.CodeInvalidInput, "index is not supported for column type: %s", index.ColumnType)
+		}
+
+		i.Indexes[idx].JSONDataType = jsonDataType
+	}
+
+	return nil
+}

pkg/types/telemetrytypes/field.go

@@ -17,6 +17,10 @@ var (
 	FieldSelectorMatchTypeFuzzy = FieldSelectorMatchType{valuer.NewString("fuzzy")}
 )
 
+// BodyJSONStringSearchPrefix is the prefix used for body JSON search queries
+// e.g., "body.status" where "body." is the prefix
+const BodyJSONStringSearchPrefix = `body.`
+
 type TelemetryFieldKey struct {
 	Name          string        `json:"name"`
 	Description   string        `json:"description,omitempty"`
@@ -24,7 +28,10 @@ type TelemetryFieldKey struct {
 	Signal        Signal        `json:"signal,omitempty"`
 	FieldContext  FieldContext  `json:"fieldContext,omitempty"`
 	FieldDataType FieldDataType `json:"fieldDataType,omitempty"`
-	Materialized  bool          `json:"-"`
+
+	JSONDataType *JSONDataType       `json:"-,omitempty"`
+	Indexes      []JSONDataTypeIndex `json:"-"`
+	Materialized bool                `json:"-"` // refers to promoted in case of body.... fields
 }
 
 func (f TelemetryFieldKey) String() string {

pkg/types/telemetrytypes/field_context.go

@@ -36,6 +36,9 @@ import (
 //
 // - Use `log.` for explicit log context
 //   - `log.severity_text` will always resolve to `severity_text` of log record
+//
+// - Use `body.` to indicate and enforce body context
+//   - `body.key` will look for `key` in the body field
 type FieldContext struct {
 	valuer.String
 }
@@ -49,6 +52,7 @@ var (
 	FieldContextScope       = FieldContext{valuer.NewString("scope")}
 	FieldContextAttribute   = FieldContext{valuer.NewString("attribute")}
 	FieldContextEvent       = FieldContext{valuer.NewString("event")}
+	FieldContextBody        = FieldContext{valuer.NewString("body")}
 	FieldContextUnspecified = FieldContext{valuer.NewString("")}
 
 	// Map string representations to FieldContext values
@@ -65,6 +69,7 @@ var (
 		"point":      FieldContextAttribute,
 		"attribute":  FieldContextAttribute,
 		"event":      FieldContextEvent,
+		"body":       FieldContextBody,
 		"spanfield":  FieldContextSpan,
 		"span":       FieldContextSpan,
 		"logfield":   FieldContextLog,
@@ -144,6 +149,8 @@ func (f FieldContext) TagType() string {
 		return "metricfield"
 	case FieldContextEvent:
 		return "eventfield"
+	case FieldContextBody:
+		return "body"
 	}
 	return ""
 }

pkg/types/telemetrytypes/field_datatype.go

@@ -31,6 +31,9 @@ var (
 	FieldDataTypeArrayInt64  = FieldDataType{valuer.NewString("[]int64")}
 	FieldDataTypeArrayNumber = FieldDataType{valuer.NewString("[]number")}
 
+	FieldDataTypeArrayObject  = FieldDataType{valuer.NewString("[]object")}
+	FieldDataTypeArrayDynamic = FieldDataType{valuer.NewString("[]dynamic")}
+
 	// Map string representations to FieldDataType values
 	// We want to handle all the possible string representations of the data types.
 	// Even if the user uses some non-standard representation, we want to be able to

pkg/types/telemetrytypes/field_test.go

@@ -1,6 +1,7 @@
 package telemetrytypes
 
 import (
+	"reflect"
 	"testing"
 )
 
@@ -86,7 +87,7 @@ func TestGetFieldKeyFromKeyText(t *testing.T) {
 
 	for _, testCase := range testCases {
 		result := GetFieldKeyFromKeyText(testCase.keyText)
-		if result != testCase.expected {
+		if !reflect.DeepEqual(result, testCase.expected) {
 			t.Errorf("expected %v, got %v", testCase.expected, result)
 		}
 	}

pkg/types/telemetrytypes/json_access_plan.go

@@ -0,0 +1,83 @@
+package telemetrytypes
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/SigNoz/signoz-otel-collector/exporter/jsontypeexporter"
+	"github.com/SigNoz/signoz-otel-collector/pkg/keycheck"
+)
+
+type JSONAccessBranchType string
+type JSONAccessPlan = []*JSONAccessNode
+
+const (
+	BranchJSON    JSONAccessBranchType = "json"
+	BranchDynamic JSONAccessBranchType = "dynamic"
+)
+
+type TerminalConfig struct {
+	Key       *TelemetryFieldKey
+	ElemType  JSONDataType
+	ValueType JSONDataType
+}
+
+// Node is now a tree structure representing the complete JSON path traversal
+// that precomputes all possible branches and their types
+type JSONAccessNode struct {
+	// Node information
+	Name       string
+	IsTerminal bool
+	isRoot     bool // marked true for only body_json and body_json_promoted
+
+	// Precomputed type information (single source of truth)
+	AvailableTypes []JSONDataType
+
+	// Array type branches (Array(JSON) vs Array(Dynamic))
+	Branches map[JSONAccessBranchType]*JSONAccessNode
+
+	// Terminal configuration
+	TerminalConfig *TerminalConfig
+
+	// Parent reference for traversal
+	Parent *JSONAccessNode
+
+	// JSON progression parameters (precomputed during planning)
+	MaxDynamicTypes int
+	MaxDynamicPaths int
+}
+
+func NewRootJSONAccessNode(name string, maxDynamicTypes, maxDynamicPaths int) *JSONAccessNode {
+	return &JSONAccessNode{
+		Name:            name,
+		isRoot:          true,
+		MaxDynamicTypes: maxDynamicTypes,
+		MaxDynamicPaths: maxDynamicPaths,
+	}
+}
+
+func (n *JSONAccessNode) Alias() string {
+	if n.isRoot {
+		return n.Name
+	} else if n.Parent == nil {
+		return fmt.Sprintf("`%s`", n.Name)
+	}
+
+	parentAlias := strings.TrimLeft(n.Parent.Alias(), "`")
+	parentAlias = strings.TrimRight(parentAlias, "`")
+
+	sep := jsontypeexporter.ArraySeparator
+	if n.Parent.isRoot {
+		sep = "."
+	}
+	return fmt.Sprintf("`%s%s%s`", parentAlias, sep, n.Name)
+}
+
+func (n *JSONAccessNode) FieldPath() string {
+	key := n.Name
+	if keycheck.IsBacktickRequired(key) {
+		key = "`" + key + "`"
+	}
+
+	return n.Parent.Alias() + "." + key
+}

pkg/types/telemetrytypes/json_datatype.go

@@ -0,0 +1,80 @@
+package telemetrytypes
+
+type JSONDataTypeIndex struct {
+	Type             JSONDataType
+	ColumnExpression string
+	IndexExpression  string
+}
+
+type JSONDataType struct {
+	str            string // Store the correct case for ClickHouse
+	IsArray        bool
+	ScalerType     string
+	IndexSupported bool
+}
+
+// Override StringValue to return the correct case
+func (jdt JSONDataType) StringValue() string {
+	return jdt.str
+}
+
+var (
+	String       = JSONDataType{"String", false, "", true}
+	Int64        = JSONDataType{"Int64", false, "", true}
+	Float64      = JSONDataType{"Float64", false, "", true}
+	Bool         = JSONDataType{"Bool", false, "", false}
+	Dynamic      = JSONDataType{"Dynamic", false, "", false}
+	ArrayString  = JSONDataType{"Array(Nullable(String))", true, "String", false}
+	ArrayInt64   = JSONDataType{"Array(Nullable(Int64))", true, "Int64", false}
+	ArrayFloat64 = JSONDataType{"Array(Nullable(Float64))", true, "Float64", false}
+	ArrayBool    = JSONDataType{"Array(Nullable(Bool))", true, "Bool", false}
+	ArrayDynamic = JSONDataType{"Array(Dynamic)", true, "Dynamic", false}
+	ArrayJSON    = JSONDataType{"Array(JSON)", true, "JSON", false}
+)
+
+var MappingStringToJSONDataType = map[string]JSONDataType{
+	"String":                   String,
+	"Int64":                    Int64,
+	"Float64":                  Float64,
+	"Bool":                     Bool,
+	"Dynamic":                  Dynamic,
+	"Array(Nullable(String))":  ArrayString,
+	"Array(Nullable(Int64))":   ArrayInt64,
+	"Array(Nullable(Float64))": ArrayFloat64,
+	"Array(Nullable(Bool))":    ArrayBool,
+	"Array(Dynamic)":           ArrayDynamic,
+	"Array(JSON)":              ArrayJSON,
+}
+
+var ScalerTypeToArrayType = map[JSONDataType]JSONDataType{
+	String:  ArrayString,
+	Int64:   ArrayInt64,
+	Float64: ArrayFloat64,
+	Bool:    ArrayBool,
+	Dynamic: ArrayDynamic,
+}
+
+var MappingFieldDataTypeToJSONDataType = map[FieldDataType]JSONDataType{
+	FieldDataTypeString:       String,
+	FieldDataTypeInt64:        Int64,
+	FieldDataTypeFloat64:      Float64,
+	FieldDataTypeNumber:       Float64,
+	FieldDataTypeBool:         Bool,
+	FieldDataTypeArrayString:  ArrayString,
+	FieldDataTypeArrayInt64:   ArrayInt64,
+	FieldDataTypeArrayFloat64: ArrayFloat64,
+	FieldDataTypeArrayBool:    ArrayBool,
+}
+
+var MappingJSONDataTypeToFieldDataType = map[JSONDataType]FieldDataType{
+	String:       FieldDataTypeString,
+	Int64:        FieldDataTypeInt64,
+	Float64:      FieldDataTypeFloat64,
+	Bool:         FieldDataTypeBool,
+	ArrayString:  FieldDataTypeArrayString,
+	ArrayInt64:   FieldDataTypeArrayInt64,
+	ArrayFloat64: FieldDataTypeArrayFloat64,
+	ArrayBool:    FieldDataTypeArrayBool,
+	ArrayDynamic: FieldDataTypeArrayDynamic,
+	ArrayJSON:    FieldDataTypeArrayObject,
+}

pkg/utils/utils.go

@@ -0,0 +1,11 @@
+package utils
+
+func Filter[T any](in []T, fn func(T) bool) []T {
+	out := make([]T, 0, len(in))
+	for _, v := range in {
+		if fn(v) {
+			out = append(out, v)
+		}
+	}
+	return out
+}