Merge pull request #6664 from SigNoz/release/v0.64.x

Release/v0.64.x

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -146,11 +146,12 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.61.0
+    image: signoz/query-service:0.64.0
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -186,7 +187,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.61.0
+    image: signoz/frontend:0.64.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -199,7 +200,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -237,7 +238,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.111.14
+      image: signoz/signoz-schema-migrator:0.111.16
       deploy:
         restart_policy:
           condition: on-failure

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -110,6 +110,7 @@ exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/signoz_traces
     low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+    use_new_schema: true
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -69,7 +69,7 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator
     command:
       - "sync" 
@@ -86,7 +86,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
     command:
       [
         "--config=/etc/otel-collector-config.yaml",

deploy/docker/clickhouse-setup/docker-compose-local.yaml

@@ -25,7 +25,8 @@ services:
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
       ]
     ports:
       - "6060:6060"

deploy/docker/clickhouse-setup/docker-compose-minimal.yaml

@@ -162,12 +162,13 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
     container_name: signoz-query-service
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -201,7 +202,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -213,7 +214,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator-sync:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator-sync
     command:
       - "sync"
@@ -228,7 +229,7 @@ services:
       #   condition: service_healthy
 
   otel-collector-migrator-async:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator-async
     command:
       - "async"
@@ -245,7 +246,7 @@ services:
       #   condition: service_healthy
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/docker-compose.testing.yaml

@@ -167,13 +167,14 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
     container_name: signoz-query-service
     command:
       [
         "-config=/root/config/prometheus.yml",
         "-gateway-url=https://api.staging.signoz.cloud",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -208,7 +209,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -220,7 +221,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -234,7 +235,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -119,6 +119,7 @@ exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/signoz_traces
     low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+    use_new_schema: true
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.111.14
+	github.com/SigNoz/signoz-otel-collector v0.111.16
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3

go.sum

@@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
 github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
-github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
+github.com/SigNoz/signoz-otel-collector v0.111.16 h1:535uKH5Oux+35EsI+L3C6pnAP/Ye0PTCbVizXoL+VqE=
-github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
+github.com/SigNoz/signoz-otel-collector v0.111.16/go.mod h1:HJ4m0LY1MPsuZmuRF7Ixb+bY8rxgRzI0VXzOedESsjg=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

pkg/query-service/app/http_handler.go

@@ -4075,10 +4075,9 @@ func (aH *APIHandler) CreateLogsPipeline(w http.ResponseWriter, r *http.Request)
 			zap.L().Warn("found no pipelines in the http request, this will delete all the pipelines")
 		}
 
-		for _, p := range postable {
+		validationErr := aH.LogsParsingPipelineController.ValidatePipelines(ctx, postable)
-			if err := p.IsValid(); err != nil {
+		if validationErr != nil {
-				return nil, model.BadRequestStr(err.Error())
+			return nil, validationErr
-			}
 		}
 
 		return aH.LogsParsingPipelineController.ApplyPipelines(ctx, postable)

pkg/query-service/app/logparsingpipeline/controller.go

@@ -94,6 +94,45 @@ func (ic *LogParsingPipelineController) ApplyPipelines(
 	return ic.GetPipelinesByVersion(ctx, cfg.Version)
 }
 
+func (ic *LogParsingPipelineController) ValidatePipelines(
+	ctx context.Context,
+	postedPipelines []PostablePipeline,
+) *model.ApiError {
+	for _, p := range postedPipelines {
+		if err := p.IsValid(); err != nil {
+			return model.BadRequestStr(err.Error())
+		}
+	}
+
+	// Also run a collector simulation to ensure config is fit
+	// for e2e use with a collector
+	pipelines := []Pipeline{}
+	for _, pp := range postedPipelines {
+		pipelines = append(pipelines, Pipeline{
+			Id:          uuid.New().String(),
+			OrderId:     pp.OrderId,
+			Enabled:     pp.Enabled,
+			Name:        pp.Name,
+			Alias:       pp.Alias,
+			Description: &pp.Description,
+			Filter:      pp.Filter,
+			Config:      pp.Config,
+		})
+	}
+
+	sampleLogs := []model.SignozLog{{Body: ""}}
+	_, _, simulationErr := SimulatePipelinesProcessing(
+		ctx, pipelines, sampleLogs,
+	)
+	if simulationErr != nil {
+		return model.BadRequest(fmt.Errorf(
+			"invalid pipelines config: %w", simulationErr.ToError(),
+		))
+	}
+
+	return nil
+}
+
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(

pkg/query-service/tests/integration/logparsingpipeline_test.go

@@ -350,6 +350,27 @@ func TestLogPipelinesValidation(t *testing.T) {
 				},
 			},
 			ExpectedResponseStatusCode: 400,
+		}, {
+			Name: "Invalid from field path",
+			Pipeline: logparsingpipeline.PostablePipeline{
+				OrderId: 1,
+				Name:    "pipeline 1",
+				Alias:   "pipeline1",
+				Enabled: true,
+				Filter:  validPipelineFilterSet,
+				Config: []logparsingpipeline.PipelineOperator{
+					{
+						OrderId: 1,
+						ID:      "move",
+						Type:    "move",
+						From:    `attributes.temp_parsed_body."@l"`,
+						To:      "attributes.test",
+						Enabled: true,
+						Name:    "test move",
+					},
+				},
+			},
+			ExpectedResponseStatusCode: 400,
 		},
 	}