Merge pull request #6664 from SigNoz/release/v0.64.x

Release/v0.64.x
chore(signoz): pin versions: SigNoz 0.64.0
2024-12-18 18:29:05 +05:30 · 2024-12-18 17:57:42 +05:30 · 2024-12-18 17:55:57 +05:30 · 2024-12-18 17:55:22 +07:00 · 2024-12-18 15:41:24 +05:30 · 2024-12-18 15:25:20 +05:30
12 changed files with 91 additions and 26 deletions
--- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
+++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
@@ -146,11 +146,12 @@ services:
        condition: on-failure

  query-service:
-    image: signoz/query-service:0.61.0
+    image: signoz/query-service:0.64.0
    command:
      [
        "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -186,7 +187,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:0.61.0
+    image: signoz/frontend:0.64.0
    deploy:
      restart_policy:
        condition: on-failure
@@ -199,7 +200,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector:
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
    command:
      [
        "--config=/etc/otel-collector-config.yaml",
@@ -237,7 +238,7 @@ services:
      - query-service

  otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.111.14
+      image: signoz/signoz-schema-migrator:0.111.16
      deploy:
        restart_policy:
          condition: on-failure
--- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
+++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
@@ -110,6 +110,7 @@ exporters:
  clickhousetraces:
    datasource: tcp://clickhouse:9000/signoz_traces
    low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+    use_new_schema: true
  clickhousemetricswrite:
    endpoint: tcp://clickhouse:9000/signoz_metrics
    resource_to_telemetry_conversion:
--- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml
@@ -69,7 +69,7 @@ services:
      - --storage.path=/data

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
    container_name: otel-migrator
    command:
      - "sync" 
@@ -86,7 +86,7 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
  otel-collector:
    container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
    command:
      [
        "--config=/etc/otel-collector-config.yaml",
--- a/deploy/docker/clickhouse-setup/docker-compose-local.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose-local.yaml
@@ -25,7 +25,8 @@ services:
    command:
      [
        "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
      ]
    ports:
      - "6060:6060"
--- a/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose-minimal.yaml
@@ -162,12 +162,13 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`

  query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
    container_name: signoz-query-service
    command:
      [
        "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -201,7 +202,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
    container_name: signoz-frontend
    restart: on-failure
    depends_on:
@@ -213,7 +214,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector-migrator-sync:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
    container_name: otel-migrator-sync
    command:
      - "sync"
@@ -228,7 +229,7 @@ services:
      #   condition: service_healthy

  otel-collector-migrator-async:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
    container_name: otel-migrator-async
    command:
      - "async"
@@ -245,7 +246,7 @@ services:
      #   condition: service_healthy

  otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
    container_name: signoz-otel-collector
    command:
      [
--- a/deploy/docker/clickhouse-setup/docker-compose.testing.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose.testing.yaml
@@ -167,13 +167,14 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`

  query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
    container_name: signoz-query-service
    command:
      [
        "-config=/root/config/prometheus.yml",
        "-gateway-url=https://api.staging.signoz.cloud",
-        "--use-logs-new-schema=true"
+        "--use-logs-new-schema=true",
+        "--use-trace-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -208,7 +209,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
    container_name: signoz-frontend
    restart: on-failure
    depends_on:
@@ -220,7 +221,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
    container_name: otel-migrator
    command:
      - "--dsn=tcp://clickhouse:9000"
@@ -234,7 +235,7 @@ services:


  otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
    container_name: signoz-otel-collector
    command:
      [
--- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml
+++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
@@ -119,6 +119,7 @@ exporters:
  clickhousetraces:
    datasource: tcp://clickhouse:9000/signoz_traces
    low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+    use_new_schema: true
  clickhousemetricswrite:
    endpoint: tcp://clickhouse:9000/signoz_metrics
    resource_to_telemetry_conversion:
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.111.14
+	github.com/SigNoz/signoz-otel-collector v0.111.16
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3
--- a/go.sum
+++ b/go.sum
@@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
 github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
-github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
-github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
+github.com/SigNoz/signoz-otel-collector v0.111.16 h1:535uKH5Oux+35EsI+L3C6pnAP/Ye0PTCbVizXoL+VqE=
+github.com/SigNoz/signoz-otel-collector v0.111.16/go.mod h1:HJ4m0LY1MPsuZmuRF7Ixb+bY8rxgRzI0VXzOedESsjg=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=
--- a/pkg/query-service/app/http_handler.go
+++ b/pkg/query-service/app/http_handler.go
@@ -4075,10 +4075,9 @@ func (aH *APIHandler) CreateLogsPipeline(w http.ResponseWriter, r *http.Request)
 			zap.L().Warn("found no pipelines in the http request, this will delete all the pipelines")
 		}

-		for _, p := range postable {
-			if err := p.IsValid(); err != nil {
-				return nil, model.BadRequestStr(err.Error())
-			}
+		validationErr := aH.LogsParsingPipelineController.ValidatePipelines(ctx, postable)
+		if validationErr != nil {
+			return nil, validationErr
 		}

 		return aH.LogsParsingPipelineController.ApplyPipelines(ctx, postable)
--- a/pkg/query-service/app/logparsingpipeline/controller.go
+++ b/pkg/query-service/app/logparsingpipeline/controller.go
@@ -94,6 +94,45 @@ func (ic *LogParsingPipelineController) ApplyPipelines(
 	return ic.GetPipelinesByVersion(ctx, cfg.Version)
 }

+func (ic *LogParsingPipelineController) ValidatePipelines(
+	ctx context.Context,
+	postedPipelines []PostablePipeline,
+) *model.ApiError {
+	for _, p := range postedPipelines {
+		if err := p.IsValid(); err != nil {
+			return model.BadRequestStr(err.Error())
+		}
+	}
+
+	// Also run a collector simulation to ensure config is fit
+	// for e2e use with a collector
+	pipelines := []Pipeline{}
+	for _, pp := range postedPipelines {
+		pipelines = append(pipelines, Pipeline{
+			Id:          uuid.New().String(),
+			OrderId:     pp.OrderId,
+			Enabled:     pp.Enabled,
+			Name:        pp.Name,
+			Alias:       pp.Alias,
+			Description: &pp.Description,
+			Filter:      pp.Filter,
+			Config:      pp.Config,
+		})
+	}
+
+	sampleLogs := []model.SignozLog{{Body: ""}}
+	_, _, simulationErr := SimulatePipelinesProcessing(
+		ctx, pipelines, sampleLogs,
+	)
+	if simulationErr != nil {
+		return model.BadRequest(fmt.Errorf(
+			"invalid pipelines config: %w", simulationErr.ToError(),
+		))
+	}
+
+	return nil
+}
+
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(
--- a/pkg/query-service/tests/integration/logparsingpipeline_test.go
+++ b/pkg/query-service/tests/integration/logparsingpipeline_test.go
@@ -350,6 +350,27 @@ func TestLogPipelinesValidation(t *testing.T) {
 				},
 			},
 			ExpectedResponseStatusCode: 400,
+		}, {
+			Name: "Invalid from field path",
+			Pipeline: logparsingpipeline.PostablePipeline{
+				OrderId: 1,
+				Name:    "pipeline 1",
+				Alias:   "pipeline1",
+				Enabled: true,
+				Filter:  validPipelineFilterSet,
+				Config: []logparsingpipeline.PipelineOperator{
+					{
+						OrderId: 1,
+						ID:      "move",
+						Type:    "move",
+						From:    `attributes.temp_parsed_body."@l"`,
+						To:      "attributes.test",
+						Enabled: true,
+						Name:    "test move",
+					},
+				},
+			},
+			ExpectedResponseStatusCode: 400,
 		},
 	}
Author	SHA1	Message	Date
Prashant Shahi	823f84f857	Merge pull request #6664 from SigNoz/release/v0.64.x Release/v0.64.x	2024-12-18 18:29:05 +05:30
Prashant Shahi	8a4d45084d	chore(signoz): pin versions: SigNoz 0.64.0 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-12-18 17:57:42 +05:30
Prashant Shahi	5bc6c33899	Merge branch 'main' into release/v0.64.x	2024-12-18 17:55:57 +05:30
Nityananda Gohain	7031c866e8	fix: add flags for using trace new schema (#6651 )	2024-12-18 17:55:22 +07:00
Prashant Shahi	46bc7c7a21	Merge pull request #6662 from SigNoz/release/v0.63.x Release/v0.63.x	2024-12-18 15:41:24 +05:30
Prashant Shahi	6d9741c3a4	chore(signoz): pin versions: SigNoz 0.63.0, SigNoz OtelCollector 0.111.16 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-12-18 15:25:20 +05:30
Prashant Shahi	610a8ec704	Merge branch 'main' into release/v0.63.x	2024-12-18 15:07:57 +05:30
Raj Kamal Singh	cd9f27ab08	Fix: QS: logs pipelines: better validation of pipelines being saved (#6652 ) * chore: add test validating invalid field paths in pipeline operators are rejected * chore: refactor posted pipelines validation to use a controller method * fix: run a collector simulation to validate pipeline config being saved * chore: minor cleanup	2024-12-18 10:42:14 +05:30
Prashant Shahi	2b5a0ec496	Merge pull request #6625 from SigNoz/release/v0.62.x Release/v0.62.x	2024-12-12 21:02:17 +05:30
Prashant Shahi	a9440c010c	chore(signoz): pin versions: SigNoz 0.62.0, SigNoz OtelCollector 0.111.15 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-12-12 15:28:09 +05:30
Prashant Shahi	f9e7eff357	Merge branch 'main' into release/v0.62.x	2024-12-12 15:22:47 +05:30
Prashant Shahi	47d8c9e3e7	Merge pull request #6593 from SigNoz/release-sync/v0.61.x Release Sync/v0.61.x	2024-12-04 21:28:47 +05:30