feat: enable new logs schema by default (#6077 )

#### Summary - update all docker compose YAMLs to use new logs schema - enable `use_new_schema ` flag for clickhouselogsexporter in otel-collector config YAMLs - remove prefer delta from docker-compose YAMLs --------- Signed-off-by: Prashant Shahi <prashant@signoz.io>
Merge pull request #6075 from SigNoz/release/v0.55.x
2024-09-26 00:31:06 +05:30 · 2024-09-25 22:25:37 +05:30 · 2024-09-25 21:46:12 +05:30 · 2024-09-25 21:43:48 +05:30 · 2024-09-11 15:33:09 +05:30 · 2024-09-11 13:47:02 +05:30
11 changed files with 30 additions and 29 deletions
--- a/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
+++ b/deploy/docker-swarm/clickhouse-setup/docker-compose.yaml
@@ -146,11 +146,11 @@ services:
        condition: on-failure

  query-service:
-    image: signoz/query-service:0.49.1
+    image: signoz/query-service:0.55.0
    command:
      [
        "-config=/root/config/prometheus.yml",
-        # "--prefer-delta=true"
+        "--use-logs-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -186,7 +186,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:0.48.0
+    image: signoz/frontend:0.55.0
    deploy:
      restart_policy:
        condition: on-failure
@@ -199,7 +199,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector:
-    image: signoz/signoz-otel-collector:0.102.2
+    image: signoz/signoz-otel-collector:0.102.10
    command:
      [
        "--config=/etc/otel-collector-config.yaml",
@@ -238,7 +238,7 @@ services:
      - query-service

  otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.102.2
+      image: signoz/signoz-schema-migrator:0.102.10
      deploy:
        restart_policy:
          condition: on-failure
--- a/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
+++ b/deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml
@@ -144,6 +144,7 @@ exporters:
    dsn: tcp://clickhouse:9000/signoz_logs
    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
    timeout: 10s
+    use_new_schema: true
 extensions:
  health_check:
    endpoint: 0.0.0.0:13133
--- a/deploy/docker/clickhouse-setup/docker-compose-core.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose-core.yaml
@@ -66,7 +66,7 @@ services:
      - --storage.path=/data

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.10}
    container_name: otel-migrator
    command:
      - "--dsn=tcp://clickhouse:9000"
@@ -81,7 +81,7 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
  otel-collector:
    container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.102.2
+    image: signoz/signoz-otel-collector:0.102.10
    command:
      [
        "--config=/etc/otel-collector-config.yaml",
--- a/deploy/docker/clickhouse-setup/docker-compose-local.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose-local.yaml
@@ -25,7 +25,7 @@ services:
    command:
      [
        "-config=/root/config/prometheus.yml",
-        # "--prefer-delta=true"
+        "--use-logs-new-schema=true"
      ]
    ports:
      - "6060:6060"
--- a/deploy/docker/clickhouse-setup/docker-compose.testing.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose.testing.yaml
@@ -164,13 +164,13 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`

  query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.49.1}
+    image: signoz/query-service:${DOCKER_TAG:-0.55.0}
    container_name: signoz-query-service
    command:
      [
        "-config=/root/config/prometheus.yml",
-        "-gateway-url=https://api.staging.signoz.cloud"
-        # "--prefer-delta=true"
+        "-gateway-url=https://api.staging.signoz.cloud",
+        "--use-logs-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -204,7 +204,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.49.1}
+    image: signoz/frontend:${DOCKER_TAG:-0.55.0}
    container_name: signoz-frontend
    restart: on-failure
    depends_on:
@@ -216,7 +216,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.10}
    container_name: otel-migrator
    command:
      - "--dsn=tcp://clickhouse:9000"
@@ -230,7 +230,7 @@ services:


  otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.10}
    container_name: signoz-otel-collector
    command:
      [
--- a/deploy/docker/clickhouse-setup/docker-compose.yaml
+++ b/deploy/docker/clickhouse-setup/docker-compose.yaml
@@ -164,12 +164,12 @@ services:
  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`

  query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.49.1}
+    image: signoz/query-service:${DOCKER_TAG:-0.55.0}
    container_name: signoz-query-service
    command:
      [
-        "-config=/root/config/prometheus.yml"
-        # "--prefer-delta=true"
+        "-config=/root/config/prometheus.yml",
+        "--use-logs-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -203,7 +203,7 @@ services:
    <<: *db-depend

  frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.49.1}
+    image: signoz/frontend:${DOCKER_TAG:-0.55.0}
    container_name: signoz-frontend
    restart: on-failure
    depends_on:
@@ -215,7 +215,7 @@ services:
      - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.10}
    container_name: otel-migrator
    command:
      - "--dsn=tcp://clickhouse:9000"
@@ -229,7 +229,7 @@ services:


  otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.10}
    container_name: signoz-otel-collector
    command:
      [
--- a/deploy/docker/clickhouse-setup/otel-collector-config.yaml
+++ b/deploy/docker/clickhouse-setup/otel-collector-config.yaml
@@ -154,6 +154,7 @@ exporters:
    dsn: tcp://clickhouse:9000/signoz_logs
    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
    timeout: 10s
+    use_new_schema: true
  # logging: {}

 service:
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.23.2
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.102.2
+	github.com/SigNoz/signoz-otel-collector v0.102.10
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3
@@ -30,7 +30,6 @@ require (
 	github.com/mailru/easyjson v0.7.7
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
 	github.com/minio/minio-go/v6 v6.0.57
-	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c
 	github.com/oklog/oklog v0.3.2
 	github.com/open-telemetry/opamp-go v0.5.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.102.0
@@ -141,6 +140,7 @@ require (
 	github.com/minio/sha256-simd v0.1.1 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
--- a/go.sum
+++ b/go.sum
@@ -64,8 +64,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.11.1 h1:roM8ugYf4UxaeKKujEeBvoX7ybq3IrS+TB26KiRtIJg=
 github.com/SigNoz/prometheus v1.11.1/go.mod h1:uv4mQwZQtx7y4GQ6EdHOi8Wsk07uHNn2XHd1zM85m6I=
-github.com/SigNoz/signoz-otel-collector v0.102.2 h1:SmjsBZjMjTVVpuOlfJXlsDJQbdefQP/9Wz3CyzSuZuU=
-github.com/SigNoz/signoz-otel-collector v0.102.2/go.mod h1:ISAXYhZenojCWg6CdDJtPMpfS6Zwc08+uoxH25tc6Y0=
+github.com/SigNoz/signoz-otel-collector v0.102.10 h1:1zjU31OcRZL6fS0IIag8LA8bdhP4S28dzovDwuOg7Lg=
+github.com/SigNoz/signoz-otel-collector v0.102.10/go.mod h1:APoBVD4aRu9vIny1vdzZSi2wPY3elyjHA/I/rh1hKfs=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=
@@ -714,8 +714,6 @@ github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/srikanthccv/ClickHouse-go-mock v0.8.0 h1:DeeM8XLbTFl6sjYPPwazPEXx7kmRV8TgPFVkt1SqT0Y=
-github.com/srikanthccv/ClickHouse-go-mock v0.8.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24=
 github.com/srikanthccv/ClickHouse-go-mock v0.9.0 h1:XKr1Tb7GL1HlifKH874QGR3R6l0e6takXasROUiZawU=
 github.com/srikanthccv/ClickHouse-go-mock v0.9.0/go.mod h1:pgJm+apjvi7FHxEdgw1Bt4MRbUYpVxyhKQ/59Wkig24=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
--- a/pkg/query-service/tests/test-deploy/docker-compose.yaml
+++ b/pkg/query-service/tests/test-deploy/docker-compose.yaml
@@ -158,7 +158,7 @@ services:
    command:
      [
        "-config=/root/config/prometheus.yml",
-        # "--prefer-delta=true"
+        "--use-logs-new-schema=true"
      ]
    # ports:
    #   - "6060:6060"     # pprof port
@@ -192,7 +192,7 @@ services:
    <<: *db-depend

  otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.2}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.10}
    container_name: otel-migrator
    command:
      - "--dsn=tcp://clickhouse:9000"
@@ -205,7 +205,7 @@ services:
      #   condition: service_healthy

  otel-collector:
-    image: signoz/signoz-otel-collector:0.102.2
+    image: signoz/signoz-otel-collector:0.102.10
    container_name: signoz-otel-collector
    command:
      [
--- a/pkg/query-service/tests/test-deploy/otel-collector-config.yaml
+++ b/pkg/query-service/tests/test-deploy/otel-collector-config.yaml
@@ -115,6 +115,7 @@ exporters:
    dsn: tcp://clickhouse:9000/signoz_logs
    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
    timeout: 10s
+    use_new_schema: true
  # logging: {}

 service:
Author	SHA1	Message	Date
Prashant Shahi	87499d1ead	feat: enable new logs schema by default (#6077 ) #### Summary - update all docker compose YAMLs to use new logs schema - enable `use_new_schema ` flag for clickhouselogsexporter in otel-collector config YAMLs - remove prefer delta from docker-compose YAMLs --------- Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-09-26 00:31:06 +05:30
Prashant Shahi	5fa8686fcf	Merge pull request #6075 from SigNoz/release/v0.55.x Release/v0.55.x	2024-09-25 22:25:37 +05:30
Prashant Shahi	dc2db524c7	chore(signoz): 📌 pin versions: SigNoz 0.55.0, SigNoz OtelCollector 0.102.10 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-09-25 21:46:12 +05:30
Prashant Shahi	b3545b767a	Merge branch 'main' into release/v0.55.x	2024-09-25 21:43:48 +05:30
Prashant Shahi	08f3b089f4	Merge pull request #5922 from SigNoz/release/v0.54.x Release/v0.54.x	2024-09-11 15:33:09 +05:30
Prashant Shahi	1d8e5b6c0f	chore(signoz): 📌 pin versions: SigNoz 0.54.0, SigNoz OtelCollector 0.102.8 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-09-11 13:47:02 +05:30
Prashant Shahi	0dcded59e5	Merge branch 'main' into release/v0.54.x	2024-09-11 13:45:00 +05:30
Prashant Shahi	262beef8f9	Merge pull request #5800 from SigNoz/release/v0.53.x Release/v0.53.x	2024-08-30 15:20:27 +05:30
Prashant Shahi	43cc6dea92	chore(signoz): 📌 pin versions: SigNoz OtelCollector 0.102.7 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-08-30 15:06:52 +05:30
Prashant Shahi	6684640abe	Merge branch 'develop' into release/v0.53.x	2024-08-30 12:50:35 +05:30
Prashant Shahi	0a146910d6	chore(signoz): 📌 pin versions: SigNoz 0.53.0, SigNoz OtelCollector 0.102.6 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-08-29 19:25:34 +05:30
Prashant Shahi	690ed0f7f1	Merge branch 'main' into release/v0.53.x	2024-08-29 19:23:57 +05:30
Prashant Shahi	5bcf7de440	Merge pull request #5704 from SigNoz/release/v0.52.x Release/v0.52.x	2024-08-16 21:00:32 +05:30
Prashant Shahi	703983a5f9	chore(signoz): 📌 pin versions: SigNoz 0.52.0, SigNoz OtelCollector 0.102.3 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-08-15 23:12:36 +05:30
Prashant Shahi	766a2123c5	Merge branch 'main' into release/v0.52.x	2024-08-15 13:42:02 +05:30
Prashant Shahi	a476c68f7e	Merge pull request #5618 from SigNoz/release/v0.51.x Release/v0.51.x	2024-07-31 22:30:09 +05:30
Prashant Shahi	fc15aa6f1c	Merge branch 'develop' into release/v0.51.x	2024-07-31 21:29:07 +05:30
Prashant Shahi	4192fd573d	chore(signoz): 📌 pin versions: SigNoz 0.51.0, SigNoz OtelCollector 0.102.3 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-07-31 20:29:09 +05:30
Prashant Shahi	ca13d80205	Merge branch 'main' into release/v0.51.x	2024-07-31 20:27:51 +05:30
Prashant Shahi	8d84ce8f06	Merge pull request #5509 from SigNoz/release/v0.50.x Release/v0.50.x	2024-07-17 20:16:19 +05:30
Prashant Shahi	09ea7b9eb5	chore(signoz): 📌 pin versions: SigNoz 0.50.0 Signed-off-by: Prashant Shahi <prashant@signoz.io>	2024-07-17 19:01:03 +05:30