Merge branch 'main' into release/v0.19.0

Signed-off-by: Prashant Shahi <prashant@signoz.io>

.github/CODEOWNERS

@@ -2,6 +2,6 @@
 # Owners are automatically requested for review for PRs that changes code
 # that they own.
 * @ankitnayan
-/frontend/ @palashgdev @pranshuchittora
+/frontend/ @palashgdev
 /deploy/ @prashant-shahi
-/pkg/query-service/ @srikanthccv
+**/query-service/ @srikanthccv

.github/workflows/build.yaml

@@ -32,6 +32,10 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: Run tests
+        shell: bash
+        run: |
+          make test
       - name: Build query-service image
         shell: bash
         run: |

.github/workflows/commitlint.yml

@@ -7,12 +7,7 @@ jobs:
   lint-commits:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.1
+      - uses: actions/checkout@v3
         with:
-          # we actually need "github.event.pull_request.commits + 1" commit
           fetch-depth: 0
-      - uses: actions/setup-node@v2.1.0
-      # or just "yarn" if you depend on "@commitlint/cli" already
-      - run: yarn add @commitlint/cli
-      - run: yarn add @commitlint/config-conventional
-      - run: yarn run commitlint --config ./node_modules/@commitlint/config-conventional/index.js  --from HEAD~${{ github.event.pull_request.commits }} --to HEAD
+      - uses: wagoid/commitlint-github-action@v5

.github/workflows/pr_verify_linked_issue.yml

@@ -5,7 +5,7 @@ name: VerifyIssue
 
 on:
   pull_request:
-    types: [edited, synchronize, opened, reopened]
+    types: [edited, opened]
   check_run:
 
 jobs:
@@ -14,7 +14,6 @@ jobs:
     name: Ensure Pull Request has a linked issue.
     steps:
       - name: Verify Linked Issue
-        uses: hattan/verify-linked-issue-action@v1.1.0
+        uses: srikanthccv/verify-linked-issue-action@v0.70
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      

.github/workflows/remove-label.yaml

@@ -11,6 +11,6 @@ jobs:
       - name: Remove label
         uses: buildsville/add-remove-label@v1
         with:
-          label: ok-to-test
+          label: ok-to-test,testing-deploy
           type: remove
           token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/repo-stats.yml

@@ -1,25 +0,0 @@
-on:
-  schedule:
-    # Run this once per day, towards the end of the day for keeping the most
-    # recent data point most meaningful (hours are interpreted in UTC).
-    - cron: "0 8 * * *"
-  workflow_dispatch: # Allow for running this manually.
-
-jobs:
-  j1:
-    name: repostats
-    runs-on: ubuntu-latest
-    steps:
-      - name: run-ghrs
-        uses: jgehrcke/github-repo-stats@v1.1.0
-        with:
-          # Define the stats repository (the repo to fetch
-          # stats for and to generate the report for).
-          # Remove the parameter when the stats repository
-          # and the data repository are the same.
-          repository: signoz/signoz
-          # Set a GitHub API token that can read the stats
-          # repository, and that can push to the data
-          # repository (which this workflow file lives in),
-          # to store data and the report files.
-          ghtoken: ${{ github.token }}

.github/workflows/sonar.yml

@@ -24,4 +24,3 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-      

.github/workflows/staging-deployment.yaml

@@ -0,0 +1,40 @@
+name: staging-deployment
+# Trigger deployment only on push to develop branch
+on:
+  push:
+    branches:
+      - develop
+jobs:
+  deploy:
+    name: Deploy latest develop branch to staging
+    runs-on: ubuntu-latest
+    environment: staging
+    steps:
+      - name: Executing remote ssh commands using ssh key
+        uses: appleboy/ssh-action@v0.1.8
+        env:
+          GITHUB_BRANCH: develop
+          GITHUB_SHA: ${{ github.sha }}
+        with:
+          host: ${{ secrets.HOST_DNS }}
+          username: ${{ secrets.USERNAME }}
+          key: ${{ secrets.SSH_KEY }}
+          envs: GITHUB_BRANCH,GITHUB_SHA
+          command_timeout: 60m
+          script: |
+            echo "GITHUB_BRANCH: ${GITHUB_BRANCH}"
+            echo "GITHUB_SHA: ${GITHUB_SHA}"
+            export DOCKER_TAG="${GITHUB_SHA:0:7}" # needed for child process to access it
+            export OTELCOL_TAG="main"
+            docker system prune --force
+            docker pull signoz/signoz-otel-collector:main
+            cd ~/signoz
+            git status
+            git add .
+            git stash push -m "stashed on $(date --iso-8601=seconds)"
+            git fetch origin
+            git checkout ${GITHUB_BRANCH}
+            git pull
+            make build-ee-query-service-amd64
+            make build-frontend-amd64
+            make run-signoz

.github/workflows/testing-deployment.yaml

@@ -0,0 +1,39 @@
+name: testing-deployment
+# Trigger deployment only on testing-deploy label on pull request
+on:
+  pull_request:
+    types: [labeled]
+jobs:
+  deploy:
+    name: Deploy PR branch to testing
+    runs-on: ubuntu-latest
+    environment: testing
+    if: ${{ github.event.label.name == 'testing-deploy' }}
+    steps:
+      - name: Executing remote ssh commands using ssh key
+        uses: appleboy/ssh-action@v0.1.8
+        env:
+          GITHUB_BRANCH: ${{ github.head_ref || github.ref_name }}
+          GITHUB_SHA: ${{ github.sha }}
+        with:
+          host: ${{ secrets.HOST_DNS }}
+          username: ${{ secrets.USERNAME }}
+          key: ${{ secrets.SSH_KEY }}
+          envs: GITHUB_BRANCH,GITHUB_SHA
+          command_timeout: 60m
+          script: |
+            echo "GITHUB_BRANCH: ${GITHUB_BRANCH}"
+            echo "GITHUB_SHA: ${GITHUB_SHA}"
+            export DOCKER_TAG="${GITHUB_SHA:0:7}" # needed for child process to access it
+            export DEV_BUILD="1"
+            docker system prune --force
+            cd ~/signoz
+            git status
+            git add .
+            git stash push -m "stashed on $(date --iso-8601=seconds)"
+            git fetch origin
+            git checkout ${GITHUB_BRANCH}
+            git pull
+            make build-ee-query-service-amd64
+            make build-frontend-amd64
+            make run-signoz

.gitignore

@@ -1,7 +1,5 @@
 
 node_modules
-yarn.lock
-package.json
 
 deploy/docker/environment_tiny/common_test
 frontend/node_modules
@@ -53,3 +51,5 @@ ee/query-service/tests/test-deploy/data/
 /deploy/docker/clickhouse-setup/data/
 /deploy/docker-swarm/clickhouse-setup/data/
 bin/
+
+*/query-service/queries.active

CONTRIBUTING.md

@@ -215,9 +215,26 @@ Please ping us in the [`#contributing`](https://signoz-community.slack.com/archi
 
 # 4. Contribute to Backend (Query-Service) 🌑
 
-[**https://github.com/SigNoz/signoz/tree/develop/pkg/query-service**](https://github.com/SigNoz/signoz/tree/develop/pkg/query-service)
+**Need to Update: [https://github.com/SigNoz/signoz/tree/develop/pkg/query-service](https://github.com/SigNoz/signoz/tree/develop/pkg/query-service)**
 
-## 4.1 To run ClickHouse setup (recommended for local development)
+## 4.1 Prerequisites
+
+### 4.1.1 Install SQLite3
+
+- Run `sqlite3` command to check if you already have SQLite3 installed on your machine.
+
+- If not installed already, Install using below command
+  - on Linux
+    - on Debian / Ubuntu
+      ```
+        sudo apt install sqlite3
+      ```
+    - on CentOS / Fedora / RedHat
+      ```
+        sudo yum install sqlite3
+      ```
+
+## 4.2 To run ClickHouse setup (recommended for local development)
 
 - Clone the SigNoz repository and cd into signoz directory,
     ```

Makefile

@@ -45,7 +45,7 @@ build-frontend-amd64:
 	@echo "--> Building frontend docker image for amd64"
 	@echo "------------------"
 	@cd $(FRONTEND_DIRECTORY) && \
-	docker build --file Dockerfile  --no-cache -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \
+	docker build --file Dockerfile -t $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) \
 	--build-arg TARGETPLATFORM="linux/amd64" .
 
 # Step to build and push docker image of frontend(used in push pipeline)
@@ -54,7 +54,7 @@ build-push-frontend:
 	@echo "--> Building and pushing frontend docker image"
 	@echo "------------------"
 	@cd $(FRONTEND_DIRECTORY) && \
-	docker buildx build --file Dockerfile --progress plane --no-cache --push --platform linux/amd64 \
+	docker buildx build --file Dockerfile --progress plane --push --platform linux/arm64,linux/amd64 \
 	--tag $(REPONAME)/$(FRONTEND_DOCKER_IMAGE):$(DOCKER_TAG) .
 
 # Steps to build and push docker image of query service
@@ -65,7 +65,7 @@ build-query-service-amd64:
 	@echo "--> Building query-service docker image for amd64"
 	@echo "------------------"
 	@docker build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile \
-	--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
+	-t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
 	--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" .
 
 # Step to build and push docker image of query in amd64 and arm64 (used in push pipeline)
@@ -73,7 +73,7 @@ build-push-query-service:
 	@echo "------------------"
 	@echo "--> Building and pushing query-service docker image"
 	@echo "------------------"
-	@docker buildx build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile --progress plane --no-cache \
+	@docker buildx build --file $(QUERY_SERVICE_DIRECTORY)/Dockerfile --progress plane \
 	--push --platform linux/arm64,linux/amd64 --build-arg LD_FLAGS="$(LD_FLAGS)" \
 	--tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) .
 
@@ -84,11 +84,11 @@ build-ee-query-service-amd64:
 	@echo "------------------"
 	@if [ $(DEV_BUILD) != "" ]; then \
 		docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
-		--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
+		-t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
 		--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="${LD_FLAGS} ${DEV_LD_FLAGS}" .; \
 	else \
 		docker build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
-		--no-cache -t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
+		-t $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) \
 		--build-arg TARGETPLATFORM="linux/amd64" --build-arg LD_FLAGS="$(LD_FLAGS)" .; \
 	fi
 
@@ -98,7 +98,7 @@ build-push-ee-query-service:
 	@echo "--> Building and pushing query-service docker image"
 	@echo "------------------"
 	@docker buildx build --file $(EE_QUERY_SERVICE_DIRECTORY)/Dockerfile \
-	--progress plane --no-cache --push --platform linux/arm64,linux/amd64 \
+	--progress plane --push --platform linux/arm64,linux/amd64 \
 	--build-arg LD_FLAGS="$(LD_FLAGS)" --tag $(REPONAME)/$(QUERY_SERVICE_DOCKER_IMAGE):$(DOCKER_TAG) .
 
 dev-setup:
@@ -119,16 +119,23 @@ down-local:
 	$(STANDALONE_DIRECTORY)/docker-compose-core.yaml -f $(STANDALONE_DIRECTORY)/docker-compose-local.yaml \
 	down -v
 
-run-x86:
+pull-signoz:
+	@docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml pull
+
+run-signoz:
 	@docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml up --build -d
 
-down-x86:
+down-signoz:
 	@docker-compose -f $(STANDALONE_DIRECTORY)/docker-compose.yaml down -v
 
 clear-standalone-data:
 	@docker run --rm -v "$(PWD)/$(STANDALONE_DIRECTORY)/data:/pwd" busybox \
-	sh -c "cd /pwd && rm -rf alertmanager/* clickhouse/* signoz/*"
+	sh -c "cd /pwd && rm -rf alertmanager/* clickhouse*/* signoz/* zookeeper-*/*"
 
 clear-swarm-data:
 	@docker run --rm -v "$(PWD)/$(SWARM_DIRECTORY)/data:/pwd" busybox \
-	sh -c "cd /pwd && rm -rf alertmanager/* clickhouse/* signoz/*"
+	sh -c "cd /pwd && rm -rf alertmanager/* clickhouse*/* signoz/* zookeeper-*/*"
+
+test:
+	go test ./pkg/query-service/app/metrics/...
+	go test ./pkg/query-service/app/...

README.de-de.md

@@ -85,9 +85,9 @@ Hier findest du die vollständige Liste von unterstützten Programmiersprachen -
   
 ### Bereitstellung mit Docker
 
-Bitte folge den [hier](https://signoz.io/docs/deployment/docker/) aufgelisteten Schritten um deine Anwendung mit Docker bereitzustellen.
+Bitte folge den [hier](https://signoz.io/docs/install/docker/) aufgelisteten Schritten um deine Anwendung mit Docker bereitzustellen.
 
-Die [Anleitungen zur Fehlerbehebung](https://signoz.io/docs/deployment/troubleshooting) könnten hilfreich sein, falls du auf irgendwelche Schwierigkeiten stößt.
+Die [Anleitungen zur Fehlerbehebung](https://signoz.io/docs/install/troubleshooting/) könnten hilfreich sein, falls du auf irgendwelche Schwierigkeiten stößt.
 
 <p>&nbsp  </p>
   

README.md

@@ -23,7 +23,9 @@
 
 ##
 
-SigNoz helps developers monitor applications and troubleshoot problems in their deployed applications. SigNoz uses distributed tracing to gain visibility into your software stack.
+SigNoz helps developers monitor applications and troubleshoot problems in their deployed applications. With SigNoz, you can:
+
+👉 Visualise Metrics, Traces and Logs in a single pane of glass
 
 👉 You can see metrics like p99 latency, error rates for your services, external API calls and individual end points.
 
@@ -31,15 +33,43 @@ SigNoz helps developers monitor applications and troubleshoot problems in their
 
 👉 Run aggregates on trace data to get business relevant metrics
 
-![screenzy-1644432902955](https://user-images.githubusercontent.com/504541/153270713-1b2156e6-ec03-42de-975b-3c02b8ec1836.png)
-<br />
-![screenzy-1644432986784](https://user-images.githubusercontent.com/504541/153270725-0efb73b3-06ed-4207-bf13-9b7e2e17c4b8.png)
-<br />
-![screenzy-1647005040573](https://user-images.githubusercontent.com/504541/157875938-a3d57904-ea6d-4278-b929-bd1408d7f94c.png)
+👉 Filter and query logs, build dashboards and alerts based on attributes in logs
+
+👉 Record exceptions automatically in Python, Java, Ruby, and Javascript
+
+👉 Easy to set alerts with DIY query builder
+
+
+### Application Metrics
+
+![application_metrics](https://user-images.githubusercontent.com/83692067/226637410-900dbc5e-6705-4b11-a10c-bd0faeb2a92f.png)
+
+
+### Distributed Tracing
+<img width="2068" alt="distributed_tracing_2 2" src="https://user-images.githubusercontent.com/83692067/226536447-bae58321-6a22-4ed3-af80-e3e964cb3489.png">
+
+<img width="2068" alt="distributed_tracing_1" src="https://user-images.githubusercontent.com/83692067/226536462-939745b6-4f9d-45a6-8016-814837e7f7b4.png">
+
+### Logs Management
+
+<img width="2068" alt="logs_management" src="https://user-images.githubusercontent.com/83692067/226536482-b8a5c4af-b69c-43d5-969c-338bd5eaf1a5.png">
+
+### Infrastructure Monitoring
+
+<img width="2068" alt="infrastructure_monitoring" src="https://user-images.githubusercontent.com/83692067/226536496-f38c4dbf-e03c-4158-8be0-32d4a61158c7.png">
+
+### Exceptions Monitoring
+
+![exceptions_light](https://user-images.githubusercontent.com/83692067/226637967-4188d024-3ac9-4799-be95-f5ea9c45436f.png)
+
+
+### Alerts
+
+<img width="2068" alt="alerts_management" src="https://user-images.githubusercontent.com/83692067/226536548-2c81e2e8-c12d-47e8-bad7-c6be79055def.png">
+
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/Contributing.svg" width="50px" />
 
 ## Join our Slack community
 
@@ -47,20 +77,22 @@ Come say Hi to us on [Slack](https://signoz.io/slack) 👋
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/Features.svg" width="50px" />
 
 ## Features:
 
+- Unified UI for metrics, traces and logs. No need to switch from Prometheus to Jaeger to debug issues, or use a logs tool like Elastic separate from your metrics and traces stack. 
 - Application overview metrics like RPS, 50th/90th/99th Percentile latencies, and Error Rate
 - Slowest endpoints in your application
 - See exact request trace to figure out issues in downstream services, slow DB queries, call to 3rd party services like payment gateways, etc
 - Filter traces by service name, operation, latency, error, tags/annotations.
 - Run aggregates on trace data (events/spans) to get business relevant metrics. e.g. You can get error rate and 99th percentile latency of `customer_type: gold` or `deployment_version: v2` or `external_call: paypal`
-- Unified UI for metrics and traces. No need to switch from Prometheus to Jaeger to debug issues.
+- Native support for OpenTelemetry Logs, advanced log query builder, and automatic log collection from k8s cluster
+- Lightning quick log analytics ([Logs Perf. Benchmark](https://signoz.io/blog/logs-performance-benchmark/))
+- End-to-End visibility into infrastructure performance, ingest metrics from all kinds of host environments
+- Easy to set alerts with DIY query builder
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/WhatsCool.svg" width="50px" />
 
 ## Why SigNoz?
 
@@ -89,15 +121,14 @@ You can find the complete list of languages here - https://opentelemetry.io/docs
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/Philosophy.svg" width="50px" />
 
 ## Getting Started
 
 ### Deploy using Docker
 
-Please follow the steps listed [here](https://signoz.io/docs/deployment/docker/) to install using docker
+Please follow the steps listed [here](https://signoz.io/docs/install/docker/) to install using docker
 
-The [troubleshooting instructions](https://signoz.io/docs/deployment/troubleshooting) may be helpful if you face any issues.
+The [troubleshooting instructions](https://signoz.io/docs/install/troubleshooting/) may be helpful if you face any issues.
 
 <p>&nbsp  </p>
   
@@ -108,7 +139,6 @@ Please follow the steps listed [here](https://signoz.io/docs/deployment/helm_cha
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/UseSigNoz.svg" width="50px" />
 
 ## Comparisons to Familiar Tools
 
@@ -129,9 +159,27 @@ Moreover, SigNoz has few more advanced features wrt Jaeger:
 - Jaegar UI doesn’t show any metrics on traces or on filtered traces
 - Jaeger can’t get aggregates on filtered traces. For example, p99 latency of requests which have tag - customer_type='premium'. This can be done easily on SigNoz
 
+<p>&nbsp  </p>
+
+### SigNoz vs Elastic 
+
+- SigNoz Logs management are based on ClickHouse, a columnar OLAP datastore which makes aggregate log analytics queries much more efficient
+- 50% lower resource requirement compared to Elastic during ingestion
+
+We have published benchmarks comparing Elastic with SigNoz. Check it out [here](https://signoz.io/blog/logs-performance-benchmark/?utm_source=github-readme&utm_medium=logs-benchmark)
+
+<p>&nbsp  </p>
+
+### SigNoz vs Loki
+
+- SigNoz supports aggregations on high-cardinality data over a huge volume while loki doesn’t.
+- SigNoz supports indexes over high cardinality data and has no limitations on the number of indexes, while Loki reaches max streams with a few indexes added to it.
+- Searching over a huge volume of data is difficult and slow in Loki compared to SigNoz
+
+We have published benchmarks comparing Loki with SigNoz. Check it out [here](https://signoz.io/blog/logs-performance-benchmark/?utm_source=github-readme&utm_medium=logs-benchmark)
+
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/Contributors.svg" width="50px" />
 
 ## Contributing
 
@@ -158,7 +206,6 @@ Not sure how to get started? Just ping us on `#contributing` in our [slack commu
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/DevelopingLocally.svg" width="50px" />
 
 ## Documentation
 
@@ -166,7 +213,6 @@ You can find docs at https://signoz.io/docs/. If you need any clarification or f
 
 <br /><br />
 
-<img align="left" src="https://signoz-public.s3.us-east-2.amazonaws.com/Contributing.svg" width="50px" />
 
 ## Community
 

README.pt-br.md

@@ -84,9 +84,9 @@ Você pode encontrar a lista completa de linguagens aqui - https://opentelemetry
   
 ### Implantar usando Docker
 
-Siga as etapas listadas [aqui](https://signoz.io/docs/deployment/docker/) para instalar usando o Docker.
+Siga as etapas listadas [aqui](https://signoz.io/docs/install/docker/) para instalar usando o Docker.
 
-Esse [guia para solução de problemas](https://signoz.io/docs/deployment/troubleshooting) pode ser útil se você enfrentar quaisquer problemas. 
+Esse [guia para solução de problemas](https://signoz.io/docs/install/troubleshooting/) pode ser útil se você enfrentar quaisquer problemas. 
 
 <p>&nbsp  </p>
   

README.zh-cn.md

@@ -80,9 +80,9 @@ SigNoz帮助开发人员监控应用并排查已部署应用中的问题。SigNo
 
 ### 使用Docker部署
 
-请按照[这里](https://signoz.io/docs/deployment/docker/)列出的步骤使用Docker来安装
+请按照[这里](https://signoz.io/docs/install/docker/)列出的步骤使用Docker来安装
 
-如果你遇到任何问题，这个[排查指南](https://signoz.io/docs/deployment/troubleshooting)会对你有帮助。
+如果你遇到任何问题，这个[排查指南](https://signoz.io/docs/install/troubleshooting/)会对你有帮助。
 
 <p>&nbsp  </p>
 

deploy/README.md

@@ -27,12 +27,6 @@ For x86 chip (amd):
 docker-compose -f docker/clickhouse-setup/docker-compose.yaml up -d
 ```
 
-For Mac with Apple chip (arm):
-
-```sh
-docker-compose -f docker/clickhouse-setup/docker-compose.arm.yaml up -d
-```
-
 Open http://localhost:3301 in your favourite browser. In couple of minutes, you should see
 the data generated from hotrod in SigNoz UI.
 

deploy/docker-swarm/clickhouse-setup/clickhouse-cluster.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
+      -->
+    <zookeeper>
+        <node index="1">
+            <host>zookeeper-1</host>
+            <port>2181</port>
+        </node>
+        <!-- <node index="2">
+            <host>zookeeper-2</host>
+            <port>2181</port>
+        </node>
+        <node index="3">
+            <host>zookeeper-3</host>
+            <port>2181</port>
+        </node> -->
+    </zookeeper>
+
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.com/docs/en/operations/table_engines/distributed/
+      -->
+    <remote_servers>
+        <cluster>
+            <!-- Inter-server per-cluster secret for Distributed queries
+                 default: no secret (no authentication will be performed)
+
+                 If set, then Distributed queries will be validated on shards, so at least:
+                 - such cluster should exist on the shard,
+                 - such cluster should have the same secret.
+
+                 And also (and which is more important), the initial_user will
+                 be used as current user for the query.
+
+                 Right now the protocol is pretty simple and it only takes into account:
+                 - cluster name
+                 - query
+
+                 Also it will be nice if the following will be implemented:
+                 - source hostname (see interserver_http_host), but then it will depends from DNS,
+                   it can use IP address instead, but then the you need to get correct on the initiator node.
+                 - target hostname / ip address (same notes as for source hostname)
+                 - time-based security tokens
+            -->
+            <!-- <secret></secret> -->
+            <shard>
+                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+                <!-- <internal_replication>false</internal_replication> -->
+                <!-- Optional. Shard weight when writing data. Default: 1. -->
+                <!-- <weight>1</weight> -->
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
+                    <!-- <priority>1</priority> -->
+                </replica>
+            </shard>
+            <!-- <shard>
+                <replica>
+                    <host>clickhouse-2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse-3</host>
+                    <port>9000</port>
+                </replica>
+            </shard> -->
+        </cluster>
+    </remote_servers>
+</clickhouse>

deploy/docker-swarm/clickhouse-setup/clickhouse-config.xml

@@ -236,8 +236,8 @@
     <openSSL>
         <server> <!-- Used for https server AND secure tcp port -->
             <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
-            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
-            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <!-- <certificateFile>/etc/clickhouse-server/server.crt</certificateFile> -->
+            <!-- <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile> -->
             <!-- dhparams are optional. You can delete the <dhParamsFile> element.
                  To generate dhparams, use the following command:
                   openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
@@ -618,148 +618,6 @@
     </jdbc_bridge>
     -->
 
-    <!-- Configuration of clusters that could be used in Distributed tables.
-         https://clickhouse.com/docs/en/operations/table_engines/distributed/
-      -->
-    <remote_servers>
-        <!-- Test only shard config for testing distributed storage -->
-        <test_shard_localhost>
-            <!-- Inter-server per-cluster secret for Distributed queries
-                 default: no secret (no authentication will be performed)
-
-                 If set, then Distributed queries will be validated on shards, so at least:
-                 - such cluster should exist on the shard,
-                 - such cluster should have the same secret.
-
-                 And also (and which is more important), the initial_user will
-                 be used as current user for the query.
-
-                 Right now the protocol is pretty simple and it only takes into account:
-                 - cluster name
-                 - query
-
-                 Also it will be nice if the following will be implemented:
-                 - source hostname (see interserver_http_host), but then it will depends from DNS,
-                   it can use IP address instead, but then the you need to get correct on the initiator node.
-                 - target hostname / ip address (same notes as for source hostname)
-                 - time-based security tokens
-            -->
-            <!-- <secret></secret> -->
-
-            <shard>
-                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
-                <!-- <internal_replication>false</internal_replication> -->
-                <!-- Optional. Shard weight when writing data. Default: 1. -->
-                <!-- <weight>1</weight> -->
-                <replica>
-                    <host>localhost</host>
-                    <port>9000</port>
-                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
-                    <!-- <priority>1</priority> -->
-                </replica>
-            </shard>
-        </test_shard_localhost>
-        <test_cluster_one_shard_three_replicas_localhost>
-            <shard>
-                <internal_replication>false</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.3</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <!--shard>
-                <internal_replication>false</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.3</host>
-                    <port>9000</port>
-                </replica>
-            </shard-->
-        </test_cluster_one_shard_three_replicas_localhost>
-        <test_cluster_two_shards_localhost>
-             <shard>
-                 <replica>
-                     <host>localhost</host>
-                     <port>9000</port>
-                 </replica>
-             </shard>
-             <shard>
-                 <replica>
-                     <host>localhost</host>
-                     <port>9000</port>
-                 </replica>
-             </shard>
-        </test_cluster_two_shards_localhost>
-        <test_cluster_two_shards>
-            <shard>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster_two_shards>
-        <test_cluster_two_shards_internal_replication>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster_two_shards_internal_replication>
-        <test_shard_localhost_secure>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>9440</port>
-                    <secure>1</secure>
-                </replica>
-            </shard>
-        </test_shard_localhost_secure>
-        <test_unavailable_shard>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>1</port>
-                </replica>
-            </shard>
-        </test_unavailable_shard>
-    </remote_servers>
-
     <!-- The list of hosts allowed to use in URL-related storage engines and table functions.
         If this section is not present in configuration, all hosts are allowed.
     -->
@@ -786,29 +644,6 @@
          Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file.
       -->
 
-    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
-         Optional. If you don't use replicated tables, you could omit that.
-
-         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
-      -->
-
-    <!--
-    <zookeeper>
-        <node>
-            <host>example1</host>
-            <port>2181</port>
-        </node>
-        <node>
-            <host>example2</host>
-            <port>2181</port>
-        </node>
-        <node>
-            <host>example3</host>
-            <port>2181</port>
-        </node>
-    </zookeeper>
-    -->
-
     <!-- Substitutions for parameters of replicated tables.
           Optional. If you don't use replicated tables, you could omit that.
 

deploy/docker-swarm/clickhouse-setup/clickhouse-storage.xml

@@ -7,9 +7,21 @@
         </default>
         <s3>
             <type>s3</type>
-            <endpoint>https://BUCKET-NAME.s3.amazonaws.com/data/</endpoint>
+            <!-- For S3 cold storage,
+                    if region is us-east-1, endpoint can be https://<bucket-name>.s3.amazonaws.com
+                    if region is not us-east-1, endpoint should be https://<bucket-name>.s3-<region>.amazonaws.com
+                For GCS cold storage,
+                    endpoint should be https://storage.googleapis.com/<bucket-name>/data/
+                -->
+            <endpoint>https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/</endpoint>
             <access_key_id>ACCESS-KEY-ID</access_key_id>
             <secret_access_key>SECRET-ACCESS-KEY</secret_access_key>
+            <!-- In case of S3, uncomment the below configuration in case you want to read
+                AWS credentials from the Environment variables if they exist. -->
+            <!-- <use_environment_credentials>true</use_environment_credentials> -->
+            <!-- In case of GCS, uncomment the below configuration, since GCS does
+                not support batch deletion and result in error messages in logs. -->
+            <!-- <support_batch_delete>false</support_batch_delete> -->
         </s3>
    </disks>
    <policies>

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -1,33 +1,130 @@
 version: "3.9"
 
+x-clickhouse-defaults: &clickhouse-defaults
+  image: clickhouse/clickhouse-server:22.8.8-alpine
+  tty: true
+  deploy:
+    restart_policy:
+      condition: on-failure
+  depends_on:
+    - zookeeper-1
+    # - zookeeper-2
+    # - zookeeper-3
+  logging:
+    options:
+      max-size: 50m
+      max-file: "3"
+  healthcheck:
+    # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
+    test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
+    interval: 30s
+    timeout: 5s
+    retries: 3
+  ulimits:
+    nproc: 65535
+    nofile:
+      soft: 262144
+      hard: 262144
+
+x-clickhouse-depend: &clickhouse-depend
+  depends_on:
+    - clickhouse
+    # - clickhouse-2
+    # - clickhouse-3
+
 services:
+  zookeeper-1:
+    image: bitnami/zookeeper:3.7.1
+    hostname: zookeeper-1
+    user: root
+    ports:
+      - "2181:2181"
+      - "2888:2888"
+      - "3888:3888"
+    volumes:
+      - ./data/zookeeper-1:/bitnami/zookeeper
+    environment:
+      - ZOO_SERVER_ID=1
+      # - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_AUTOPURGE_INTERVAL=1
+
+  # zookeeper-2:
+  #   image: bitnami/zookeeper:3.7.0
+  #   hostname: zookeeper-2
+  #   user: root
+  #   ports:
+  #     - "2182:2181"
+  #     - "2889:2888"
+  #     - "3889:3888"
+  #   volumes:
+  #     - ./data/zookeeper-2:/bitnami/zookeeper
+  #   environment:
+  #     - ZOO_SERVER_ID=2
+  #     - ZOO_SERVERS=zookeeper-1:2888:3888,0.0.0.0:2888:3888,zookeeper-3:2888:3888
+  #     - ALLOW_ANONYMOUS_LOGIN=yes
+  #     - ZOO_AUTOPURGE_INTERVAL=1
+
+  # zookeeper-3:
+  #   image: bitnami/zookeeper:3.7.0
+  #   hostname: zookeeper-3
+  #   user: root
+  #   ports:
+  #     - "2183:2181"
+  #     - "2890:2888"
+  #     - "3890:3888"
+  #   volumes:
+  #     - ./data/zookeeper-3:/bitnami/zookeeper
+  #   environment:
+  #     - ZOO_SERVER_ID=3
+  #     - ZOO_SERVERS=zookeeper-1:2888:3888,zookeeper-2:2888:3888,0.0.0.0:2888:3888
+  #     - ALLOW_ANONYMOUS_LOGIN=yes
+  #     - ZOO_AUTOPURGE_INTERVAL=1
+
   clickhouse:
-    image: clickhouse/clickhouse-server:22.8.8-alpine
+    <<: *clickhouse-defaults
+    hostname: clickhouse
     # ports:
     #   - "9000:9000"
     #   - "8123:8123"
-    tty: true
+    #   - "9181:9181"
     volumes:
       - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
       - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+      - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
       # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
       - ./data/clickhouse/:/var/lib/clickhouse/
-    deploy:
-      restart_policy:
-        condition: on-failure
-    logging:
-      options:
-        max-size: 50m
-        max-file: "3"
-    healthcheck:
-      # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
+
+  # clickhouse-2:
+  #   <<: *clickhouse-defaults
+  #   hostname: clickhouse-2
+  #   ports:
+  #     - "9001:9000"
+  #     - "8124:8123"
+  #     - "9182:9181"
+  #   volumes:
+  #     - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+  #     - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+  #     - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+  #     # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+  #     - ./data/clickhouse-2/:/var/lib/clickhouse/
+
+  # clickhouse-3:
+  #   <<: *clickhouse-defaults
+  #   hostname: clickhouse-3
+  #   ports:
+  #     - "9002:9000"
+  #     - "8125:8123"
+  #     - "9183:9181"
+  #   volumes:
+  #     - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+  #     - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+  #     - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+  #     # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+  #     - ./data/clickhouse-3/:/var/lib/clickhouse/
 
   alertmanager:
-    image: signoz/alertmanager:0.23.0-0.2
+    image: signoz/alertmanager:0.23.1
     volumes:
       - ./data/alertmanager:/data
     command:
@@ -40,7 +137,7 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.11.3
+    image: signoz/query-service:0.19.0
     command: ["-config=/root/config/prometheus.yml"]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -59,18 +156,17 @@ services:
       - TELEMETRY_ENABLED=true
       - DEPLOYMENT_TYPE=docker-swarm
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"]
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"]
       interval: 30s
       timeout: 5s
       retries: 3
     deploy:
       restart_policy:
         condition: on-failure
-    depends_on:
-      - clickhouse
+    <<: *clickhouse-depend
 
   frontend:
-    image: signoz/frontend:0.11.3
+    image: signoz/frontend:0.19.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -83,14 +179,16 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-config.yaml"]
+    image: signoz/signoz-otel-collector:0.76.1
+    command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name={{.Node.Hostname}},os.type={{.Node.Platform.OS}},dockerswarm.service.name={{.Service.Name}},dockerswarm.task.name={{.Task.Name}}
+      - DOCKER_MULTI_NODE_CLUSTER=false
+      - LOW_CARDINAL_EXCEPTION_GROUPING=false
     ports:
       # - "1777:1777"     # pprof extension
       - "4317:4317"     # OTLP gRPC receiver
@@ -107,12 +205,11 @@ services:
       mode: global
       restart_policy:
         condition: on-failure
-    depends_on:
-      - clickhouse
+    <<: *clickhouse-depend
 
   otel-collector-metrics:
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-metrics-config.yaml"]
+    image: signoz/signoz-otel-collector:0.76.1
+    command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
     # ports:
@@ -123,8 +220,7 @@ services:
     deploy:
       restart_policy:
         condition: on-failure
-    depends_on:
-      - clickhouse
+    <<: *clickhouse-depend
 
   hotrod:
     image: jaegertracing/example-hotrod:1.30

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -64,7 +64,9 @@ receivers:
         - job_name: otel-collector
           static_configs:
           - targets:
-            - localhost:8888
+              - localhost:8888
+            labels:
+              job_name: otel-collector
 
 processors:
   batch:
@@ -73,17 +75,21 @@ processors:
     timeout: 10s
   resourcedetection:
     # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
-    detectors: [env, system] # include ec2 for AWS, gce for GCP and azure for Azure.
+    detectors: [env, system] # include ec2 for AWS, gcp for GCP and azure for Azure.
     timeout: 2s
   signozspanmetrics/prometheus:
     metrics_exporter: prometheus
     latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
-    dimensions_cache_size: 10000
+    dimensions_cache_size: 100000
     dimensions:
       - name: service.namespace
         default: default
       - name: deployment.environment
         default: default
+      # This is added to ensure the uniqueness of the timeseries
+      # Otherwise, identical timeseries produced by multiple replicas of
+      # collectors result in incorrect APM metrics
+      - name: 'signoz.collector.id'
   # memory_limiter:
   #   # 80% of maximum memory up to 2G
   #   limit_mib: 1500
@@ -103,15 +109,20 @@ processors:
 exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/?database=signoz_traces
+    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
+    low_cardinal_exception_grouping: ${LOW_CARDINAL_EXCEPTION_GROUPING}
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/?database=signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
+  clickhousemetricswrite/prometheus:
+    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
   prometheus:
     endpoint: 0.0.0.0:8889
   # logging: {}
   clickhouselogsexporter:
     dsn: tcp://clickhouse:9000/
+    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     timeout: 5s
     sending_queue:
       queue_size: 100
@@ -144,9 +155,13 @@ service:
       processors: [batch]
       exporters: [clickhousemetricswrite]
     metrics/generic:
-      receivers: [hostmetrics, prometheus]
+      receivers: [hostmetrics]
       processors: [resourcedetection, batch]
       exporters: [clickhousemetricswrite]
+    metrics/prometheus:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [clickhousemetricswrite/prometheus]
     metrics/spanmetrics:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]

deploy/docker-swarm/clickhouse-setup/otel-collector-metrics-config.yaml

@@ -7,7 +7,9 @@ receivers:
           scrape_interval: 60s
           static_configs:
             - targets:
-              - localhost:8888
+                - localhost:8888
+              labels:
+                job_name: otel-collector-metrics
         # SigNoz span metrics
         - job_name: signozspanmetrics-collector
           scrape_interval: 60s

deploy/docker-swarm/common/nginx-config.conf

@@ -30,6 +30,8 @@ server {
 
     location /api {
         proxy_pass http://query-service:8080/api;
+        # connection will be closed if no data is read for 600s between successive read operations
+        proxy_read_timeout 600s; 
     }
 
     # redirect server error pages to the static page /50x.html

deploy/docker/clickhouse-setup/clickhouse-cluster.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
+         Optional. If you don't use replicated tables, you could omit that.
+
+         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
+      -->
+    <zookeeper>
+        <node index="1">
+            <host>zookeeper-1</host>
+            <port>2181</port>
+        </node>
+        <!-- <node index="2">
+            <host>zookeeper-2</host>
+            <port>2181</port>
+        </node>
+        <node index="3">
+            <host>zookeeper-3</host>
+            <port>2181</port>
+        </node> -->
+    </zookeeper>
+
+    <!-- Configuration of clusters that could be used in Distributed tables.
+         https://clickhouse.com/docs/en/operations/table_engines/distributed/
+      -->
+    <remote_servers>
+        <cluster>
+            <!-- Inter-server per-cluster secret for Distributed queries
+                 default: no secret (no authentication will be performed)
+
+                 If set, then Distributed queries will be validated on shards, so at least:
+                 - such cluster should exist on the shard,
+                 - such cluster should have the same secret.
+
+                 And also (and which is more important), the initial_user will
+                 be used as current user for the query.
+
+                 Right now the protocol is pretty simple and it only takes into account:
+                 - cluster name
+                 - query
+
+                 Also it will be nice if the following will be implemented:
+                 - source hostname (see interserver_http_host), but then it will depends from DNS,
+                   it can use IP address instead, but then the you need to get correct on the initiator node.
+                 - target hostname / ip address (same notes as for source hostname)
+                 - time-based security tokens
+            -->
+            <!-- <secret></secret> -->
+            <shard>
+                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
+                <!-- <internal_replication>false</internal_replication> -->
+                <!-- Optional. Shard weight when writing data. Default: 1. -->
+                <!-- <weight>1</weight> -->
+                <replica>
+                    <host>clickhouse</host>
+                    <port>9000</port>
+                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
+                    <!-- <priority>1</priority> -->
+                </replica>
+            </shard>
+            <!-- <shard>
+                <replica>
+                    <host>clickhouse-2</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+            <shard>
+                <replica>
+                    <host>clickhouse-3</host>
+                    <port>9000</port>
+                </replica>
+            </shard> -->
+        </cluster>
+    </remote_servers>
+</clickhouse>

deploy/docker/clickhouse-setup/clickhouse-config.xml

@@ -236,8 +236,8 @@
     <openSSL>
         <server> <!-- Used for https server AND secure tcp port -->
             <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
-            <certificateFile>/etc/clickhouse-server/server.crt</certificateFile>
-            <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile>
+            <!-- <certificateFile>/etc/clickhouse-server/server.crt</certificateFile> -->
+            <!-- <privateKeyFile>/etc/clickhouse-server/server.key</privateKeyFile> -->
             <!-- dhparams are optional. You can delete the <dhParamsFile> element.
                  To generate dhparams, use the following command:
                   openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096
@@ -618,148 +618,6 @@
     </jdbc_bridge>
     -->
 
-    <!-- Configuration of clusters that could be used in Distributed tables.
-         https://clickhouse.com/docs/en/operations/table_engines/distributed/
-      -->
-    <remote_servers>
-        <!-- Test only shard config for testing distributed storage -->
-        <test_shard_localhost>
-            <!-- Inter-server per-cluster secret for Distributed queries
-                 default: no secret (no authentication will be performed)
-
-                 If set, then Distributed queries will be validated on shards, so at least:
-                 - such cluster should exist on the shard,
-                 - such cluster should have the same secret.
-
-                 And also (and which is more important), the initial_user will
-                 be used as current user for the query.
-
-                 Right now the protocol is pretty simple and it only takes into account:
-                 - cluster name
-                 - query
-
-                 Also it will be nice if the following will be implemented:
-                 - source hostname (see interserver_http_host), but then it will depends from DNS,
-                   it can use IP address instead, but then the you need to get correct on the initiator node.
-                 - target hostname / ip address (same notes as for source hostname)
-                 - time-based security tokens
-            -->
-            <!-- <secret></secret> -->
-
-            <shard>
-                <!-- Optional. Whether to write data to just one of the replicas. Default: false (write data to all replicas). -->
-                <!-- <internal_replication>false</internal_replication> -->
-                <!-- Optional. Shard weight when writing data. Default: 1. -->
-                <!-- <weight>1</weight> -->
-                <replica>
-                    <host>localhost</host>
-                    <port>9000</port>
-                    <!-- Optional. Priority of the replica for load_balancing. Default: 1 (less value has more priority). -->
-                    <!-- <priority>1</priority> -->
-                </replica>
-            </shard>
-        </test_shard_localhost>
-        <test_cluster_one_shard_three_replicas_localhost>
-            <shard>
-                <internal_replication>false</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.3</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <!--shard>
-                <internal_replication>false</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>127.0.0.3</host>
-                    <port>9000</port>
-                </replica>
-            </shard-->
-        </test_cluster_one_shard_three_replicas_localhost>
-        <test_cluster_two_shards_localhost>
-             <shard>
-                 <replica>
-                     <host>localhost</host>
-                     <port>9000</port>
-                 </replica>
-             </shard>
-             <shard>
-                 <replica>
-                     <host>localhost</host>
-                     <port>9000</port>
-                 </replica>
-             </shard>
-        </test_cluster_two_shards_localhost>
-        <test_cluster_two_shards>
-            <shard>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster_two_shards>
-        <test_cluster_two_shards_internal_replication>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>127.0.0.1</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>127.0.0.2</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster_two_shards_internal_replication>
-        <test_shard_localhost_secure>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>9440</port>
-                    <secure>1</secure>
-                </replica>
-            </shard>
-        </test_shard_localhost_secure>
-        <test_unavailable_shard>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-            <shard>
-                <replica>
-                    <host>localhost</host>
-                    <port>1</port>
-                </replica>
-            </shard>
-        </test_unavailable_shard>
-    </remote_servers>
-
     <!-- The list of hosts allowed to use in URL-related storage engines and table functions.
         If this section is not present in configuration, all hosts are allowed.
     -->
@@ -786,29 +644,6 @@
          Values for substitutions are specified in /clickhouse/name_of_substitution elements in that file.
       -->
 
-    <!-- ZooKeeper is used to store metadata about replicas, when using Replicated tables.
-         Optional. If you don't use replicated tables, you could omit that.
-
-         See https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/replication/
-      -->
-
-    <!--
-    <zookeeper>
-        <node>
-            <host>example1</host>
-            <port>2181</port>
-        </node>
-        <node>
-            <host>example2</host>
-            <port>2181</port>
-        </node>
-        <node>
-            <host>example3</host>
-            <port>2181</port>
-        </node>
-    </zookeeper>
-    -->
-
     <!-- Substitutions for parameters of replicated tables.
           Optional. If you don't use replicated tables, you could omit that.
 
@@ -1070,7 +905,8 @@
     <dictionaries_config>*_dictionary.xml</dictionaries_config>
 
     <!-- Configuration of user defined executable functions -->
-    <user_defined_executable_functions_config>*_function.xml</user_defined_executable_functions_config>
+    <user_defined_executable_functions_config>*function.xml</user_defined_executable_functions_config>
+    <user_scripts_path>/var/lib/clickhouse/user_scripts/</user_scripts_path>
 
     <!-- Uncomment if you want data to be compressed 30-100% better.
          Don't do that if you just started using ClickHouse.

deploy/docker/clickhouse-setup/clickhouse-storage.xml

@@ -7,9 +7,21 @@
         </default>
         <s3>
             <type>s3</type>
-            <endpoint>https://BUCKET-NAME.s3.amazonaws.com/data/</endpoint>
+            <!-- For S3 cold storage,
+                    if region is us-east-1, endpoint can be https://<bucket-name>.s3.amazonaws.com
+                    if region is not us-east-1, endpoint should be https://<bucket-name>.s3-<region>.amazonaws.com
+                For GCS cold storage,
+                    endpoint should be https://storage.googleapis.com/<bucket-name>/data/
+                -->
+            <endpoint>https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/</endpoint>
             <access_key_id>ACCESS-KEY-ID</access_key_id>
             <secret_access_key>SECRET-ACCESS-KEY</secret_access_key>
+            <!-- In case of S3, uncomment the below configuration in case you want to read
+                AWS credentials from the Environment variables if they exist. -->
+            <!-- <use_environment_credentials>true</use_environment_credentials> -->
+            <!-- In case of GCS, uncomment the below configuration, since GCS does
+                not support batch deletion and result in error messages in logs. -->
+            <!-- <support_batch_delete>false</support_batch_delete> -->
         </s3>
    </disks>
    <policies>

deploy/docker/clickhouse-setup/custom-function.xml

@@ -0,0 +1,21 @@
+<functions>
+    <function>
+        <type>executable</type>
+        <name>histogramQuantile</name>
+        <return_type>Float64</return_type>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>buckets</name>
+        </argument>
+        <argument>
+            <type>Array(Float64)</type>
+            <name>counts</name>
+        </argument>
+        <argument>
+            <type>Float64</type>
+            <name>quantile</name>
+        </argument>
+        <format>CSV</format>
+        <command>./histogramQuantile</command>
+    </function>
+</functions>

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -27,7 +27,7 @@ services:
 
   alertmanager:
     container_name: alertmanager
-    image: signoz/alertmanager:0.23.0-0.2
+    image: signoz/alertmanager:0.23.1
     volumes:
       - ./data/alertmanager:/data
     depends_on:
@@ -41,8 +41,8 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: otel-collector
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-config.yaml"]
+    image: signoz/signoz-otel-collector:0.76.1
+    command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     # user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
@@ -67,8 +67,8 @@ services:
 
   otel-collector-metrics:
     container_name: otel-collector-metrics
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-metrics-config.yaml"]
+    image: signoz/signoz-otel-collector:0.76.1
+    command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
     # ports:

deploy/docker/clickhouse-setup/docker-compose-local.yaml

@@ -28,7 +28,7 @@ services:
       - "8080:8080"
     restart: on-failure
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"]
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"]
       interval: 30s
       timeout: 5s
       retries: 3

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -1,31 +1,145 @@
 version: "2.4"
 
+x-clickhouse-defaults: &clickhouse-defaults
+  restart: on-failure
+  image: clickhouse/clickhouse-server:22.8.8-alpine
+  tty: true
+  depends_on:
+    - zookeeper-1
+    # - zookeeper-2
+    # - zookeeper-3
+  logging:
+    options:
+      max-size: 50m
+      max-file: "3"
+  healthcheck:
+    # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
+    test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
+    interval: 30s
+    timeout: 5s
+    retries: 3
+  ulimits:
+    nproc: 65535
+    nofile:
+      soft: 262144
+      hard: 262144
+
+x-clickhouse-depend: &clickhouse-depend
+  depends_on:
+    clickhouse:
+      condition: service_healthy
+    # clickhouse-2:
+    #   condition: service_healthy
+    # clickhouse-3:
+    #   condition: service_healthy
+
 services:
+
+  zookeeper-1:
+    image: bitnami/zookeeper:3.7.1
+    container_name: zookeeper-1
+    hostname: zookeeper-1
+    user: root
+    ports:
+      - "2181:2181"
+      - "2888:2888"
+      - "3888:3888"
+    volumes:
+      - ./data/zookeeper-1:/bitnami/zookeeper
+    environment:
+      - ZOO_SERVER_ID=1
+      # - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888
+      - ALLOW_ANONYMOUS_LOGIN=yes
+      - ZOO_AUTOPURGE_INTERVAL=1
+
+  # zookeeper-2:
+  #   image: bitnami/zookeeper:3.7.0
+  #   container_name: zookeeper-2
+  #   hostname: zookeeper-2
+  #   user: root
+  #   ports:
+  #     - "2182:2181"
+  #     - "2889:2888"
+  #     - "3889:3888"
+  #   volumes:
+  #     - ./data/zookeeper-2:/bitnami/zookeeper
+  #   environment:
+  #     - ZOO_SERVER_ID=2
+  #     - ZOO_SERVERS=zookeeper-1:2888:3888,0.0.0.0:2888:3888,zookeeper-3:2888:3888
+  #     - ALLOW_ANONYMOUS_LOGIN=yes
+  #     - ZOO_AUTOPURGE_INTERVAL=1
+
+  # zookeeper-3:
+  #   image: bitnami/zookeeper:3.7.0
+  #   container_name: zookeeper-3
+  #   hostname: zookeeper-3
+  #   user: root
+  #   ports:
+  #     - "2183:2181"
+  #     - "2890:2888"
+  #     - "3890:3888"
+  #   volumes:
+  #     - ./data/zookeeper-3:/bitnami/zookeeper
+  #   environment:
+  #     - ZOO_SERVER_ID=3
+  #     - ZOO_SERVERS=zookeeper-1:2888:3888,zookeeper-2:2888:3888,0.0.0.0:2888:3888
+  #     - ALLOW_ANONYMOUS_LOGIN=yes
+  #     - ZOO_AUTOPURGE_INTERVAL=1
+
   clickhouse:
-    image: clickhouse/clickhouse-server:22.8.8-alpine
-    # ports:
-    # - "9000:9000"
-    # - "8123:8123"
-    tty: true
+    <<: *clickhouse-defaults
+    container_name: clickhouse
+    hostname: clickhouse
+    ports:
+      - "9000:9000"
+      - "8123:8123"
+      - "9181:9181"
     volumes:
       - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
       - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+      - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+      - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
       # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
       - ./data/clickhouse/:/var/lib/clickhouse/
-    restart: on-failure
-    logging:
-      options:
-        max-size: 50m
-        max-file: "3"
-    healthcheck:
-      # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
-      interval: 30s
-      timeout: 5s
-      retries: 3
+      - ./user_scripts:/var/lib/clickhouse/user_scripts/
+
+  # clickhouse-2:
+  #   <<: *clickhouse-defaults
+  #   container_name: clickhouse-2
+  #   hostname: clickhouse-2
+  #   ports:
+  #     - "9001:9000"
+  #     - "8124:8123"
+  #     - "9182:9181"
+  #   volumes:
+  #     - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+  #     - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+  #     - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+  #     - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+  #     # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+  #     - ./data/clickhouse-2/:/var/lib/clickhouse/
+  #     - ./user_scripts:/var/lib/clickhouse/user_scripts/
+
+  
+  # clickhouse-3:
+  #   <<: *clickhouse-defaults
+  #   container_name: clickhouse-3
+  #   hostname: clickhouse-3
+  #   ports:
+  #     - "9002:9000"
+  #     - "8125:8123"
+  #     - "9183:9181"
+  #   volumes:
+  #     - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+  #     - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+  #     - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+  #     - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+  #     # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+  #     - ./data/clickhouse-3/:/var/lib/clickhouse/
+  #     - ./user_scripts:/var/lib/clickhouse/user_scripts/
 
   alertmanager:
-    image: signoz/alertmanager:0.23.0-0.2
+    image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.1}
     volumes:
       - ./data/alertmanager:/data
     depends_on:
@@ -36,10 +150,10 @@ services:
       - --queryService.url=http://query-service:8085
       - --storage.path=/data
 
-# Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
+  # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:0.11.3
+    image: signoz/query-service:${DOCKER_TAG:-0.19.0}
     container_name: query-service
     command: ["-config=/root/config/prometheus.yml"]
     # ports:
@@ -60,16 +174,14 @@ services:
       - DEPLOYMENT_TYPE=docker-standalone-amd
     restart: on-failure
     healthcheck:
-      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/version"]
+      test: ["CMD", "wget", "--spider", "-q", "localhost:8080/api/v1/health"]
       interval: 30s
       timeout: 5s
       retries: 3
-    depends_on:
-      clickhouse:
-        condition: service_healthy
+    <<: *clickhouse-depend
 
   frontend:
-    image: signoz/frontend:0.11.3
+    image: signoz/frontend:${DOCKER_TAG:-0.19.0}
     container_name: frontend
     restart: on-failure
     depends_on:
@@ -81,14 +193,16 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-config.yaml"]
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.76.1}
+    command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     user: root # required for reading docker container logs
     volumes:
       - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
       - /var/lib/docker/containers:/var/lib/docker/containers:ro
     environment:
       - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
+      - DOCKER_MULTI_NODE_CLUSTER=false
+      - LOW_CARDINAL_EXCEPTION_GROUPING=false
     ports:
       # - "1777:1777"     # pprof extension
       - "4317:4317"     # OTLP gRPC receiver
@@ -102,13 +216,11 @@ services:
       # - "55678:55678"   # OpenCensus receiver
       # - "55679:55679"   # zPages extension
     restart: on-failure
-    depends_on:
-      clickhouse:
-        condition: service_healthy
+    <<: *clickhouse-depend
 
   otel-collector-metrics:
-    image: signoz/signoz-otel-collector:0.63.0
-    command: ["--config=/etc/otel-collector-metrics-config.yaml"]
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.76.1}
+    command: ["--config=/etc/otel-collector-metrics-config.yaml", "--feature-gates=-pkg.translator.prometheus.NormalizeName"]
     volumes:
       - ./otel-collector-metrics-config.yaml:/etc/otel-collector-metrics-config.yaml
     # ports:
@@ -117,20 +229,18 @@ services:
     #   - "13133:13133"   # Health check extension
     #   - "55679:55679"   # zPages extension
     restart: on-failure
-    depends_on:
-      clickhouse:
-        condition: service_healthy
+    <<: *clickhouse-depend
 
   hotrod:
-   image: jaegertracing/example-hotrod:1.30
-   container_name: hotrod
-   logging:
-     options:
-       max-size: 50m
-       max-file: "3"
-   command: ["all"]
-   environment:
-     - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
+    image: jaegertracing/example-hotrod:1.30
+    container_name: hotrod
+    logging:
+      options:
+        max-size: 50m
+        max-file: "3"
+    command: ["all"]
+    environment:
+      - JAEGER_ENDPOINT=http://otel-collector:14268/api/traces
 
   load-hotrod:
     image: "grubykarol/locust:1.2.3-python3.9-alpine3.12"

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -64,9 +64,46 @@ receivers:
         - job_name: otel-collector
           static_configs:
           - targets:
-            - localhost:8888
+              - localhost:8888
+            labels:
+              job_name: otel-collector
+
 
 processors:
+  logstransform/internal:
+    operators:
+      - type: trace_parser
+        if: '"trace_id" in attributes or "span_id" in attributes'
+        trace_id:
+          parse_from: attributes.trace_id
+        span_id:
+          parse_from: attributes.span_id
+        output: remove_trace_id
+      - type: trace_parser
+        if: '"traceId" in attributes or "spanId" in attributes'
+        trace_id:
+          parse_from: attributes.traceId
+        span_id:
+          parse_from: attributes.spanId
+        output: remove_traceId
+      - id: remove_traceId
+        type: remove
+        if: '"traceId" in attributes'
+        field: attributes.traceId
+        output: remove_spanId
+      - id: remove_spanId
+        type: remove
+        if: '"spanId" in attributes'
+        field: attributes.spanId
+      - id: remove_trace_id
+        type: remove
+        if: '"trace_id" in attributes'
+        field: attributes.trace_id
+        output: remove_span_id
+      - id: remove_span_id
+        type: remove
+        if: '"span_id" in attributes'
+        field: attributes.span_id
   batch:
     send_batch_size: 10000
     send_batch_max_size: 11000
@@ -74,12 +111,16 @@ processors:
   signozspanmetrics/prometheus:
     metrics_exporter: prometheus
     latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
-    dimensions_cache_size: 10000
+    dimensions_cache_size: 100000
     dimensions:
       - name: service.namespace
         default: default
       - name: deployment.environment
         default: default
+      # This is added to ensure the uniqueness of the timeseries
+      # Otherwise, identical timeseries produced by multiple replicas of
+      # collectors result in incorrect APM metrics
+      - name: 'signoz.collector.id'
   # memory_limiter:
   #   # 80% of maximum memory up to 2G
   #   limit_mib: 1500
@@ -97,7 +138,7 @@ processors:
   #   retry_on_failure: true
   resourcedetection:
     # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
-    detectors: [env, system] # include ec2 for AWS, gce for GCP and azure for Azure.
+    detectors: [env, system] # include ec2 for AWS, gcp for GCP and azure for Azure.
     timeout: 2s
 
 extensions:
@@ -111,16 +152,21 @@ extensions:
 exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/?database=signoz_traces
+    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
+    low_cardinal_exception_grouping: ${LOW_CARDINAL_EXCEPTION_GROUPING}
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/?database=signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
+  clickhousemetricswrite/prometheus:
+    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
   prometheus:
     endpoint: 0.0.0.0:8889
   # logging: {}
 
   clickhouselogsexporter:
     dsn: tcp://clickhouse:9000/
+    docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     timeout: 5s
     sending_queue:
       queue_size: 100
@@ -148,13 +194,17 @@ service:
       processors: [batch]
       exporters: [clickhousemetricswrite]
     metrics/generic:
-      receivers: [hostmetrics, prometheus]
+      receivers: [hostmetrics]
       processors: [resourcedetection, batch]
       exporters: [clickhousemetricswrite]
+    metrics/prometheus:
+      receivers: [prometheus]
+      processors: [batch]
+      exporters: [clickhousemetricswrite/prometheus]
     metrics/spanmetrics:
       receivers: [otlp/spanmetrics]
       exporters: [prometheus]
     logs:
       receivers: [otlp, filelog/dockercontainers]
-      processors: [batch]
+      processors: [logstransform/internal, batch]
       exporters: [clickhouselogsexporter]

deploy/docker/clickhouse-setup/otel-collector-metrics-config.yaml

@@ -11,7 +11,9 @@ receivers:
           scrape_interval: 60s
           static_configs:
             - targets:
-              - localhost:8888
+                - localhost:8888
+              labels:
+                job_name: otel-collector-metrics
         # SigNoz span metrics
         - job_name: signozspanmetrics-collector
           scrape_interval: 60s

deploy/docker/clickhouse-setup/user_scripts/histogramQuantile.go

@@ -0,0 +1,237 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"math"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// NOTE: executable must be built with target OS and architecture set to linux/amd64
+// env GOOS=linux GOARCH=amd64 go build -o histogramQuantile histogramQuantile.go
+
+// The following code is adapted from the following source:
+// https://github.com/prometheus/prometheus/blob/main/promql/quantile.go
+
+type bucket struct {
+	upperBound float64
+	count      float64
+}
+
+// buckets implements sort.Interface.
+type buckets []bucket
+
+func (b buckets) Len() int           { return len(b) }
+func (b buckets) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b buckets) Less(i, j int) bool { return b[i].upperBound < b[j].upperBound }
+
+// bucketQuantile calculates the quantile 'q' based on the given buckets. The
+// buckets will be sorted by upperBound by this function (i.e. no sorting
+// needed before calling this function). The quantile value is interpolated
+// assuming a linear distribution within a bucket. However, if the quantile
+// falls into the highest bucket, the upper bound of the 2nd highest bucket is
+// returned. A natural lower bound of 0 is assumed if the upper bound of the
+// lowest bucket is greater 0. In that case, interpolation in the lowest bucket
+// happens linearly between 0 and the upper bound of the lowest bucket.
+// However, if the lowest bucket has an upper bound less or equal 0, this upper
+// bound is returned if the quantile falls into the lowest bucket.
+//
+// There are a number of special cases (once we have a way to report errors
+// happening during evaluations of AST functions, we should report those
+// explicitly):
+//
+// If 'buckets' has 0 observations, NaN is returned.
+//
+// If 'buckets' has fewer than 2 elements, NaN is returned.
+//
+// If the highest bucket is not +Inf, NaN is returned.
+//
+// If q==NaN, NaN is returned.
+//
+// If q<0, -Inf is returned.
+//
+// If q>1, +Inf is returned.
+func bucketQuantile(q float64, buckets buckets) float64 {
+	if math.IsNaN(q) {
+		return math.NaN()
+	}
+	if q < 0 {
+		return math.Inf(-1)
+	}
+	if q > 1 {
+		return math.Inf(+1)
+	}
+	sort.Sort(buckets)
+	if !math.IsInf(buckets[len(buckets)-1].upperBound, +1) {
+		return math.NaN()
+	}
+
+	buckets = coalesceBuckets(buckets)
+	ensureMonotonic(buckets)
+
+	if len(buckets) < 2 {
+		return math.NaN()
+	}
+	observations := buckets[len(buckets)-1].count
+	if observations == 0 {
+		return math.NaN()
+	}
+	rank := q * observations
+	b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })
+
+	if b == len(buckets)-1 {
+		return buckets[len(buckets)-2].upperBound
+	}
+	if b == 0 && buckets[0].upperBound <= 0 {
+		return buckets[0].upperBound
+	}
+	var (
+		bucketStart float64
+		bucketEnd   = buckets[b].upperBound
+		count       = buckets[b].count
+	)
+	if b > 0 {
+		bucketStart = buckets[b-1].upperBound
+		count -= buckets[b-1].count
+		rank -= buckets[b-1].count
+	}
+	return bucketStart + (bucketEnd-bucketStart)*(rank/count)
+}
+
+// coalesceBuckets merges buckets with the same upper bound.
+//
+// The input buckets must be sorted.
+func coalesceBuckets(buckets buckets) buckets {
+	last := buckets[0]
+	i := 0
+	for _, b := range buckets[1:] {
+		if b.upperBound == last.upperBound {
+			last.count += b.count
+		} else {
+			buckets[i] = last
+			last = b
+			i++
+		}
+	}
+	buckets[i] = last
+	return buckets[:i+1]
+}
+
+// The assumption that bucket counts increase monotonically with increasing
+// upperBound may be violated during:
+//
+//   * Recording rule evaluation of histogram_quantile, especially when rate()
+//      has been applied to the underlying bucket timeseries.
+//   * Evaluation of histogram_quantile computed over federated bucket
+//      timeseries, especially when rate() has been applied.
+//
+// This is because scraped data is not made available to rule evaluation or
+// federation atomically, so some buckets are computed with data from the
+// most recent scrapes, but the other buckets are missing data from the most
+// recent scrape.
+//
+// Monotonicity is usually guaranteed because if a bucket with upper bound
+// u1 has count c1, then any bucket with a higher upper bound u > u1 must
+// have counted all c1 observations and perhaps more, so that c  >= c1.
+//
+// Randomly interspersed partial sampling breaks that guarantee, and rate()
+// exacerbates it. Specifically, suppose bucket le=1000 has a count of 10 from
+// 4 samples but the bucket with le=2000 has a count of 7 from 3 samples. The
+// monotonicity is broken. It is exacerbated by rate() because under normal
+// operation, cumulative counting of buckets will cause the bucket counts to
+// diverge such that small differences from missing samples are not a problem.
+// rate() removes this divergence.)
+//
+// bucketQuantile depends on that monotonicity to do a binary search for the
+// bucket with the φ-quantile count, so breaking the monotonicity
+// guarantee causes bucketQuantile() to return undefined (nonsense) results.
+//
+// As a somewhat hacky solution until ingestion is atomic per scrape, we
+// calculate the "envelope" of the histogram buckets, essentially removing
+// any decreases in the count between successive buckets.
+
+func ensureMonotonic(buckets buckets) {
+	max := buckets[0].count
+	for i := 1; i < len(buckets); i++ {
+		switch {
+		case buckets[i].count > max:
+			max = buckets[i].count
+		case buckets[i].count < max:
+			buckets[i].count = max
+		}
+	}
+}
+
+// End of copied code.
+
+func readLines() []string {
+	r := bufio.NewReader(os.Stdin)
+	bytes := []byte{}
+	lines := []string{}
+	for {
+		line, isPrefix, err := r.ReadLine()
+		if err != nil {
+			break
+		}
+		bytes = append(bytes, line...)
+		if !isPrefix {
+			str := strings.TrimSpace(string(bytes))
+			if len(str) > 0 {
+				lines = append(lines, str)
+				bytes = []byte{}
+			}
+		}
+	}
+	if len(bytes) > 0 {
+		lines = append(lines, string(bytes))
+	}
+	return lines
+}
+
+func main() {
+	lines := readLines()
+	for _, text := range lines {
+		// Example input
+		// "[1, 2, 4, 8, 16]", "[1, 5, 8, 10, 14]", 0.9"
+		// bounds - counts - quantile
+		parts := strings.Split(text, "\",")
+
+		var bucketNumbers []float64
+		// Strip the ends with square brackets
+		text = parts[0][2 : len(parts[0])-1]
+		// Parse the bucket bounds
+		for _, num := range strings.Split(text, ",") {
+			num = strings.TrimSpace(num)
+			number, err := strconv.ParseFloat(num, 64)
+			if err == nil {
+				bucketNumbers = append(bucketNumbers, number)
+			}
+		}
+
+		var bucketCounts []float64
+		// Strip the ends with square brackets
+		text = parts[1][2 : len(parts[1])-1]
+		// Parse the bucket counts
+		for _, num := range strings.Split(text, ",") {
+			num = strings.TrimSpace(num)
+			number, err := strconv.ParseFloat(num, 64)
+			if err == nil {
+				bucketCounts = append(bucketCounts, number)
+			}
+		}
+
+		// Parse the quantile
+		q, err := strconv.ParseFloat(parts[2], 64)
+		var b buckets
+
+		if err == nil {
+			for i := 0; i < len(bucketNumbers); i++ {
+				b = append(b, bucket{upperBound: bucketNumbers[i], count: bucketCounts[i]})
+			}
+		}
+		fmt.Println(bucketQuantile(q, b))
+	}
+}

deploy/docker/common/nginx-config.conf

@@ -30,6 +30,8 @@ server {
 
     location /api {
         proxy_pass http://query-service:8080/api;
+        # connection will be closed if no data is read for 600s between successive read operations
+        proxy_read_timeout 600s; 
     }
 
     # redirect server error pages to the static page /50x.html

deploy/install.sh

@@ -51,7 +51,7 @@ check_os() {
     os_name="$(cat /etc/*-release | awk -F= '$1 == "NAME" { gsub(/"/, ""); print $2; exit }')"
 
     case "$os_name" in
-        Ubuntu*)
+        Ubuntu*|Pop!_OS)
             desired_os=1
             os="ubuntu"
             package_manager="apt-get"
@@ -81,6 +81,11 @@ check_os() {
             os="centos"
             package_manager="yum"
             ;;
+        Rocky*)
+            desired_os=1
+            os="centos"
+            package_manager="yum"
+            ;;
         SLES*)
             desired_os=1
             os="sles"
@@ -120,7 +125,7 @@ check_ports_occupied() {
 
         echo "+++++++++++ ERROR ++++++++++++++++++++++"
         echo "SigNoz requires ports 3301 & 4317 to be open. Please shut down any other service(s) that may be running on these ports."
-        echo "You can run SigNoz on another port following this guide https://signoz.io/docs/deployment/docker#troubleshooting"
+        echo "You can run SigNoz on another port following this guide https://signoz.io/docs/install/troubleshooting/"
         echo "++++++++++++++++++++++++++++++++++++++++"
         echo ""
         exit 1
@@ -223,7 +228,7 @@ wait_for_containers_start() {
 
     # The while loop is important because for-loops don't work for dynamic values
     while [[ $timeout -gt 0 ]]; do
-        status_code="$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3301/api/v1/services/list || true)"
+        status_code="$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3301/api/v1/health?live=1" || true)"
         if [[ status_code -eq 200 ]]; then
             break
         else
@@ -244,7 +249,7 @@ bye() {  # Prints a friendly good bye message and exits the script.
         echo ""
         echo -e "$sudo_cmd docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml ps -a"
 
-        # echo "Please read our troubleshooting guide https://signoz.io/docs/deployment/docker#troubleshooting"
+        echo "Please read our troubleshooting guide https://signoz.io/docs/install/troubleshooting/"
         echo "or reach us for support in #help channel in our Slack Community https://signoz.io/slack"
         echo "++++++++++++++++++++++++++++++++++++++++"
 
@@ -495,7 +500,7 @@ if [[ $status_code -ne 200 ]]; then
 
     echo -e "$sudo_cmd docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml ps -a"
 
-    echo "Please read our troubleshooting guide https://signoz.io/docs/deployment/docker/#troubleshooting-of-common-issues"
+    echo "Please read our troubleshooting guide https://signoz.io/docs/install/troubleshooting/"
     echo "or reach us on SigNoz for support https://signoz.io/slack"
     echo "++++++++++++++++++++++++++++++++++++++++"
 
@@ -511,13 +516,15 @@ else
     echo ""
     echo -e "🟢 Your frontend is running on http://localhost:3301"
     echo ""
+    echo "ℹ️  By default, retention period is set to 7 days for logs and traces, and 30 days for metrics." 
+    echo -e "To change this, navigate to the General tab on the Settings page of SigNoz UI. For more details, refer to https://signoz.io/docs/userguide/retention-period \n"
 
     echo "ℹ️  To bring down SigNoz and clean volumes : $sudo_cmd docker-compose -f ./docker/clickhouse-setup/docker-compose.yaml down -v"
 
     echo ""
     echo "+++++++++++++++++++++++++++++++++++++++++++++++++"
     echo ""
-    echo "👉 Need help Getting Started?"
+    echo "👉 Need help in Getting Started?"
     echo -e "Join us on Slack https://signoz.io/slack"
     echo ""
     echo -e "\n📨 Please share your email to receive support & updates about SigNoz!"

ee/query-service/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17-buster AS builder
+FROM golang:1.18-buster AS builder
 
 # LD_FLAGS is passed as argument from Makefile. It will be empty, if no argument passed
 ARG LD_FLAGS

ee/query-service/app/api/api.go

@@ -9,6 +9,7 @@ import (
 	"go.signoz.io/signoz/ee/query-service/license"
 	baseapp "go.signoz.io/signoz/pkg/query-service/app"
 	baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
+	basemodel "go.signoz.io/signoz/pkg/query-service/model"
 	rules "go.signoz.io/signoz/pkg/query-service/rules"
 	"go.signoz.io/signoz/pkg/query-service/version"
 )
@@ -68,60 +69,75 @@ func (ah *APIHandler) CheckFeature(f string) bool {
 }
 
 // RegisterRoutes registers routes for this handler on the given router
-func (ah *APIHandler) RegisterRoutes(router *mux.Router) {
+func (ah *APIHandler) RegisterRoutes(router *mux.Router, am *baseapp.AuthMiddleware) {
 	// note: add ee override methods first
 
 	// routes available only in ee version
 	router.HandleFunc("/api/v1/licenses",
-		baseapp.AdminAccess(ah.listLicenses)).
+		am.AdminAccess(ah.listLicenses)).
 		Methods(http.MethodGet)
 
 	router.HandleFunc("/api/v1/licenses",
-		baseapp.AdminAccess(ah.applyLicense)).
+		am.AdminAccess(ah.applyLicense)).
 		Methods(http.MethodPost)
 
 	router.HandleFunc("/api/v1/featureFlags",
-		baseapp.OpenAccess(ah.getFeatureFlags)).
+		am.OpenAccess(ah.getFeatureFlags)).
 		Methods(http.MethodGet)
 
 	router.HandleFunc("/api/v1/loginPrecheck",
-		baseapp.OpenAccess(ah.precheckLogin)).
+		am.OpenAccess(ah.precheckLogin)).
 		Methods(http.MethodGet)
 
 	// paid plans specific routes
 	router.HandleFunc("/api/v1/complete/saml",
-		baseapp.OpenAccess(ah.receiveSAML)).
+		am.OpenAccess(ah.receiveSAML)).
 		Methods(http.MethodPost)
 
+	router.HandleFunc("/api/v1/complete/google",
+		am.OpenAccess(ah.receiveGoogleAuth)).
+		Methods(http.MethodGet)
+
 	router.HandleFunc("/api/v1/orgs/{orgId}/domains",
-		baseapp.AdminAccess(ah.listDomainsByOrg)).
+		am.AdminAccess(ah.listDomainsByOrg)).
 		Methods(http.MethodGet)
 
 	router.HandleFunc("/api/v1/domains",
-		baseapp.AdminAccess(ah.postDomain)).
+		am.AdminAccess(ah.postDomain)).
 		Methods(http.MethodPost)
 
 	router.HandleFunc("/api/v1/domains/{id}",
-		baseapp.AdminAccess(ah.putDomain)).
+		am.AdminAccess(ah.putDomain)).
 		Methods(http.MethodPut)
 
 	router.HandleFunc("/api/v1/domains/{id}",
-		baseapp.AdminAccess(ah.deleteDomain)).
+		am.AdminAccess(ah.deleteDomain)).
 		Methods(http.MethodDelete)
 
 	// base overrides
-	router.HandleFunc("/api/v1/version", baseapp.OpenAccess(ah.getVersion)).Methods(http.MethodGet)
-	router.HandleFunc("/api/v1/invite/{token}", baseapp.OpenAccess(ah.getInvite)).Methods(http.MethodGet)
-	router.HandleFunc("/api/v1/register", baseapp.OpenAccess(ah.registerUser)).Methods(http.MethodPost)
-	router.HandleFunc("/api/v1/login", baseapp.OpenAccess(ah.loginUser)).Methods(http.MethodPost)
-	router.HandleFunc("/api/v1/traces/{traceId}", baseapp.ViewAccess(ah.searchTraces)).Methods(http.MethodGet)
-	router.HandleFunc("/api/v2/metrics/query_range", baseapp.ViewAccess(ah.queryRangeMetricsV2)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/version", am.OpenAccess(ah.getVersion)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/invite/{token}", am.OpenAccess(ah.getInvite)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/register", am.OpenAccess(ah.registerUser)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/login", am.OpenAccess(ah.loginUser)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/traces/{traceId}", am.ViewAccess(ah.searchTraces)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v2/metrics/query_range", am.ViewAccess(ah.queryRangeMetricsV2)).Methods(http.MethodPost)
 
-	ah.APIHandler.RegisterRoutes(router)
+	// PAT APIs
+	router.HandleFunc("/api/v1/pat", am.OpenAccess(ah.createPAT)).Methods(http.MethodPost)
+	router.HandleFunc("/api/v1/pat", am.OpenAccess(ah.getPATs)).Methods(http.MethodGet)
+	router.HandleFunc("/api/v1/pat/{id}", am.OpenAccess(ah.deletePAT)).Methods(http.MethodDelete)
+
+	ah.APIHandler.RegisterRoutes(router, am)
 
 }
 
 func (ah *APIHandler) getVersion(w http.ResponseWriter, r *http.Request) {
 	version := version.GetVersion()
-	ah.WriteJSON(w, r, map[string]string{"version": version, "ee": "Y"})
+	versionResponse := basemodel.GetVersionResponse{
+		Version:        version,
+		EE:             "Y",
+		SetupCompleted: ah.SetupCompleted,
+	}
+
+	ah.WriteJSON(w, r, versionResponse)
 }

ee/query-service/app/api/auth.go

@@ -8,9 +8,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
-	"strings"
 
-	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"go.signoz.io/signoz/ee/query-service/constants"
 	"go.signoz.io/signoz/ee/query-service/model"
@@ -90,9 +88,16 @@ func (ah *APIHandler) registerUser(w http.ResponseWriter, r *http.Request) {
 
 	// get invite object
 	invite, err := baseauth.ValidateInvite(ctx, req)
-	if err != nil || invite == nil {
+	if err != nil {
 		zap.S().Errorf("failed to validate invite token", err)
+		RespondError(w, model.BadRequest(err), nil)
+		return
+	}
+
+	if invite == nil {
+		zap.S().Errorf("failed to validate invite token: it is either empty or invalid", err)
 		RespondError(w, model.BadRequest(basemodel.ErrSignupFailed{}), nil)
+		return
 	}
 
 	// get auth domain from email domain
@@ -184,114 +189,149 @@ func (ah *APIHandler) precheckLogin(w http.ResponseWriter, r *http.Request) {
 	ah.Respond(w, resp)
 }
 
-func (ah *APIHandler) receiveSAML(w http.ResponseWriter, r *http.Request) {
-	// this is the source url that initiated the login request
+func handleSsoError(w http.ResponseWriter, r *http.Request, redirectURL string) {
+	ssoError := []byte("Login failed. Please contact your system administrator")
+	dst := make([]byte, base64.StdEncoding.EncodedLen(len(ssoError)))
+	base64.StdEncoding.Encode(dst, ssoError)
+
+	http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectURL, string(dst)), http.StatusSeeOther)
+}
+
+// receiveGoogleAuth completes google OAuth response and forwards a request
+// to front-end to sign user in
+func (ah *APIHandler) receiveGoogleAuth(w http.ResponseWriter, r *http.Request) {
 	redirectUri := constants.GetDefaultSiteURL()
 	ctx := context.Background()
 
-	var apierr basemodel.BaseApiError
-
-	redirectOnError := func() {
-		ssoError := []byte("Login failed. Please contact your system administrator")
-		dst := make([]byte, base64.StdEncoding.EncodedLen(len(ssoError)))
-		base64.StdEncoding.Encode(dst, ssoError)
-
-		http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, string(dst)), http.StatusMovedPermanently)
-	}
-
 	if !ah.CheckFeature(model.SSO) {
-		zap.S().Errorf("[ReceiveSAML] sso requested but feature unavailable %s in org domain %s", model.SSO)
+		zap.S().Errorf("[receiveGoogleAuth] sso requested but feature unavailable %s in org domain %s", model.SSO)
 		http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, "feature unavailable, please upgrade your billing plan to access this feature"), http.StatusMovedPermanently)
 		return
 	}
 
-	err := r.ParseForm()
-	if err != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to process response - invalid response from IDP", err, r)
-		redirectOnError()
+	q := r.URL.Query()
+	if errType := q.Get("error"); errType != "" {
+		zap.S().Errorf("[receiveGoogleAuth] failed to login with google auth", q.Get("error_description"))
+		http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, "failed to login through SSO "), http.StatusMovedPermanently)
 		return
 	}
 
-	// the relay state is sent when a login request is submitted to
-	// Idp.
-	relayState := r.FormValue("RelayState")
-	zap.S().Debug("[ReceiveML] relay state", zap.String("relayState", relayState))
+	relayState := q.Get("state")
+	zap.S().Debug("[receiveGoogleAuth] relay state received", zap.String("state", relayState))
 
 	parsedState, err := url.Parse(relayState)
 	if err != nil || relayState == "" {
-		zap.S().Errorf("[ReceiveSAML] failed to process response - invalid response from IDP", err, r)
-		redirectOnError()
+		zap.S().Errorf("[receiveGoogleAuth] failed to process response - invalid response from IDP", err, r)
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
 	// upgrade redirect url from the relay state for better accuracy
 	redirectUri = fmt.Sprintf("%s://%s%s", parsedState.Scheme, parsedState.Host, "/login")
 
-	// derive domain id from relay state now
-	var domainIdStr string
-	for k, v := range parsedState.Query() {
-		if k == "domainId" && len(v) > 0 {
-			domainIdStr = strings.Replace(v[0], ":", "-", -1)
-		}
-	}
-
-	domainId, err := uuid.Parse(domainIdStr)
+	// fetch domain by parsing relay state.
+	domain, err := ah.AppDao().GetDomainFromSsoResponse(ctx, parsedState)
 	if err != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to process request- failed to parse domain id ifrom relay", zap.Error(err))
-		redirectOnError()
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
-	domain, apierr := ah.AppDao().GetDomain(ctx, domainId)
-	if (apierr != nil) || domain == nil {
-		zap.S().Errorf("[ReceiveSAML] failed to process request- invalid domain", domainIdStr, zap.Error(apierr))
-		redirectOnError()
+	// now that we have domain, use domain to fetch sso settings.
+	// prepare google callback handler using parsedState -
+	// which contains redirect URL (front-end endpoint)
+	callbackHandler, err := domain.PrepareGoogleOAuthProvider(parsedState)
+
+	identity, err := callbackHandler.HandleCallback(r)
+	if err != nil {
+		zap.S().Errorf("[receiveGoogleAuth] failed to process HandleCallback ", domain.String(), zap.Error(err))
+		handleSsoError(w, r, redirectUri)
+		return
+	}
+
+	nextPage, err := ah.AppDao().PrepareSsoRedirect(ctx, redirectUri, identity.Email)
+	if err != nil {
+		zap.S().Errorf("[receiveGoogleAuth] failed to generate redirect URI after successful login ", domain.String(), zap.Error(err))
+		handleSsoError(w, r, redirectUri)
+		return
+	}
+
+	http.Redirect(w, r, nextPage, http.StatusSeeOther)
+}
+
+// receiveSAML completes a SAML request and gets user logged in
+func (ah *APIHandler) receiveSAML(w http.ResponseWriter, r *http.Request) {
+	// this is the source url that initiated the login request
+	redirectUri := constants.GetDefaultSiteURL()
+	ctx := context.Background()
+
+	if !ah.CheckFeature(model.SSO) {
+		zap.S().Errorf("[receiveSAML] sso requested but feature unavailable %s in org domain %s", model.SSO)
+		http.Redirect(w, r, fmt.Sprintf("%s?ssoerror=%s", redirectUri, "feature unavailable, please upgrade your billing plan to access this feature"), http.StatusMovedPermanently)
+		return
+	}
+
+	err := r.ParseForm()
+	if err != nil {
+		zap.S().Errorf("[receiveSAML] failed to process response - invalid response from IDP", err, r)
+		handleSsoError(w, r, redirectUri)
+		return
+	}
+
+	// the relay state is sent when a login request is submitted to
+	// Idp.
+	relayState := r.FormValue("RelayState")
+	zap.S().Debug("[receiveML] relay state", zap.String("relayState", relayState))
+
+	parsedState, err := url.Parse(relayState)
+	if err != nil || relayState == "" {
+		zap.S().Errorf("[receiveSAML] failed to process response - invalid response from IDP", err, r)
+		handleSsoError(w, r, redirectUri)
+		return
+	}
+
+	// upgrade redirect url from the relay state for better accuracy
+	redirectUri = fmt.Sprintf("%s://%s%s", parsedState.Scheme, parsedState.Host, "/login")
+
+	// fetch domain by parsing relay state.
+	domain, err := ah.AppDao().GetDomainFromSsoResponse(ctx, parsedState)
+	if err != nil {
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
 	sp, err := domain.PrepareSamlRequest(parsedState)
 	if err != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to prepare saml request for domain (%s): %v", domainId, err)
-		redirectOnError()
+		zap.S().Errorf("[receiveSAML] failed to prepare saml request for domain (%s): %v", domain.String(), err)
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
 	assertionInfo, err := sp.RetrieveAssertionInfo(r.FormValue("SAMLResponse"))
 	if err != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to retrieve assertion info from  saml response for organization (%s): %v", domainId, err)
-		redirectOnError()
+		zap.S().Errorf("[receiveSAML] failed to retrieve assertion info from  saml response for organization (%s): %v", domain.String(), err)
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
 	if assertionInfo.WarningInfo.InvalidTime {
-		zap.S().Errorf("[ReceiveSAML] expired saml response for organization (%s): %v", domainId, err)
-		redirectOnError()
+		zap.S().Errorf("[receiveSAML] expired saml response for organization (%s): %v", domain.String(), err)
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
 	email := assertionInfo.NameID
-
-	// user email found, now start preparing jwt response
-	userPayload, baseapierr := ah.AppDao().GetUserByEmail(ctx, email)
-	if baseapierr != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to find or register a new user for email %s and org %s", email, domainId, zap.Error(baseapierr.Err))
-		redirectOnError()
+	if email == "" {
+		zap.S().Errorf("[receiveSAML] invalid email in the SSO response (%s)", domain.String())
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
-	tokenStore, err := baseauth.GenerateJWTForUser(&userPayload.User)
+	nextPage, err := ah.AppDao().PrepareSsoRedirect(ctx, redirectUri, email)
 	if err != nil {
-		zap.S().Errorf("[ReceiveSAML] failed to generate access token for email %s and org %s", email, domainId, zap.Error(err))
-		redirectOnError()
+		zap.S().Errorf("[receiveSAML] failed to generate redirect URI after successful login ", domain.String(), zap.Error(err))
+		handleSsoError(w, r, redirectUri)
 		return
 	}
 
-	userID := userPayload.User.Id
-	nextPage := fmt.Sprintf("%s?jwt=%s&usr=%s&refreshjwt=%s",
-		redirectUri,
-		tokenStore.AccessJwt,
-		userID,
-		tokenStore.RefreshJwt)
-
-	http.Redirect(w, r, nextPage, http.StatusMovedPermanently)
+	http.Redirect(w, r, nextPage, http.StatusSeeOther)
 }

ee/query-service/app/api/featureFlags.go

@@ -5,6 +5,10 @@ import (
 )
 
 func (ah *APIHandler) getFeatureFlags(w http.ResponseWriter, r *http.Request) {
-	featureSet := ah.FF().GetFeatureFlags()
+	featureSet, err := ah.FF().GetFeatureFlags()
+	if err != nil {
+		ah.HandleError(w, err, http.StatusInternalServerError)
+		return
+	}
 	ah.Respond(w, featureSet)
 }

ee/query-service/app/api/pat.go

@@ -0,0 +1,107 @@
+package api
+
+import (
+	"context"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/gorilla/mux"
+	"go.signoz.io/signoz/ee/query-service/model"
+	"go.signoz.io/signoz/pkg/query-service/auth"
+	"go.uber.org/zap"
+)
+
+func generatePATToken() string {
+	// Generate a 32-byte random token.
+	token := make([]byte, 32)
+	rand.Read(token)
+	// Encode the token in base64.
+	encodedToken := base64.StdEncoding.EncodeToString(token)
+	return encodedToken
+}
+
+func (ah *APIHandler) createPAT(w http.ResponseWriter, r *http.Request) {
+	ctx := context.Background()
+
+	req := model.PAT{}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		RespondError(w, model.BadRequest(err), nil)
+		return
+	}
+	user, err := auth.GetUserFromRequest(r)
+	if err != nil {
+		RespondError(w, &model.ApiError{
+			Typ: model.ErrorUnauthorized,
+			Err: err,
+		}, nil)
+		return
+	}
+
+	// All the PATs are associated with the user creating the PAT. Hence, the permissions
+	// associated with the PAT is also equivalent to that of the user.
+	req.UserID = user.Id
+	req.CreatedAt = time.Now().Unix()
+	req.Token = generatePATToken()
+
+	zap.S().Debugf("Got PAT request: %+v", req)
+	if apierr := ah.AppDao().CreatePAT(ctx, &req); apierr != nil {
+		RespondError(w, apierr, nil)
+		return
+	}
+
+	ah.Respond(w, &req)
+}
+
+func (ah *APIHandler) getPATs(w http.ResponseWriter, r *http.Request) {
+	ctx := context.Background()
+	user, err := auth.GetUserFromRequest(r)
+	if err != nil {
+		RespondError(w, &model.ApiError{
+			Typ: model.ErrorUnauthorized,
+			Err: err,
+		}, nil)
+		return
+	}
+	zap.S().Infof("Get PATs for user: %+v", user.Id)
+	pats, apierr := ah.AppDao().ListPATs(ctx, user.Id)
+	if apierr != nil {
+		RespondError(w, apierr, nil)
+		return
+	}
+	ah.Respond(w, pats)
+}
+
+func (ah *APIHandler) deletePAT(w http.ResponseWriter, r *http.Request) {
+	ctx := context.Background()
+	id := mux.Vars(r)["id"]
+	user, err := auth.GetUserFromRequest(r)
+	if err != nil {
+		RespondError(w, &model.ApiError{
+			Typ: model.ErrorUnauthorized,
+			Err: err,
+		}, nil)
+		return
+	}
+	pat, apierr := ah.AppDao().GetPATByID(ctx, id)
+	if apierr != nil {
+		RespondError(w, apierr, nil)
+		return
+	}
+	if pat.UserID != user.Id {
+		RespondError(w, &model.ApiError{
+			Typ: model.ErrorUnauthorized,
+			Err: fmt.Errorf("unauthorized PAT delete request"),
+		}, nil)
+		return
+	}
+	zap.S().Debugf("Delete PAT with id: %+v", id)
+	if apierr := ah.AppDao().DeletePAT(ctx, id); apierr != nil {
+		RespondError(w, apierr, nil)
+		return
+	}
+	ah.Respond(w, map[string]string{"data": "pat deleted successfully"})
+}

ee/query-service/app/server.go

@@ -1,8 +1,11 @@
 package app
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	_ "net/http/pprof" // http profiler
@@ -19,13 +22,23 @@ import (
 	"go.signoz.io/signoz/ee/query-service/app/db"
 	"go.signoz.io/signoz/ee/query-service/dao"
 	"go.signoz.io/signoz/ee/query-service/interfaces"
-	licensepkg "go.signoz.io/signoz/ee/query-service/license"
+	baseInterface "go.signoz.io/signoz/pkg/query-service/interfaces"
 
+	licensepkg "go.signoz.io/signoz/ee/query-service/license"
+	"go.signoz.io/signoz/ee/query-service/usage"
+
+	"go.signoz.io/signoz/pkg/query-service/agentConf"
+	baseapp "go.signoz.io/signoz/pkg/query-service/app"
 	"go.signoz.io/signoz/pkg/query-service/app/dashboards"
+	baseexplorer "go.signoz.io/signoz/pkg/query-service/app/explorer"
+	"go.signoz.io/signoz/pkg/query-service/app/opamp"
+	opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model"
+	baseauth "go.signoz.io/signoz/pkg/query-service/auth"
 	baseconst "go.signoz.io/signoz/pkg/query-service/constants"
 	"go.signoz.io/signoz/pkg/query-service/healthcheck"
 	basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager"
 	baseint "go.signoz.io/signoz/pkg/query-service/interfaces"
+	basemodel "go.signoz.io/signoz/pkg/query-service/model"
 	pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine"
 	rules "go.signoz.io/signoz/pkg/query-service/rules"
 	"go.signoz.io/signoz/pkg/query-service/telemetry"
@@ -33,6 +46,8 @@ import (
 	"go.uber.org/zap"
 )
 
+const AppDbEngine = "sqlite"
+
 type ServerOptions struct {
 	PromConfigPath  string
 	HTTPHostPort    string
@@ -76,6 +91,8 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		return nil, err
 	}
 
+	baseexplorer.InitWithDSN(baseconst.RELATIONAL_DATASOURCE_PATH)
+
 	localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
 
 	if err != nil {
@@ -111,12 +128,34 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
 		serverOptions.RuleRepoURL,
 		localDB,
 		reader,
-		serverOptions.DisableRules)
+		serverOptions.DisableRules,
+		lm)
 
 	if err != nil {
 		return nil, err
 	}
 
+	// initiate opamp
+	_, err = opAmpModel.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)
+	if err != nil {
+		return nil, err
+	}
+
+	// initiate agent config handler
+	if err := agentConf.Initiate(localDB, AppDbEngine); err != nil {
+		return nil, err
+	}
+
+	// start the usagemanager
+	usageManager, err := usage.New("sqlite", localDB, lm.GetRepo(), reader.GetConn())
+	if err != nil {
+		return nil, err
+	}
+	err = usageManager.Start()
+	if err != nil {
+		return nil, err
+	}
+
 	telemetry.GetInstance().SetReader(reader)
 
 	apiOpts := api.APIHandlerOptions{
@@ -173,7 +212,7 @@ func (s *Server) createPrivateServer(apiHandler *api.APIHandler) (*http.Server,
 		// ip here for alert manager
 		AllowedOrigins: []string{"*"},
 		AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH"},
-		AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"},
+		AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "SIGNOZ-API-KEY"},
 	})
 
 	handler := c.Handler(r)
@@ -188,13 +227,33 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e
 
 	r := mux.NewRouter()
 
+	getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) {
+		patToken := r.Header.Get("SIGNOZ-API-KEY")
+		if len(patToken) > 0 {
+			zap.S().Debugf("Received a non-zero length PAT token")
+			ctx := context.Background()
+			dao := apiHandler.AppDao()
+
+			user, err := dao.GetUserByPAT(ctx, patToken)
+			if err == nil && user != nil {
+				zap.S().Debugf("Found valid PAT user: %+v", user)
+				return user, nil
+			}
+			if err != nil {
+				zap.S().Debugf("Error while getting user for PAT: %+v", err)
+			}
+		}
+		return baseauth.GetUserFromRequest(r)
+	}
+	am := baseapp.NewAuthMiddleware(getUserFromRequest)
 	r.Use(setTimeoutMiddleware)
 	r.Use(s.analyticsMiddleware)
 	r.Use(loggingMiddleware)
 
-	apiHandler.RegisterRoutes(r)
-	apiHandler.RegisterMetricsRoutes(r)
-	apiHandler.RegisterLogsRoutes(r)
+	apiHandler.RegisterRoutes(r, am)
+	apiHandler.RegisterMetricsRoutes(r, am)
+	apiHandler.RegisterLogsRoutes(r, am)
+	apiHandler.RegisterQueryRangeV3Routes(r, am)
 
 	c := cors.New(cors.Options{
 		AllowedOrigins: []string{"*"},
@@ -255,15 +314,82 @@ func (lrw *loggingResponseWriter) Flush() {
 	lrw.ResponseWriter.(http.Flusher).Flush()
 }
 
+func extractDashboardMetaData(path string, r *http.Request) (map[string]interface{}, bool) {
+	pathToExtractBodyFrom := "/api/v2/metrics/query_range"
+
+	data := map[string]interface{}{}
+	var postData *basemodel.QueryRangeParamsV2
+
+	if path == pathToExtractBodyFrom && (r.Method == "POST") {
+		if r.Body != nil {
+			bodyBytes, err := ioutil.ReadAll(r.Body)
+			if err != nil {
+				return nil, false
+			}
+			r.Body.Close() //  must close
+			r.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
+			json.Unmarshal(bodyBytes, &postData)
+
+		} else {
+			return nil, false
+		}
+
+	} else {
+		return nil, false
+	}
+
+	signozMetricNotFound := false
+
+	if postData != nil {
+		signozMetricNotFound = telemetry.GetInstance().CheckSigNozMetricsV2(postData.CompositeMetricQuery)
+
+		if postData.CompositeMetricQuery != nil {
+			data["queryType"] = postData.CompositeMetricQuery.QueryType
+			data["panelType"] = postData.CompositeMetricQuery.PanelType
+		}
+
+		data["datasource"] = postData.DataSource
+	}
+
+	if signozMetricNotFound {
+		telemetry.GetInstance().AddActiveMetricsUser()
+		telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_DASHBOARDS_METADATA, data, true)
+	}
+
+	return data, true
+}
+
+func getActiveLogs(path string, r *http.Request) {
+	// if path == "/api/v1/dashboards/{uuid}" {
+	// 	telemetry.GetInstance().AddActiveMetricsUser()
+	// }
+	if path == "/api/v1/logs" {
+		hasFilters := len(r.URL.Query().Get("q"))
+		if hasFilters > 0 {
+			telemetry.GetInstance().AddActiveLogsUser()
+		}
+
+	}
+
+}
+
 func (s *Server) analyticsMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		route := mux.CurrentRoute(r)
 		path, _ := route.GetPathTemplate()
 
+		dashboardMetadata, metadataExists := extractDashboardMetaData(path, r)
+		getActiveLogs(path, r)
+
 		lrw := NewLoggingResponseWriter(w)
 		next.ServeHTTP(lrw, r)
 
 		data := map[string]interface{}{"path": path, "statusCode": lrw.statusCode}
+		if metadataExists {
+			for key, value := range dashboardMetadata {
+				data[key] = value
+			}
+		}
 
 		if _, ok := telemetry.IgnoredPaths()[path]; !ok {
 			telemetry.GetInstance().SendEvent(telemetry.TELEMETRY_EVENT_PATH, data)
@@ -279,7 +405,7 @@ func setTimeoutMiddleware(next http.Handler) http.Handler {
 		// check if route is not excluded
 		url := r.URL.Path
 		if _, ok := baseconst.TimeoutExcludedRoutes[url]; !ok {
-			ctx, cancel = context.WithTimeout(r.Context(), baseconst.ContextTimeout*time.Second)
+			ctx, cancel = context.WithTimeout(r.Context(), baseconst.ContextTimeout)
 			defer cancel()
 		}
 
@@ -365,7 +491,7 @@ func (s *Server) Start() error {
 	if port, err := utils.GetPort(s.privateConn.Addr()); err == nil {
 		privatePort = port
 	}
-	fmt.Println("starting private http")
+
 	go func() {
 		zap.S().Info("Starting Private HTTP server", zap.Int("port", privatePort), zap.String("addr", s.serverOptions.PrivateHostPort))
 
@@ -381,6 +507,37 @@ func (s *Server) Start() error {
 
 	}()
 
+	go func() {
+		zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", baseconst.OpAmpWsEndpoint))
+		err := opamp.InitalizeServer(baseconst.OpAmpWsEndpoint, &opAmpModel.AllAgents)
+		if err != nil {
+			zap.S().Info("opamp ws server failed to start", err)
+			s.unavailableChannel <- healthcheck.Unavailable
+		}
+	}()
+
+	return nil
+}
+
+func (s *Server) Stop() error {
+	if s.httpServer != nil {
+		if err := s.httpServer.Shutdown(context.Background()); err != nil {
+			return err
+		}
+	}
+
+	if s.privateHTTP != nil {
+		if err := s.privateHTTP.Shutdown(context.Background()); err != nil {
+			return err
+		}
+	}
+
+	opamp.StopServer()
+
+	if s.ruleManager != nil {
+		s.ruleManager.Stop()
+	}
+
 	return nil
 }
 
@@ -390,7 +547,8 @@ func makeRulesManager(
 	ruleRepoURL string,
 	db *sqlx.DB,
 	ch baseint.Reader,
-	disableRules bool) (*rules.Manager, error) {
+	disableRules bool,
+	fm baseInterface.FeatureLookup) (*rules.Manager, error) {
 
 	// create engine
 	pqle, err := pqle.FromConfigPath(promConfigPath)
@@ -417,6 +575,7 @@ func makeRulesManager(
 		Context:      context.Background(),
 		Logger:       nil,
 		DisableRules: disableRules,
+		FeatureFlags: fm,
 	}
 
 	// create Manager

ee/query-service/dao/interface.go

@@ -2,6 +2,7 @@ package dao
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
@@ -22,6 +23,8 @@ type ModelDao interface {
 	// auth methods
 	PrecheckLogin(ctx context.Context, email, sourceUrl string) (*model.PrecheckResponse, basemodel.BaseApiError)
 	CanUsePassword(ctx context.Context, email string) (bool, basemodel.BaseApiError)
+	PrepareSsoRedirect(ctx context.Context, redirectUri, email string) (redirectURL string, apierr basemodel.BaseApiError)
+	GetDomainFromSsoResponse(ctx context.Context, relayState *url.URL) (*model.OrgDomain, error)
 
 	// org domain (auth domains) CRUD ops
 	ListDomains(ctx context.Context, orgId string) ([]model.OrgDomain, basemodel.BaseApiError)
@@ -30,4 +33,11 @@ type ModelDao interface {
 	UpdateDomain(ctx context.Context, domain *model.OrgDomain) basemodel.BaseApiError
 	DeleteDomain(ctx context.Context, id uuid.UUID) basemodel.BaseApiError
 	GetDomainByEmail(ctx context.Context, email string) (*model.OrgDomain, basemodel.BaseApiError)
+
+	CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError
+	GetPAT(ctx context.Context, pat string) (*model.PAT, basemodel.BaseApiError)
+	GetPATByID(ctx context.Context, id string) (*model.PAT, basemodel.BaseApiError)
+	GetUserByPAT(ctx context.Context, token string) (*basemodel.UserPayload, basemodel.BaseApiError)
+	ListPATs(ctx context.Context, userID string) ([]model.PAT, basemodel.BaseApiError)
+	DeletePAT(ctx context.Context, id string) basemodel.BaseApiError
 }

ee/query-service/dao/sqlite/auth.go

@@ -10,9 +10,33 @@ import (
 	"go.signoz.io/signoz/ee/query-service/model"
 	baseconst "go.signoz.io/signoz/pkg/query-service/constants"
 	basemodel "go.signoz.io/signoz/pkg/query-service/model"
+	baseauth "go.signoz.io/signoz/pkg/query-service/auth"
 	"go.uber.org/zap"
 )
 
+// PrepareSsoRedirect prepares redirect page link after SSO response 
+// is successfully parsed (i.e. valid email is available)
+func (m *modelDao) PrepareSsoRedirect(ctx context.Context, redirectUri, email string) (redirectURL string, apierr basemodel.BaseApiError) {
+
+	userPayload, apierr := m.GetUserByEmail(ctx, email)
+	if !apierr.IsNil() {
+		zap.S().Errorf(" failed to get user with email received from auth provider", apierr.Error())
+		return "", model.BadRequestStr("invalid user email received from the auth provider")
+	}
+
+	tokenStore, err := baseauth.GenerateJWTForUser(&userPayload.User)
+	if err != nil {
+		zap.S().Errorf("failed to generate token for SSO login user", err)
+		return "", model.InternalErrorStr("failed to generate token for the user")
+	}
+
+	return fmt.Sprintf("%s?jwt=%s&usr=%s&refreshjwt=%s",
+		redirectUri,
+		tokenStore.AccessJwt,
+		userPayload.User.Id,
+		tokenStore.RefreshJwt), nil
+}
+
 func (m *modelDao) CanUsePassword(ctx context.Context, email string) (bool, basemodel.BaseApiError) {
 	domain, apierr := m.GetDomainByEmail(ctx, email)
 	if apierr != nil {

ee/query-service/dao/sqlite/domain.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"net/url"
 	"fmt"
 	"strings"
 	"time"
@@ -25,6 +26,34 @@ type StoredDomain struct {
 	UpdatedAt int64     `db:"updated_at"`
 }
 
+// GetDomainFromSsoResponse uses relay state received from IdP to fetch
+// user domain. The domain is further used to process validity of the response.
+// when sending login request to IdP we send relay state as URL (site url) 
+// with domainId as query parameter.   
+func (m *modelDao) GetDomainFromSsoResponse(ctx context.Context, relayState *url.URL) (*model.OrgDomain, error) {
+	// derive domain id from relay state now
+	var domainIdStr string 
+	for k, v := range relayState.Query() {
+		if k == "domainId" && len(v) > 0 {
+			domainIdStr = strings.Replace(v[0], ":", "-", -1)
+		}
+	}
+
+	domainId, err := uuid.Parse(domainIdStr)
+	if err != nil {
+		zap.S().Errorf("failed to parse domain id from relay state", err)
+		return nil, fmt.Errorf("failed to parse response from IdP response")
+	}
+
+	domain, err := m.GetDomain(ctx, domainId)
+	if (err != nil) || domain == nil {
+		zap.S().Errorf("failed to find domain received in IdP response", err.Error())
+		return nil, fmt.Errorf("invalid credentials")
+	}
+
+	return domain, nil
+}
+
 // GetDomain returns org domain for a given domain id
 func (m *modelDao) GetDomain(ctx context.Context, id uuid.UUID) (*model.OrgDomain, basemodel.BaseApiError) {
 

ee/query-service/dao/sqlite/modelDao.go

@@ -48,7 +48,17 @@ func InitDB(dataSourceName string) (*modelDao, error) {
 		updated_at INTEGER,
 		data TEXT  NOT NULL,
 		FOREIGN KEY(org_id) REFERENCES organizations(id)
-	);`
+	);
+	CREATE TABLE IF NOT EXISTS personal_access_tokens (
+		id INTEGER PRIMARY KEY AUTOINCREMENT,
+		user_id TEXT NOT NULL,
+		token TEXT NOT NULL UNIQUE,
+		name TEXT NOT NULL,
+		created_at INTEGER NOT NULL,
+		expires_at INTEGER NOT NULL,
+		FOREIGN KEY(user_id) REFERENCES users(id)
+	);
+	`
 
 	_, err = m.DB().Exec(table_schema)
 	if err != nil {

ee/query-service/dao/sqlite/pat.go

@@ -0,0 +1,106 @@
+package sqlite
+
+import (
+	"context"
+	"fmt"
+
+	"go.signoz.io/signoz/ee/query-service/model"
+	basemodel "go.signoz.io/signoz/pkg/query-service/model"
+	"go.uber.org/zap"
+)
+
+func (m *modelDao) CreatePAT(ctx context.Context, p *model.PAT) basemodel.BaseApiError {
+	_, err := m.DB().ExecContext(ctx,
+		"INSERT INTO personal_access_tokens (user_id, token, name, created_at, expires_at) VALUES ($1, $2, $3, $4, $5)",
+		p.UserID,
+		p.Token,
+		p.Name,
+		p.CreatedAt,
+		p.ExpiresAt)
+	if err != nil {
+		zap.S().Errorf("Failed to insert PAT in db, err: %v", zap.Error(err))
+		return model.InternalError(fmt.Errorf("PAT insertion failed"))
+	}
+	return nil
+}
+
+func (m *modelDao) ListPATs(ctx context.Context, userID string) ([]model.PAT, basemodel.BaseApiError) {
+	pats := []model.PAT{}
+
+	if err := m.DB().Select(&pats, `SELECT * FROM personal_access_tokens WHERE user_id=?;`, userID); err != nil {
+		zap.S().Errorf("Failed to fetch PATs for user: %s, err: %v", userID, zap.Error(err))
+		return nil, model.InternalError(fmt.Errorf("failed to fetch PATs"))
+	}
+	return pats, nil
+}
+
+func (m *modelDao) DeletePAT(ctx context.Context, id string) basemodel.BaseApiError {
+	_, err := m.DB().ExecContext(ctx, `DELETE from personal_access_tokens where id=?;`, id)
+	if err != nil {
+		zap.S().Errorf("Failed to delete PAT, err: %v", zap.Error(err))
+		return model.InternalError(fmt.Errorf("failed to delete PAT"))
+	}
+	return nil
+}
+
+func (m *modelDao) GetPAT(ctx context.Context, token string) (*model.PAT, basemodel.BaseApiError) {
+	pats := []model.PAT{}
+
+	if err := m.DB().Select(&pats, `SELECT * FROM personal_access_tokens WHERE token=?;`, token); err != nil {
+		return nil, model.InternalError(fmt.Errorf("failed to fetch PAT"))
+	}
+
+	if len(pats) != 1 {
+		return nil, &model.ApiError{
+			Typ: model.ErrorInternal,
+			Err: fmt.Errorf("found zero or multiple PATs with same token, %s", token),
+		}
+	}
+
+	return &pats[0], nil
+}
+
+func (m *modelDao) GetPATByID(ctx context.Context, id string) (*model.PAT, basemodel.BaseApiError) {
+	pats := []model.PAT{}
+
+	if err := m.DB().Select(&pats, `SELECT * FROM personal_access_tokens WHERE id=?;`, id); err != nil {
+		return nil, model.InternalError(fmt.Errorf("failed to fetch PAT"))
+	}
+
+	if len(pats) != 1 {
+		return nil, &model.ApiError{
+			Typ: model.ErrorInternal,
+			Err: fmt.Errorf("found zero or multiple PATs with same token"),
+		}
+	}
+
+	return &pats[0], nil
+}
+
+func (m *modelDao) GetUserByPAT(ctx context.Context, token string) (*basemodel.UserPayload, basemodel.BaseApiError) {
+	users := []basemodel.UserPayload{}
+
+	query := `SELECT
+				u.id,
+				u.name,
+				u.email,
+				u.password,
+				u.created_at,
+				u.profile_picture_url,
+				u.org_id,
+				u.group_id
+			  FROM users u, personal_access_tokens p
+			  WHERE u.id = p.user_id and p.token=?;`
+
+	if err := m.DB().Select(&users, query, token); err != nil {
+		return nil, model.InternalError(fmt.Errorf("failed to fetch user from PAT, err: %v", err))
+	}
+
+	if len(users) != 1 {
+		return nil, &model.ApiError{
+			Typ: model.ErrorInternal,
+			Err: fmt.Errorf("found zero or multiple users with same PAT token"),
+		}
+	}
+	return &users[0], nil
+}

ee/query-service/integrations/signozio/signozio.go

@@ -127,7 +127,7 @@ func NewPostRequestWithCtx(ctx context.Context, url string, contentType string,
 }
 
 // SendUsage reports the usage of signoz to license server
-func SendUsage(ctx context.Context, usage *model.UsagePayload) *model.ApiError {
+func SendUsage(ctx context.Context, usage model.UsagePayload) *model.ApiError {
 	reqString, _ := json.Marshal(usage)
 	req, err := NewPostRequestWithCtx(ctx, C.Prefix+"/usage", APPLICATION_JSON, bytes.NewBuffer(reqString))
 	if err != nil {

ee/query-service/license/db.go

@@ -2,6 +2,7 @@ package license
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -9,6 +10,7 @@ import (
 
 	"go.signoz.io/signoz/ee/query-service/license/sqlite"
 	"go.signoz.io/signoz/ee/query-service/model"
+	basemodel "go.signoz.io/signoz/pkg/query-service/model"
 	"go.uber.org/zap"
 )
 
@@ -125,3 +127,79 @@ func (r *Repo) UpdatePlanDetails(ctx context.Context,
 
 	return nil
 }
+
+func (r *Repo) CreateFeature(req *basemodel.Feature) *basemodel.ApiError {
+
+	_, err := r.db.Exec(
+		`INSERT INTO feature_status (name, active, usage, usage_limit, route)
+		VALUES (?, ?, ?, ?, ?);`,
+		req.Name, req.Active, req.Usage, req.UsageLimit, req.Route)
+	if err != nil {
+		return &basemodel.ApiError{Typ: basemodel.ErrorInternal, Err: err}
+	}
+	return nil
+}
+
+func (r *Repo) GetFeature(featureName string) (basemodel.Feature, error) {
+
+	var feature basemodel.Feature
+
+	err := r.db.Get(&feature,
+		`SELECT * FROM feature_status WHERE name = ?;`, featureName)
+	if err != nil {
+		return feature, err
+	}
+	if feature.Name == "" {
+		return feature, basemodel.ErrFeatureUnavailable{Key: featureName}
+	}
+	return feature, nil
+}
+
+func (r *Repo) GetAllFeatures() ([]basemodel.Feature, error) {
+
+	var feature []basemodel.Feature
+
+	err := r.db.Select(&feature,
+		`SELECT * FROM feature_status;`)
+	if err != nil {
+		return feature, err
+	}
+
+	return feature, nil
+}
+
+func (r *Repo) UpdateFeature(req basemodel.Feature) error {
+
+	_, err := r.db.Exec(
+		`UPDATE feature_status SET active = ?, usage = ?, usage_limit = ?, route = ? WHERE name = ?;`,
+		req.Active, req.Usage, req.UsageLimit, req.Route, req.Name)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (r *Repo) InitFeatures(req basemodel.FeatureSet) error {
+	// get a feature by name, if it doesn't exist, create it. If it does exist, update it.
+	for _, feature := range req {
+		currentFeature, err := r.GetFeature(feature.Name)
+		if err != nil && err == sql.ErrNoRows {
+			err := r.CreateFeature(&feature)
+			if err != nil {
+				return err
+			}
+			continue
+		} else if err != nil {
+			return err
+		}
+		feature.Usage = currentFeature.Usage
+		if feature.Usage >= feature.UsageLimit && feature.UsageLimit != -1 {
+			feature.Active = false
+		}
+		err = r.UpdateFeature(feature)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

ee/query-service/license/manager.go

@@ -96,6 +96,11 @@ func (lm *Manager) SetActive(l *model.License) {
 	lm.activeFeatures = l.FeatureSet
 	// set default features
 	setDefaultFeatures(lm)
+
+	err := lm.InitFeatures(lm.activeFeatures)
+	if err != nil {
+		zap.S().Panicf("Couldn't activate features: %v", err)
+	}
 	if !lm.validatorRunning {
 		// we want to make sure only one validator runs,
 		// we already have lock() so good to go
@@ -106,9 +111,7 @@ func (lm *Manager) SetActive(l *model.License) {
 }
 
 func setDefaultFeatures(lm *Manager) {
-	for k, v := range baseconstants.DEFAULT_FEATURE_SET {
-		lm.activeFeatures[k] = v
-	}
+	lm.activeFeatures = append(lm.activeFeatures, baseconstants.DEFAULT_FEATURE_SET...)
 }
 
 // LoadActiveLicense loads the most recent active license
@@ -123,8 +126,13 @@ func (lm *Manager) LoadActiveLicense() error {
 	} else {
 		zap.S().Info("No active license found, defaulting to basic plan")
 		// if no active license is found, we default to basic(free) plan with all default features
-		lm.activeFeatures = basemodel.BasicPlan
+		lm.activeFeatures = model.BasicPlan
 		setDefaultFeatures(lm)
+		err := lm.InitFeatures(lm.activeFeatures)
+		if err != nil {
+			zap.S().Error("Couldn't initialize features: ", err)
+			return err
+		}
 	}
 
 	return nil
@@ -291,18 +299,31 @@ func (lm *Manager) Activate(ctx context.Context, key string) (licenseResponse *m
 // CheckFeature will be internally used by backend routines
 // for feature gating
 func (lm *Manager) CheckFeature(featureKey string) error {
-	if value, ok := lm.activeFeatures[featureKey]; ok {
-		if value {
-			return nil
-		}
-		return basemodel.ErrFeatureUnavailable{Key: featureKey}
+	feature, err := lm.repo.GetFeature(featureKey)
+	if err != nil {
+		return err
+	}
+	if feature.Active {
+		return nil
 	}
 	return basemodel.ErrFeatureUnavailable{Key: featureKey}
 }
 
 // GetFeatureFlags returns current active features
-func (lm *Manager) GetFeatureFlags() basemodel.FeatureSet {
-	return lm.activeFeatures
+func (lm *Manager) GetFeatureFlags() (basemodel.FeatureSet, error) {
+	return lm.repo.GetAllFeatures()
+}
+
+func (lm *Manager) InitFeatures(features basemodel.FeatureSet) error {
+	return lm.repo.InitFeatures(features)
+}
+
+func (lm *Manager) UpdateFeatureFlag(feature basemodel.Feature) error {
+	return lm.repo.UpdateFeature(feature)
+}
+
+func (lm *Manager) GetFeatureFlag(key string) (basemodel.Feature, error) {
+	return lm.repo.GetFeature(key)
 }
 
 // GetRepo return the license repo

ee/query-service/license/sqlite/init.go

@@ -2,6 +2,7 @@ package sqlite
 
 import (
 	"fmt"
+
 	"github.com/jmoiron/sqlx"
 )
 
@@ -33,5 +34,19 @@ func InitDB(db *sqlx.DB) error {
 	if err != nil {
 		return fmt.Errorf("Error in creating licenses table: %s", err.Error())
 	}
+
+	table_schema = `CREATE TABLE IF NOT EXISTS feature_status (
+		name TEXT PRIMARY KEY,
+		active bool,
+		usage INTEGER DEFAULT 0,
+		usage_limit INTEGER DEFAULT 0,
+		route TEXT
+	);`
+
+	_, err = db.Exec(table_schema)
+	if err != nil {
+		return fmt.Errorf("Error in creating feature_status table: %s", err.Error())
+	}
+
 	return nil
 }

ee/query-service/model/domain.go

@@ -9,8 +9,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	saml2 "github.com/russellhaering/gosaml2"
-	"go.signoz.io/signoz/ee/query-service/saml"
+	"go.signoz.io/signoz/ee/query-service/sso/saml"
+	"go.signoz.io/signoz/ee/query-service/sso"
 	basemodel "go.signoz.io/signoz/pkg/query-service/model"
+	"go.uber.org/zap"
 )
 
 type SSOType string
@@ -20,12 +22,6 @@ const (
 	GoogleAuth SSOType = "GOOGLE_AUTH"
 )
 
-type SamlConfig struct {
-	SamlEntity string `json:"samlEntity"`
-	SamlIdp    string `json:"samlIdp"`
-	SamlCert   string `json:"samlCert"`
-}
-
 // OrgDomain identify org owned web domains for auth and other purposes
 type OrgDomain struct {
 	Id         uuid.UUID   `json:"id"`
@@ -33,10 +29,17 @@ type OrgDomain struct {
 	OrgId      string      `json:"orgId"`
 	SsoEnabled bool        `json:"ssoEnabled"`
 	SsoType    SSOType     `json:"ssoType"`
+
 	SamlConfig *SamlConfig `json:"samlConfig"`
+	GoogleAuthConfig *GoogleOAuthConfig `json:"googleAuthConfig"`
+
 	Org        *basemodel.Organization
 }
 
+func (od *OrgDomain) String() string {
+	return fmt.Sprintf("[%s]%s-%s ", od.Name, od.Id.String(), od.SsoType)
+}
+
 // Valid is used a pipeline function to check if org domain
 // loaded from db is valid
 func (od *OrgDomain) Valid(err error) error {
@@ -97,6 +100,16 @@ func (od *OrgDomain) GetSAMLCert() string {
 	return ""
 }
 
+// PrepareGoogleOAuthProvider creates GoogleProvider that is used in 
+// requesting OAuth and also used in processing response from google 
+func (od *OrgDomain) PrepareGoogleOAuthProvider(siteUrl *url.URL) (sso.OAuthCallbackProvider, error) {
+	if od.GoogleAuthConfig == nil {
+		return nil, fmt.Errorf("Google auth is not setup correctly for this domain")
+	}
+
+	return od.GoogleAuthConfig.GetProvider(od.Name, siteUrl)
+}
+
 // PrepareSamlRequest creates a request accordingly gosaml2
 func (od *OrgDomain) PrepareSamlRequest(siteUrl *url.URL) (*saml2.SAMLServiceProvider, error) {
 
@@ -125,18 +138,47 @@ func (od *OrgDomain) PrepareSamlRequest(siteUrl *url.URL) (*saml2.SAMLServicePro
 
 func (od *OrgDomain) BuildSsoUrl(siteUrl *url.URL) (ssoUrl string, err error) {
 	
-	sp, err := od.PrepareSamlRequest(siteUrl)
-	if err != nil {
-		return "", err
-	}
 
 	fmtDomainId := strings.Replace(od.Id.String(), "-", ":", -1)
 	
-	relayState := fmt.Sprintf("%s://%s%s?domainId=%s",
-		siteUrl.Scheme,
-		siteUrl.Host,
-		siteUrl.Path,
-		fmtDomainId)
+	// build redirect url from window.location sent by frontend
+	redirectURL := fmt.Sprintf("%s://%s%s", siteUrl.Scheme, siteUrl.Host, siteUrl.Path)
+
+	// prepare state that gets relayed back when the auth provider
+	// calls back our url. here we pass the app url (where signoz runs)
+	// and the domain Id. The domain Id helps in identifying sso config
+	// when the call back occurs and the app url is useful in redirecting user 
+	// back to the right path. 
+	// why do we need to pass app url? the callback typically is handled by backend
+	// and sometimes backend might right at a different port or is unaware of frontend
+	// endpoint (unless SITE_URL param is set). hence, we receive this build sso request
+	// along with frontend window.location and use it to relay the information through 
+	// auth provider to the backend (HandleCallback or HandleSSO method). 
+	relayState := fmt.Sprintf("%s?domainId=%s", redirectURL, fmtDomainId)
+		
+
+	switch (od.SsoType) {
+	case SAML:
+
+		sp, err := od.PrepareSamlRequest(siteUrl)
+		if err != nil {
+			return "", err
+		}
+	
+		return sp.BuildAuthURL(relayState)
+	
+	case GoogleAuth:
+		
+		googleProvider, err := od.PrepareGoogleOAuthProvider(siteUrl)
+		if err != nil {
+			return "", err
+		}
+		return googleProvider.BuildAuthURL(relayState)
+
+	default:
+		zap.S().Errorf("found unsupported SSO config for the org domain", zap.String("orgDomain", od.Name))
+		return "", fmt.Errorf("unsupported SSO config for the domain") 
+	}
+
 
-	return sp.BuildAuthURL(relayState)
 }

ee/query-service/model/errors.go

@@ -1,6 +1,7 @@
 package model
 
 import (
+	"fmt"
 	basemodel "go.signoz.io/signoz/pkg/query-service/model"
 )
 
@@ -44,6 +45,14 @@ func BadRequest(err error) *ApiError {
 	}
 }
 
+// BadRequestStr returns a ApiError object of bad request for string input
+func BadRequestStr(s string) *ApiError {
+	return &ApiError{
+		Typ: basemodel.ErrorBadData,
+		Err: fmt.Errorf(s),
+	}
+}
+
 // InternalError returns a ApiError object of internal type
 func InternalError(err error) *ApiError {
 	return &ApiError{
@@ -52,6 +61,14 @@ func InternalError(err error) *ApiError {
 	}
 }
 
+
+// InternalErrorStr returns a ApiError object of internal type for string input
+func InternalErrorStr(s string) *ApiError {
+	return &ApiError{
+		Typ: basemodel.ErrorInternal,
+		Err: fmt.Errorf(s),
+	}
+}
 var (
 	ErrorNone                  basemodel.ErrorType = ""
 	ErrorTimeout               basemodel.ErrorType = "timeout"

ee/query-service/model/pat.go

@@ -0,0 +1,10 @@
+package model
+
+type PAT struct {
+	Id        string `json:"id" db:"id"`
+	UserID    string `json:"userId" db:"user_id"`
+	Token     string `json:"token" db:"token"`
+	Name      string `json:"name" db:"name"`
+	CreatedAt int64  `json:"createdAt" db:"created_at"`
+	ExpiresAt int64  `json:"expiresAt" db:"expires_at"` // unused as of now
+}

ee/query-service/model/plans.go

@@ -11,21 +11,143 @@ const Enterprise = "ENTERPRISE_PLAN"
 const DisableUpsell = "DISABLE_UPSELL"
 
 var BasicPlan = basemodel.FeatureSet{
-	Basic:         true,
-	SSO:           false,
-	DisableUpsell: false,
+	basemodel.Feature{
+		Name:       SSO,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.OSS,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       DisableUpsell,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.SmartTraceDetail,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.CustomMetricsFunction,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderPanels,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: 5,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderAlerts,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: 5,
+		Route:      "",
+	},
 }
 
 var ProPlan = basemodel.FeatureSet{
-	Pro:                             true,
-	SSO:                             true,
-	basemodel.SmartTraceDetail:      true,
-	basemodel.CustomMetricsFunction: true,
+	basemodel.Feature{
+		Name:       SSO,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.OSS,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.SmartTraceDetail,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.CustomMetricsFunction,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderPanels,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderAlerts,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }
 
 var EnterprisePlan = basemodel.FeatureSet{
-	Enterprise:                      true,
-	SSO:                             true,
-	basemodel.SmartTraceDetail:      true,
-	basemodel.CustomMetricsFunction: true,
+	basemodel.Feature{
+		Name:       SSO,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.OSS,
+		Active:     false,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.SmartTraceDetail,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.CustomMetricsFunction,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderPanels,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
+	basemodel.Feature{
+		Name:       basemodel.QueryBuilderAlerts,
+		Active:     true,
+		Usage:      0,
+		UsageLimit: -1,
+		Route:      "",
+	},
 }

ee/query-service/model/sso.go

@@ -0,0 +1,68 @@
+package model
+
+import (
+	"fmt"
+	"context"
+	"net/url"
+	"golang.org/x/oauth2"
+	"github.com/coreos/go-oidc/v3/oidc"
+	"go.signoz.io/signoz/ee/query-service/sso"
+)
+
+// SamlConfig contans SAML params to generate and respond to the requests 
+// from SAML provider
+type SamlConfig struct {
+	SamlEntity string `json:"samlEntity"`
+	SamlIdp    string `json:"samlIdp"`
+	SamlCert   string `json:"samlCert"`
+}
+
+// GoogleOauthConfig contains a generic config to support oauth 
+type GoogleOAuthConfig struct {
+	ClientID     string `json:"clientId"`
+	ClientSecret string `json:"clientSecret"`
+	RedirectURI  string `json:"redirectURI"`
+}
+
+
+const (
+	googleIssuerURL = "https://accounts.google.com"
+)
+
+func (g *GoogleOAuthConfig) GetProvider(domain string, siteUrl *url.URL) (sso.OAuthCallbackProvider, error) {
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	provider, err := oidc.NewProvider(ctx, googleIssuerURL)
+	if err != nil {
+		cancel()
+		return nil, fmt.Errorf("failed to get provider: %v", err)
+	}
+
+	// default to email and profile scope as we just use google auth
+	// to verify identity and start a session. 
+	scopes := []string{"email"}
+
+	// this is the url google will call after login completion
+	redirectURL := fmt.Sprintf("%s://%s/%s",
+		siteUrl.Scheme,
+		siteUrl.Host,
+		"api/v1/complete/google")
+
+	return &sso.GoogleOAuthProvider{
+		RedirectURI: g.RedirectURI,
+		OAuth2Config: &oauth2.Config{
+			ClientID:     g.ClientID,
+			ClientSecret: g.ClientSecret,
+			Endpoint:     provider.Endpoint(),
+			Scopes:       scopes,
+			RedirectURL:  redirectURL,
+		},
+		Verifier: provider.Verifier(
+			&oidc.Config{ClientID: g.ClientID},
+		),
+		Cancel:                         cancel,
+		HostedDomain:                  domain,
+	}, nil
+}
+

ee/query-service/model/usage.go

@@ -6,30 +6,27 @@ import (
 	"github.com/google/uuid"
 )
 
-type UsageSnapshot struct {
-	CurrentLogSizeBytes            uint64 `json:"currentLogSizeBytes"`
-	CurrentLogSizeBytesColdStorage uint64 `json:"currentLogSizeBytesColdStorage"`
-	CurrentSpansCount              uint64 `json:"currentSpansCount"`
-	CurrentSpansCountColdStorage   uint64 `json:"currentSpansCountColdStorage"`
-	CurrentSamplesCount            uint64 `json:"currentSamplesCount"`
-	CurrentSamplesCountColdStorage uint64 `json:"currentSamplesCountColdStorage"`
-}
-
-type UsageBase struct {
-	Id                uuid.UUID `json:"id" db:"id"`
-	InstallationId    uuid.UUID `json:"installationId" db:"installation_id"`
-	ActivationId      uuid.UUID `json:"activationId" db:"activation_id"`
-	CreatedAt         time.Time `json:"createdAt" db:"created_at"`
-	FailedSyncRequest int       `json:"failedSyncRequest" db:"failed_sync_request_count"`
-}
-
 type UsagePayload struct {
-	UsageBase
-	Metrics      UsageSnapshot `json:"metrics"`
-	SnapshotDate time.Time     `json:"snapshotDate"`
+	InstallationId uuid.UUID `json:"installationId"`
+	LicenseKey     uuid.UUID `json:"licenseKey"`
+	Usage          []Usage   `json:"usage"`
 }
 
 type Usage struct {
-	UsageBase
-	Snapshot string `db:"snapshot"`
+	CollectorID string    `json:"collectorId"`
+	ExporterID  string    `json:"exporterId"`
+	Type        string    `json:"type"`
+	Tenant      string    `json:"tenant"`
+	TimeStamp   time.Time `json:"timestamp"`
+	Count       int64     `json:"count"`
+	Size        int64     `json:"size"`
+}
+
+type UsageDB struct {
+	CollectorID string    `ch:"collector_id" json:"collectorId"`
+	ExporterID  string    `ch:"exporter_id" json:"exporterId"`
+	Type        string    `ch:"-" json:"type"`
+	TimeStamp   time.Time `ch:"timestamp" json:"timestamp"`
+	Tenant      string    `ch:"tenant" json:"tenant"`
+	Data        string    `ch:"data" json:"data"`
 }

ee/query-service/sso/google.go

@@ -0,0 +1,92 @@
+package sso
+
+import (
+	"fmt"
+	"errors"
+	"context"
+	"net/http"
+	"github.com/coreos/go-oidc/v3/oidc"
+	"golang.org/x/oauth2"
+)
+
+type GoogleOAuthProvider struct {
+	RedirectURI                    string
+	OAuth2Config                   *oauth2.Config
+	Verifier                       *oidc.IDTokenVerifier
+	Cancel                         context.CancelFunc
+	HostedDomain 				   string
+}
+
+
+func (g *GoogleOAuthProvider) BuildAuthURL(state string) (string, error) {
+	var opts []oauth2.AuthCodeOption
+	
+	// set hosted domain. google supports multiple hosted domains but in our case
+	// we have one config per host domain. 
+	opts = append(opts, oauth2.SetAuthURLParam("hd", g.HostedDomain))
+
+	return g.OAuth2Config.AuthCodeURL(state, opts...), nil
+}
+
+type oauth2Error struct{
+	error            string
+	errorDescription string
+}
+
+func (e *oauth2Error) Error() string {
+	if e.errorDescription == "" {
+		return e.error
+	}
+	return e.error + ": " + e.errorDescription
+}
+
+func (g *GoogleOAuthProvider) HandleCallback(r *http.Request) (identity *SSOIdentity, err error) {
+	q := r.URL.Query()
+	if errType := q.Get("error"); errType != "" {
+		return identity, &oauth2Error{errType, q.Get("error_description")}
+	}
+
+	token, err := g.OAuth2Config.Exchange(r.Context(), q.Get("code"))
+	if err != nil {
+		return identity, fmt.Errorf("google: failed to get token: %v", err)
+	}
+
+	return g.createIdentity(r.Context(), token)
+}
+
+	
+func (g *GoogleOAuthProvider) createIdentity(ctx context.Context, token *oauth2.Token) (identity *SSOIdentity, err error) {
+	rawIDToken, ok := token.Extra("id_token").(string)
+	if !ok {
+		return identity, errors.New("google: no id_token in token response")
+	}
+	idToken, err := g.Verifier.Verify(ctx, rawIDToken)
+	if err != nil {
+		return identity, fmt.Errorf("google: failed to verify ID Token: %v", err)
+	}
+
+	var claims struct {
+		Username      string `json:"name"`
+		Email         string `json:"email"`
+		EmailVerified bool   `json:"email_verified"`
+		HostedDomain  string `json:"hd"`
+	}
+	if err := idToken.Claims(&claims); err != nil {
+		return identity, fmt.Errorf("oidc: failed to decode claims: %v", err)
+	}
+
+	if claims.HostedDomain != g.HostedDomain {
+		return identity, fmt.Errorf("oidc: unexpected hd claim %v", claims.HostedDomain)	
+	}
+
+	identity = &SSOIdentity{
+		UserID:        idToken.Subject,
+		Username:      claims.Username,
+		Email:         claims.Email,
+		EmailVerified: claims.EmailVerified,
+		ConnectorData: []byte(token.RefreshToken),
+	}
+
+	return identity, nil
+}
+

ee/query-service/sso/model.go

@@ -0,0 +1,31 @@
+package sso 
+
+import (
+	"net/http"
+)
+
+// SSOIdentity contains details of user received from SSO provider 
+type SSOIdentity struct {
+	UserID            string
+	Username          string
+	PreferredUsername string
+	Email             string
+	EmailVerified     bool
+	ConnectorData []byte
+}
+
+// OAuthCallbackProvider is an interface implemented by connectors which use an OAuth
+// style redirect flow to determine user information.
+type OAuthCallbackProvider interface {
+	// The initial URL user would be redirect to.  
+	// OAuth2 implementations support various scopes but we only need profile and user as 
+	// the roles are still being managed in SigNoz. 
+	BuildAuthURL(state string) (string, error)
+
+	// Handle the callback to the server (after login at oauth provider site)
+	// and return a email identity. 
+	// At the moment we dont support auto signup flow (based on domain), so 
+	// the full identity (including name, group etc) is not required outside of the 
+	// connector 
+	HandleCallback(r *http.Request) (identity *SSOIdentity, err error)
+}

ee/query-service/usage/manager.go

@@ -4,18 +4,19 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strings"
 	"sync/atomic"
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/google/uuid"
 	"github.com/jmoiron/sqlx"
+
 	"go.uber.org/zap"
 
 	licenseserver "go.signoz.io/signoz/ee/query-service/integrations/signozio"
 	"go.signoz.io/signoz/ee/query-service/license"
 	"go.signoz.io/signoz/ee/query-service/model"
-	"go.signoz.io/signoz/ee/query-service/usage/repository"
 	"go.signoz.io/signoz/pkg/query-service/utils/encryption"
 )
 
@@ -27,9 +28,6 @@ const (
 )
 
 var (
-	// collect usage every hour
-	collectionFrequency = 1 * time.Hour
-
 	// send usage every 24 hour
 	uploadFrequency = 24 * time.Hour
 
@@ -37,8 +35,6 @@ var (
 )
 
 type Manager struct {
-	repository *repository.Repository
-
 	clickhouseConn clickhouse.Conn
 
 	licenseRepo *license.Repo
@@ -52,15 +48,9 @@ type Manager struct {
 }
 
 func New(dbType string, db *sqlx.DB, licenseRepo *license.Repo, clickhouseConn clickhouse.Conn) (*Manager, error) {
-	repo := repository.New(db)
-
-	err := repo.Init(dbType)
-	if err != nil {
-		return nil, fmt.Errorf("failed to initiate usage repo: %v", err)
-	}
 
 	m := &Manager{
-		repository:     repo,
+		// repository:     repo,
 		clickhouseConn: clickhouseConn,
 		licenseRepo:    licenseRepo,
 	}
@@ -74,6 +64,28 @@ func (lm *Manager) Start() error {
 		return fmt.Errorf("usage exporter is locked")
 	}
 
+	go lm.UsageExporter(context.Background())
+
+	return nil
+}
+
+func (lm *Manager) UsageExporter(ctx context.Context) {
+	defer close(lm.terminated)
+
+	uploadTicker := time.NewTicker(uploadFrequency)
+	defer uploadTicker.Stop()
+
+	for {
+		select {
+		case <-lm.done:
+			return
+		case <-uploadTicker.C:
+			lm.UploadUsage(ctx)
+		}
+	}
+}
+
+func (lm *Manager) UploadUsage(ctx context.Context) error {
 	// check if license is present or not
 	license, err := lm.licenseRepo.GetActiveLicense(context.Background())
 	if err != nil {
@@ -85,203 +97,81 @@ func (lm *Manager) Start() error {
 		return nil
 	}
 
-	// upload previous snapshots if any
-	err = lm.UploadUsage(context.Background())
-	if err != nil {
-		return err
-	}
-
-	// collect snapshot if incase it wasn't collect in (t - collectionFrequency)
-	err = lm.CollectCurrentUsage(context.Background())
-	if err != nil {
-		return err
-	}
-
-	go lm.UsageExporter(context.Background())
-
-	return nil
-}
-
-// CollectCurrentUsage checks if needs to collect usage data
-func (lm *Manager) CollectCurrentUsage(ctx context.Context) error {
-	// check the DB if anything exist  where timestamp > t - collectionFrequency
-	ts := time.Now().Add(-collectionFrequency)
-	alreadyCreated, err := lm.repository.CheckSnapshotGtCreatedAt(ctx, ts)
-	if err != nil {
-		return err
-	}
-	if !alreadyCreated {
-		zap.S().Info("Collecting current usage")
-		exportError := lm.CollectAndStoreUsage(ctx)
-		if exportError != nil {
-			return exportError
-		}
-	} else {
-		zap.S().Info("Nothing to collect")
-	}
-	return nil
-}
-
-func (lm *Manager) UsageExporter(ctx context.Context) {
-	defer close(lm.terminated)
-
-	collectionTicker := time.NewTicker(collectionFrequency)
-	defer collectionTicker.Stop()
-
-	uploadTicker := time.NewTicker(uploadFrequency)
-	defer uploadTicker.Stop()
-
-	for {
-		select {
-		case <-lm.done:
-			return
-		case <-collectionTicker.C:
-			lm.CollectAndStoreUsage(ctx)
-		case <-uploadTicker.C:
-			lm.UploadUsage(ctx)
-			// remove the old snapshots
-			lm.repository.DropOldSnapshots(ctx)
-		}
-	}
-}
-
-type TableSize struct {
-	Table             string `ch:"table"`
-	DiskName          string `ch:"disk_name"`
-	Rows              uint64 `ch:"rows"`
-	UncompressedBytes uint64 `ch:"uncompressed_bytes"`
-}
-
-func (lm *Manager) CollectAndStoreUsage(ctx context.Context) error {
-	snap, err := lm.GetUsageFromClickHouse(ctx)
-	if err != nil {
-		return err
-	}
-
-	license, err := lm.licenseRepo.GetActiveLicense(ctx)
-	if err != nil {
-		return err
-	}
-
-	activationId, _ := uuid.Parse(license.ActivationId)
-	// TODO (nitya) : Add installation ID in the payload
-	payload := model.UsagePayload{
-		UsageBase: model.UsageBase{
-			ActivationId:      activationId,
-			FailedSyncRequest: 0,
-		},
-		Metrics:      *snap,
-		SnapshotDate: time.Now(),
-	}
-
-	err = lm.repository.InsertSnapshot(ctx, &payload)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (lm *Manager) GetUsageFromClickHouse(ctx context.Context) (*model.UsageSnapshot, error) {
-	tableSizes := []TableSize{}
-	snap := model.UsageSnapshot{}
+	usages := []model.UsageDB{}
 
 	// get usage from clickhouse
+	dbs := []string{"signoz_logs", "signoz_traces", "signoz_metrics"}
 	query := `
-		SELECT
-			table,
-			disk_name,
-			sum(rows) as rows,
-			sum(data_uncompressed_bytes) AS uncompressed_bytes
-		FROM system.parts
-		WHERE active AND (database in ('signoz_logs', 'signoz_metrics', 'signoz_traces')) AND (table in ('logs','samples_v2', 'signoz_index_v2'))
-		GROUP BY
-			table,
-			disk_name
-		ORDER BY table
+		SELECT tenant, collector_id, exporter_id, timestamp, data
+		FROM %s.distributed_usage as u1 
+			GLOBAL INNER JOIN 
+				(SELECT  
+					tenant, collector_id, exporter_id, MAX(timestamp) as ts 
+					FROM %s.distributed_usage as u2 
+					where timestamp >= $1 
+					GROUP BY tenant, collector_id, exporter_id 
+				) as t1
+		ON 
+		u1.tenant = t1.tenant AND u1.collector_id = t1.collector_id AND u1.exporter_id = t1.exporter_id and u1.timestamp = t1.ts 
+		order by timestamp
 	`
-	err := lm.clickhouseConn.Select(ctx, &tableSizes, query)
-	if err != nil {
-		return nil, err
-	}
 
-	for _, val := range tableSizes {
-		switch val.Table {
-		case "logs":
-			if val.DiskName == "default" {
-				snap.CurrentLogSizeBytes = val.UncompressedBytes
-			} else {
-				snap.CurrentLogSizeBytesColdStorage = val.UncompressedBytes
-			}
-		case "samples_v2":
-			if val.DiskName == "default" {
-				snap.CurrentSamplesCount = val.Rows
-			} else {
-				snap.CurrentSamplesCountColdStorage = val.Rows
-			}
-		case "signoz_index_v2":
-			if val.DiskName == "default" {
-				snap.CurrentSpansCount = val.Rows
-			} else {
-				snap.CurrentSpansCountColdStorage = val.Rows
-			}
+	for _, db := range dbs {
+		dbusages := []model.UsageDB{}
+		err := lm.clickhouseConn.Select(ctx, &dbusages, fmt.Sprintf(query, db, db), time.Now().Add(-(24 * time.Hour)))
+		if err != nil && !strings.Contains(err.Error(), "doesn't exist") {
+			return err
+		}
+		for _, u := range dbusages {
+			u.Type = db
+			usages = append(usages, u)
 		}
 	}
 
-	return &snap, nil
-}
-
-func (lm *Manager) UploadUsage(ctx context.Context) error {
-	snapshots, err := lm.repository.GetSnapshotsNotSynced(ctx)
-	if err != nil {
-		return err
-	}
-
-	if len(snapshots) <= 0 {
+	if len(usages) <= 0 {
 		zap.S().Info("no snapshots to upload, skipping.")
 		return nil
 	}
 
-	zap.S().Info("uploading snapshots")
-	for _, snap := range snapshots {
-		metricsBytes, err := encryption.Decrypt([]byte(snap.ActivationId.String()[:32]), []byte(snap.Snapshot))
+	zap.S().Info("uploading usage data")
+
+	usagesPayload := []model.Usage{}
+	for _, usage := range usages {
+		usageDataBytes, err := encryption.Decrypt([]byte(usage.ExporterID[:32]), []byte(usage.Data))
 		if err != nil {
 			return err
 		}
 
-		metrics := model.UsageSnapshot{}
-		err = json.Unmarshal(metricsBytes, &metrics)
+		usageData := model.Usage{}
+		err = json.Unmarshal(usageDataBytes, &usageData)
 		if err != nil {
 			return err
 		}
 
-		err = lm.UploadUsageWithExponentalBackOff(ctx, model.UsagePayload{
-			UsageBase: model.UsageBase{
-				Id:                snap.Id,
-				InstallationId:    snap.InstallationId,
-				ActivationId:      snap.ActivationId,
-				FailedSyncRequest: snap.FailedSyncRequest,
-			},
-			SnapshotDate: snap.CreatedAt,
-			Metrics:      metrics,
-		})
-		if err != nil {
-			return err
-		}
+		usageData.CollectorID = usage.CollectorID
+		usageData.ExporterID = usage.ExporterID
+		usageData.Type = usage.Type
+		usageData.Tenant = usage.Tenant
+		usagesPayload = append(usagesPayload, usageData)
+	}
+
+	key, _ := uuid.Parse(license.Key)
+	payload := model.UsagePayload{
+		LicenseKey: key,
+		Usage:      usagesPayload,
+	}
+	err = lm.UploadUsageWithExponentalBackOff(ctx, payload)
+	if err != nil {
+		return err
 	}
 	return nil
 }
 
 func (lm *Manager) UploadUsageWithExponentalBackOff(ctx context.Context, payload model.UsagePayload) error {
 	for i := 1; i <= MaxRetries; i++ {
-		apiErr := licenseserver.SendUsage(ctx, &payload)
+		apiErr := licenseserver.SendUsage(ctx, payload)
 		if apiErr != nil && i == MaxRetries {
-			err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
-			if err != nil {
-				zap.S().Errorf("failed to updated the failure count for snapshot in DB : ", zap.Error(err))
-				return err
-			}
-			zap.S().Errorf("retries stopped : %v", zap.Error(err))
+			zap.S().Errorf("retries stopped : %v", zap.Error(apiErr))
 			// not returning error here since it is captured in the failed count
 			return nil
 		} else if apiErr != nil {
@@ -289,24 +179,10 @@ func (lm *Manager) UploadUsageWithExponentalBackOff(ctx context.Context, payload
 			sleepDuration := RetryInterval * time.Duration(i)
 			zap.S().Errorf("failed to upload snapshot retrying after %v secs : %v", sleepDuration.Seconds(), zap.Error(apiErr.Err))
 			time.Sleep(sleepDuration)
-
-			// update the failed request count
-			err := lm.repository.IncrementFailedRequestCount(ctx, payload.Id)
-			if err != nil {
-				zap.S().Errorf("failed to updated the failure count for snapshot in DB : %v", zap.Error(err))
-				return err
-			}
 		} else {
 			break
 		}
 	}
-
-	// update the database that it is synced
-	err := lm.repository.MoveToSynced(ctx, payload.Id)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

ee/query-service/usage/repository/repository.go

@@ -1,139 +0,0 @@
-package repository
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/jmoiron/sqlx"
-	"go.uber.org/zap"
-
-	"go.signoz.io/signoz/ee/query-service/model"
-	"go.signoz.io/signoz/ee/query-service/usage/sqlite"
-	"go.signoz.io/signoz/pkg/query-service/utils/encryption"
-)
-
-const (
-	MaxFailedSyncCount = 9 // a snapshot will be ignored if the max failed count is greater than or equal to 9
-	SnapShotLife       = 3 * 24 * time.Hour
-)
-
-// Repository is usage Repository which stores usage snapshot in a secured DB
-type Repository struct {
-	db *sqlx.DB
-}
-
-// New initiates a new usage Repository
-func New(db *sqlx.DB) *Repository {
-	return &Repository{
-		db: db,
-	}
-}
-
-func (r *Repository) Init(engine string) error {
-	switch engine {
-	case "sqlite3", "sqlite":
-		return sqlite.InitDB(r.db)
-	default:
-		return fmt.Errorf("unsupported db")
-	}
-}
-
-func (r *Repository) InsertSnapshot(ctx context.Context, usage *model.UsagePayload) error {
-
-	snapshotBytes, err := json.Marshal(usage.Metrics)
-	if err != nil {
-		return err
-	}
-
-	usage.Id = uuid.New()
-
-	encryptedSnapshot, err := encryption.Encrypt([]byte(usage.ActivationId.String()[:32]), snapshotBytes)
-	if err != nil {
-		return err
-	}
-
-	query := `INSERT INTO usage(id, activation_id, snapshot)
- 				VALUES ($1, $2, $3)`
-	_, err = r.db.ExecContext(ctx,
-		query,
-		usage.Id,
-		usage.ActivationId,
-		string(encryptedSnapshot),
-	)
-	if err != nil {
-		zap.S().Errorf("error inserting usage data: %v", zap.Error(err))
-		return fmt.Errorf("failed to insert usage in db: %v", err)
-	}
-	return nil
-}
-
-func (r *Repository) MoveToSynced(ctx context.Context, id uuid.UUID) error {
-
-	query := `UPDATE usage 
-						SET synced = 'true',
-						synced_at = $1
-						WHERE id = $2`
-
-	_, err := r.db.ExecContext(ctx, query, time.Now(), id)
-
-	if err != nil {
-		zap.S().Errorf("error in updating usage: %v", zap.Error(err))
-		return fmt.Errorf("failed to update usage in db: %v", err)
-	}
-
-	return nil
-}
-
-func (r *Repository) IncrementFailedRequestCount(ctx context.Context, id uuid.UUID) error {
-
-	query := `UPDATE usage SET failed_sync_request_count = failed_sync_request_count + 1 WHERE id = $1`
-	_, err := r.db.ExecContext(ctx, query, id)
-	if err != nil {
-		zap.S().Errorf("error in updating usage: %v", zap.Error(err))
-		return fmt.Errorf("failed to update usage in db: %v", err)
-	}
-
-	return nil
-}
-
-func (r *Repository) GetSnapshotsNotSynced(ctx context.Context) ([]*model.Usage, error) {
-	snapshots := []*model.Usage{}
-
-	query := `SELECT id,created_at, activation_id, snapshot, failed_sync_request_count from usage where synced!='true' and failed_sync_request_count < $1 order by created_at asc `
-
-	err := r.db.SelectContext(ctx, &snapshots, query, MaxFailedSyncCount)
-	if err != nil {
-		return nil, err
-	}
-
-	return snapshots, nil
-}
-
-func (r *Repository) DropOldSnapshots(ctx context.Context) error {
-	query := `delete from usage where created_at <= $1`
-
-	_, err := r.db.ExecContext(ctx, query, time.Now().Add(-(SnapShotLife)))
-	if err != nil {
-		zap.S().Errorf("failed to remove old snapshots from db: %v", zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-// CheckSnapshotGtCreatedAt checks if there is any snapshot greater than the provided timestamp
-func (r *Repository) CheckSnapshotGtCreatedAt(ctx context.Context, ts time.Time) (bool, error) {
-
-	var snapshots uint64
-	query := `SELECT count() from usage where created_at > '$1'`
-
-	err := r.db.QueryRowContext(ctx, query, ts).Scan(&snapshots)
-	if err != nil {
-		return false, err
-	}
-
-	return snapshots > 0, err
-}

ee/query-service/usage/sqlite/init.go

@@ -1,32 +0,0 @@
-package sqlite
-
-import (
-	"fmt"
-
-	"github.com/jmoiron/sqlx"
-)
-
-func InitDB(db *sqlx.DB) error {
-	var err error
-	if db == nil {
-		return fmt.Errorf("invalid db connection")
-	}
-
-	table_schema := `CREATE TABLE IF NOT EXISTS usage(
-		id UUID PRIMARY KEY,
-		created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, 
-		updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-		activation_id UUID, 
-		snapshot TEXT,
-		synced BOOLEAN DEFAULT 'false',
-		synced_at TIMESTAMP,
-		failed_sync_request_count INTEGER DEFAULT 0
-	);
-	`
-
-	_, err = db.Exec(table_schema)
-	if err != nil {
-		return fmt.Errorf("error in creating usage table: %v", err.Error())
-	}
-	return nil
-}

frontend/.babelrc

@@ -1,7 +1,7 @@
 {
 	"presets": [
 		"@babel/preset-env",
-		"@babel/preset-react",
+		["@babel/preset-react", { "runtime": "automatic" }],
 		"@babel/preset-typescript"
 	],
 	"plugins": [

frontend/.eslintrc.js

@@ -16,6 +16,7 @@ module.exports = {
 		'plugin:sonarjs/recommended',
 		'plugin:import/errors',
 		'plugin:import/warnings',
+		'plugin:react/jsx-runtime',
 	],
 	parser: '@typescript-eslint/parser',
 	parserOptions: {
@@ -58,7 +59,7 @@ module.exports = {
 		'react/no-array-index-key': 'error',
 		'linebreak-style': [
 			'error',
-			process.platform === 'win32' ? 'windows' : 'unix',
+			process.env.platform === 'win32' ? 'windows' : 'unix',
 		],
 		'@typescript-eslint/default-param-last': 'off',
 
@@ -102,9 +103,10 @@ module.exports = {
 			},
 		],
 		'@typescript-eslint/no-unused-vars': 'error',
+		'func-style': ['error', 'declaration', { allowArrowFunctions: true }],
+		'arrow-body-style': ['error', 'as-needed'],
 
 		// eslint rules need to remove
-		'no-shadow': 'off',
 		'@typescript-eslint/no-shadow': 'off',
 		'import/no-cycle': 'off',
 

frontend/.husky/commit-msg

@@ -1,4 +1,4 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
-cd frontend && npm run commitlint
+cd frontend && yarn run commitlint --edit $1

frontend/.yarnrc

@@ -0,0 +1,2 @@
+network-timeout 600000
+save-prefix ""

frontend/Dockerfile

@@ -1,5 +1,5 @@
 # Builder stage
-FROM node:16.15.0-slim as builder
+FROM node:16.15.0 as builder
 
 # Add Maintainer Info
 LABEL maintainer="signoz"
@@ -9,8 +9,11 @@ ARG TARGETARCH
 
 WORKDIR /frontend
 
-# Copy the package.json to install dependencies
+# Copy the package.json and .yarnrc files prior to install dependencies
 COPY package.json ./
+# Copy lock file
+COPY yarn.lock ./
+COPY .yarnrc ./
 
 # Install the dependencies and make the folder
 RUN CI=1 yarn install

frontend/jest.config.ts

@@ -15,7 +15,7 @@ const config: Config.InitialOptions = {
 			useESM: true,
 		},
 	},
-	testMatch: ['<rootDir>/src/**/?(*.)(test).(ts|js)?(x)'],
+	testMatch: ['<rootDir>/src/**/*?(*.)(test).(ts|js)?(x)'],
 	preset: 'ts-jest/presets/js-with-ts-esm',
 	transform: {
 		'^.+\\.(ts|tsx)?$': 'ts-jest',
@@ -25,6 +25,7 @@ const config: Config.InitialOptions = {
 	setupFilesAfterEnv: ['<rootDir>jest.setup.ts'],
 	testPathIgnorePatterns: ['/node_modules/', '/public/'],
 	moduleDirectories: ['node_modules', 'src'],
+	testEnvironment: 'jest-environment-jsdom',
 	testEnvironmentOptions: {
 		'jest-playwright': {
 			browsers: ['chromium', 'firefox', 'webkit'],

frontend/jest.setup.ts

@@ -1,5 +1,20 @@
+/* eslint-disable @typescript-eslint/explicit-function-return-type */
+/* eslint-disable object-shorthand */
+/* eslint-disable func-names */
+
 /**
  * Adds custom matchers from the react testing library to all tests
  */
 import '@testing-library/jest-dom';
 import 'jest-styled-components';
+
+// Mock window.matchMedia
+window.matchMedia =
+	window.matchMedia ||
+	function (): any {
+		return {
+			matches: false,
+			addListener: function () {},
+			removeListener: function () {},
+		};
+	};

frontend/package.json

@@ -27,16 +27,13 @@
 	"author": "",
 	"license": "ISC",
 	"dependencies": {
-		"@ant-design/colors": "^6.0.0",
-		"@ant-design/icons": "^4.6.2",
+		"@ant-design/colors": "6.0.0",
+		"@ant-design/icons": "4.8.0",
 		"@grafana/data": "^8.4.3",
 		"@monaco-editor/react": "^4.3.1",
-		"@testing-library/jest-dom": "^5.11.4",
-		"@testing-library/react": "^11.1.0",
-		"@testing-library/user-event": "^12.1.10",
-		"@welldone-software/why-did-you-render": "^6.2.1",
 		"@xstate/react": "^3.0.0",
-		"antd": "4.19.2",
+		"ansi-to-html": "0.7.2",
+		"antd": "5.0.5",
 		"axios": "^0.21.0",
 		"babel-eslint": "^10.1.0",
 		"babel-jest": "^26.6.0",
@@ -44,21 +41,19 @@
 		"babel-plugin-named-asset-import": "^0.3.7",
 		"babel-preset-minify": "^0.5.1",
 		"babel-preset-react-app": "^10.0.0",
-		"chart.js": "^3.4.0",
+		"chart.js": "3.9.1",
 		"chartjs-adapter-date-fns": "^2.0.0",
 		"chartjs-plugin-annotation": "^1.4.0",
 		"color": "^4.2.1",
 		"cross-env": "^7.0.3",
 		"css-loader": "4.3.0",
 		"css-minimizer-webpack-plugin": "^3.2.0",
-		"d3": "^6.2.0",
-		"d3-flame-graph": "^3.1.1",
-		"d3-tip": "^0.9.1",
 		"dayjs": "^1.10.7",
+		"dompurify": "3.0.0",
 		"dotenv": "8.2.0",
 		"event-source-polyfill": "1.0.31",
 		"file-loader": "6.1.1",
-		"flat": "^5.0.2",
+		"fontfaceobserver": "2.3.0",
 		"history": "4.10.1",
 		"html-webpack-plugin": "5.1.0",
 		"i18next": "^21.6.12",
@@ -70,17 +65,18 @@
 		"less-loader": "^10.2.0",
 		"lodash-es": "^4.17.21",
 		"mini-css-extract-plugin": "2.4.5",
-		"react": "17.0.0",
-		"react-dom": "17.0.0",
+		"papaparse": "5.4.1",
+		"react": "18.2.0",
+		"react-dom": "18.2.0",
 		"react-force-graph": "^1.41.0",
-		"react-graph-vis": "^1.0.5",
 		"react-grid-layout": "^1.3.4",
 		"react-i18next": "^11.16.1",
+		"react-intersection-observer": "9.4.1",
 		"react-query": "^3.34.19",
 		"react-redux": "^7.2.2",
 		"react-router-dom": "^5.2.0",
 		"react-use": "^17.3.2",
-		"react-vis": "^1.11.7",
+		"react-virtuoso": "4.0.3",
 		"redux": "^4.0.5",
 		"redux-thunk": "^2.3.0",
 		"stream": "^0.0.2",
@@ -120,31 +116,33 @@
 		"@commitlint/config-conventional": "^16.2.4",
 		"@jest/globals": "^27.5.1",
 		"@playwright/test": "^1.22.0",
-		"@testing-library/react-hooks": "^7.0.2",
+		"@testing-library/jest-dom": "5.16.5",
+		"@testing-library/react": "13.4.0",
+		"@testing-library/user-event": "14.4.3",
 		"@types/color": "^3.0.3",
 		"@types/compression-webpack-plugin": "^9.0.0",
 		"@types/copy-webpack-plugin": "^8.0.1",
-		"@types/d3": "^6.2.0",
-		"@types/d3-tip": "^3.5.5",
+		"@types/dompurify": "^2.4.0",
 		"@types/event-source-polyfill": "^1.0.0",
-		"@types/flat": "^5.0.2",
+		"@types/fontfaceobserver": "2.1.0",
 		"@types/jest": "^27.5.1",
 		"@types/lodash-es": "^4.17.4",
 		"@types/mini-css-extract-plugin": "^2.5.1",
 		"@types/node": "^16.10.3",
-		"@types/react": "^17.0.0",
-		"@types/react-dom": "^16.9.9",
+		"@types/papaparse": "5.3.7",
+		"@types/react": "18.0.26",
+		"@types/react-dom": "18.0.10",
 		"@types/react-grid-layout": "^1.1.2",
 		"@types/react-redux": "^7.1.11",
+		"@types/react-resizable": "3.0.3",
 		"@types/react-router-dom": "^5.1.6",
-		"@types/redux": "^3.6.0",
 		"@types/styled-components": "^5.1.4",
 		"@types/uuid": "^8.3.1",
-		"@types/vis": "^4.21.21",
 		"@types/webpack": "^5.28.0",
 		"@types/webpack-dev-server": "^4.3.0",
 		"@typescript-eslint/eslint-plugin": "^4.28.2",
 		"@typescript-eslint/parser": "^4.28.2",
+		"@welldone-software/why-did-you-render": "6.2.1",
 		"autoprefixer": "^9.0.0",
 		"babel-plugin-styled-components": "^1.12.0",
 		"compression-webpack-plugin": "9.0.0",
@@ -169,11 +167,12 @@
 		"is-ci": "^3.0.1",
 		"jest-playwright-preset": "^1.7.0",
 		"jest-styled-components": "^7.0.8",
-		"less-plugin-npm-import": "^2.1.0",
 		"lint-staged": "^12.3.7",
 		"portfinder-sync": "^0.0.2",
 		"prettier": "2.2.1",
+		"react-hooks-testing-library": "0.6.0",
 		"react-hot-loader": "^4.13.0",
+		"react-resizable": "3.0.4",
 		"ts-jest": "^27.1.4",
 		"ts-node": "^10.2.1",
 		"typescript-plugin-css-modules": "^3.4.0",
@@ -186,7 +185,7 @@
 		]
 	},
 	"resolutions": {
-		"@types/react": "17.0.0",
-		"@types/react-dom": "17.0.0"
+		"@types/react": "18.0.26",
+		"@types/react-dom": "18.0.10"
 	}
 }

frontend/public/locales/en-GB/traceDetails.json

@@ -0,0 +1,3 @@
+{
+	"search_tags": "Search Tag Names"
+}

frontend/public/locales/en/traceDetails.json

@@ -0,0 +1,3 @@
+{
+	"search_tags": "Search Tag Names"
+}

frontend/public/locales/en/translation.json

@@ -6,7 +6,7 @@
 	"release_notes": "Release Notes",
 	"read_how_to_upgrade": "Read instructions on how to upgrade",
 	"latest_version_signoz": "You are running the latest version of SigNoz.",
-	"stale_version": "You are on an older version and may be losing out on the latest features we have shipped. We recommend to upgrade to the latest version",
+	"stale_version": "You are on an older version and may be missing out on the latest features we have shipped. We recommend to upgrade to the latest version",
 	"oops_something_went_wrong_version": "Oops.. facing issues with fetching updated version information",
 	"n_a": "N/A",
 	"routes": {

frontend/src/AppRoutes/Private.tsx

@@ -1,13 +1,13 @@
 /* eslint-disable react-hooks/exhaustive-deps */
-import { notification } from 'antd';
 import getLocalStorageApi from 'api/browser/localstorage/get';
 import loginApi from 'api/user/login';
 import { Logout } from 'api/utils';
 import Spinner from 'components/Spinner';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import ROUTES from 'constants/routes';
+import { useNotifications } from 'hooks/useNotifications';
 import history from 'lib/history';
-import React, { useEffect, useMemo } from 'react';
+import { ReactChild, useEffect, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
 import { matchPath, Redirect, useLocation } from 'react-router-dom';
@@ -47,6 +47,8 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
 
 	const dispatch = useDispatch<Dispatch<AppActions>>();
 
+	const { notifications } = useNotifications();
+
 	const currentRoute = mapRoutes.get('current');
 
 	const navigateToLoginIfNotLoggedIn = (isLoggedIn = isLoggedInState): void => {
@@ -106,7 +108,7 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
 							} else {
 								Logout();
 
-								notification.error({
+								notifications.error({
 									message: response.error || t('something_went_wrong'),
 								});
 							}
@@ -159,7 +161,7 @@ function PrivateRoute({ children }: PrivateRouteProps): JSX.Element {
 }
 
 interface PrivateRouteProps {
-	children: React.ReactChild;
+	children: ReactChild;
 }
 
 export default PrivateRoute;

frontend/src/AppRoutes/index.tsx

@@ -1,37 +1,50 @@
+import { ConfigProvider } from 'antd';
 import NotFound from 'components/NotFound';
 import Spinner from 'components/Spinner';
 import AppLayout from 'container/AppLayout';
+import { useThemeConfig } from 'hooks/useDarkMode';
+import { NotificationProvider } from 'hooks/useNotifications';
+import { ResourceProvider } from 'hooks/useResourceAttribute';
 import history from 'lib/history';
-import React, { Suspense } from 'react';
+import { QueryBuilderProvider } from 'providers/QueryBuilder';
+import { Suspense } from 'react';
 import { Route, Router, Switch } from 'react-router-dom';
 
 import PrivateRoute from './Private';
 import routes from './routes';
 
 function App(): JSX.Element {
-	return (
-		<Router history={history}>
-			<PrivateRoute>
-				<AppLayout>
-					<Suspense fallback={<Spinner size="large" tip="Loading..." />}>
-						<Switch>
-							{routes.map(({ path, component, exact }) => {
-								return (
-									<Route
-										key={`${path}`}
-										exact={exact}
-										path={path}
-										component={component}
-									/>
-								);
-							})}
+	const themeConfig = useThemeConfig();
 
-							<Route path="*" component={NotFound} />
-						</Switch>
-					</Suspense>
-				</AppLayout>
-			</PrivateRoute>
-		</Router>
+	return (
+		<ConfigProvider theme={themeConfig}>
+			<Router history={history}>
+				<NotificationProvider>
+					<PrivateRoute>
+						<ResourceProvider>
+							<QueryBuilderProvider>
+								<AppLayout>
+									<Suspense fallback={<Spinner size="large" tip="Loading..." />}>
+										<Switch>
+											{routes.map(({ path, component, exact }) => (
+												<Route
+													key={`${path}`}
+													exact={exact}
+													path={path}
+													component={component}
+												/>
+											))}
+
+											<Route path="*" component={NotFound} />
+										</Switch>
+									</Suspense>
+								</AppLayout>
+							</QueryBuilderProvider>
+						</ResourceProvider>
+					</PrivateRoute>
+				</NotificationProvider>
+			</Router>
+		</ConfigProvider>
 	);
 }
 

frontend/src/AppRoutes/utils.ts

@@ -57,6 +57,7 @@ const afterLogin = async (
 				profilePictureURL: payload.profilePictureURL,
 				userId: payload.id,
 				orgId: payload.orgId,
+				userFlags: payload.flags,
 			},
 		});
 

frontend/src/api/apiV1.ts

@@ -1,6 +1,7 @@
 const apiV1 = '/api/v1/';
 
 export const apiV2 = '/api/v2/';
+export const apiV3 = '/api/v3/';
 export const apiAlertManager = '/api/alertmanager';
 
 export default apiV1;

frontend/src/api/dashboard/variables/query.ts

@@ -1,4 +1,4 @@
-import axios from 'api';
+import { ApiV2Instance as axios } from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError } from 'axios';
 import { ErrorResponse, SuccessResponse } from 'types/api';
@@ -8,9 +8,7 @@ const query = async (
 	props: Props,
 ): Promise<SuccessResponse<PayloadProps> | ErrorResponse> => {
 	try {
-		const response = await axios.get(
-			`/variables/query?query=${encodeURIComponent(props.query)}`,
-		);
+		const response = await axios.post(`/variables/query`, props);
 
 		return {
 			statusCode: 200,

frontend/src/api/errors/getAll.ts

@@ -1,7 +1,6 @@
 import axios from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError } from 'axios';
-import createQueryParams from 'lib/createQueryParams';
 import { ErrorResponse, SuccessResponse } from 'types/api';
 import { PayloadProps, Props } from 'types/api/errors/getAll';
 
@@ -9,11 +8,17 @@ const getAll = async (
 	props: Props,
 ): Promise<SuccessResponse<PayloadProps> | ErrorResponse> => {
 	try {
-		const response = await axios.get(
-			`/listErrors?${createQueryParams({
-				...props,
-			})}`,
-		);
+		const response = await axios.post(`/listErrors`, {
+			start: `${props.start}`,
+			end: `${props.end}`,
+			order: props.order,
+			orderParam: props.orderParam,
+			limit: props.limit,
+			offset: props.offset,
+			exceptionType: props.exceptionType,
+			serviceName: props.serviceName,
+			tags: props.tags,
+		});
 
 		return {
 			statusCode: 200,

frontend/src/api/errors/getErrorCounts.ts

@@ -1,7 +1,6 @@
 import axios from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError } from 'axios';
-import createQueryParams from 'lib/createQueryParams';
 import { ErrorResponse, SuccessResponse } from 'types/api';
 import { PayloadProps, Props } from 'types/api/errors/getErrorCounts';
 
@@ -9,11 +8,13 @@ const getErrorCounts = async (
 	props: Props,
 ): Promise<SuccessResponse<PayloadProps> | ErrorResponse> => {
 	try {
-		const response = await axios.get(
-			`/countErrors?${createQueryParams({
-				...props,
-			})}`,
-		);
+		const response = await axios.post(`/countErrors`, {
+			start: `${props.start}`,
+			end: `${props.end}`,
+			exceptionType: props.exceptionType,
+			serviceName: props.serviceName,
+			tags: props.tags,
+		});
 
 		return {
 			statusCode: 200,

frontend/src/api/features/getFeatures.ts

@@ -1,23 +0,0 @@
-import axios from 'api';
-import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
-import { AxiosError } from 'axios';
-import { ErrorResponse, SuccessResponse } from 'types/api';
-import { PayloadProps } from 'types/api/features/getFeatures';
-
-const getFeaturesFlags = async (): Promise<
-	SuccessResponse<PayloadProps> | ErrorResponse
-> => {
-	try {
-		const response = await axios.get(`/featureFlags`);
-		return {
-			statusCode: 200,
-			error: null,
-			message: response.data.status,
-			payload: response.data.data,
-		};
-	} catch (error) {
-		return ErrorResponseHandler(error as AxiosError);
-	}
-};
-
-export default getFeaturesFlags;

frontend/src/api/index.ts

@@ -9,7 +9,7 @@ import { ENVIRONMENT } from 'constants/env';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import store from 'store';
 
-import apiV1, { apiAlertManager, apiV2 } from './apiV1';
+import apiV1, { apiAlertManager, apiV2, apiV3 } from './apiV1';
 import { Logout } from './utils';
 
 const interceptorsResponse = (
@@ -109,6 +109,17 @@ ApiV2Instance.interceptors.response.use(
 );
 ApiV2Instance.interceptors.request.use(interceptorsRequestResponse);
 
+// axios V3
+export const ApiV3Instance = axios.create({
+	baseURL: `${ENVIRONMENT.baseURL}${apiV3}`,
+});
+ApiV3Instance.interceptors.response.use(
+	interceptorsResponse,
+	interceptorRejected,
+);
+ApiV3Instance.interceptors.request.use(interceptorsRequestResponse);
+//
+
 AxiosAlertManagerInstance.interceptors.response.use(
 	interceptorsResponse,
 	interceptorRejected,

frontend/src/api/logs/livetail.ts

@@ -4,14 +4,16 @@ import { ENVIRONMENT } from 'constants/env';
 import { LOCALSTORAGE } from 'constants/localStorage';
 import { EventSourcePolyfill } from 'event-source-polyfill';
 
-export const LiveTail = (queryParams: string): EventSourcePolyfill => {
-	const dict = {
-		headers: {
-			Authorization: `Bearer ${getLocalStorageKey(LOCALSTORAGE.AUTH_TOKEN)}`,
-		},
-	};
-	return new EventSourcePolyfill(
+// 10 min in ms
+const TIMEOUT_IN_MS = 10 * 60 * 1000;
+
+export const LiveTail = (queryParams: string): EventSourcePolyfill =>
+	new EventSourcePolyfill(
 		`${ENVIRONMENT.baseURL}${apiV1}logs/tail?${queryParams}`,
-		dict,
+		{
+			headers: {
+				Authorization: `Bearer ${getLocalStorageKey(LOCALSTORAGE.AUTH_TOKEN)}`,
+			},
+			heartbeatTimeout: TIMEOUT_IN_MS,
+		},
 	);
-};

frontend/src/api/metrics/getQueryRange.ts

@@ -1,17 +1,17 @@
-import { ApiV2Instance as axios } from 'api';
+import { ApiV3Instance as axios } from 'api';
 import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
 import { AxiosError } from 'axios';
 import { ErrorResponse, SuccessResponse } from 'types/api';
 import {
-	MetricRangePayloadProps,
+	MetricRangePayloadV3,
 	MetricsRangeProps,
 } from 'types/api/metrics/getQueryRange';
 
 export const getMetricsQueryRange = async (
 	props: MetricsRangeProps,
-): Promise<SuccessResponse<MetricRangePayloadProps> | ErrorResponse> => {
+): Promise<SuccessResponse<MetricRangePayloadV3> | ErrorResponse> => {
 	try {
-		const response = await axios.post(`/metrics/query_range`, props);
+		const response = await axios.post('/query_range', props);
 
 		return {
 			statusCode: 200,

frontend/src/api/queryBuilder/getAggregateAttribute.ts

@@ -0,0 +1,46 @@
+import { ApiV3Instance } from 'api';
+import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
+import { AxiosError, AxiosResponse } from 'axios';
+import createQueryParams from 'lib/createQueryParams';
+// ** Helpers
+import { ErrorResponse, SuccessResponse } from 'types/api';
+// ** Types
+import { IGetAggregateAttributePayload } from 'types/api/queryBuilder/getAggregatorAttribute';
+import {
+	BaseAutocompleteData,
+	IQueryAutocompleteResponse,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
+import { v4 as uuid } from 'uuid';
+
+export const getAggregateAttribute = async ({
+	aggregateOperator,
+	searchText,
+	dataSource,
+}: IGetAggregateAttributePayload): Promise<
+	SuccessResponse<IQueryAutocompleteResponse> | ErrorResponse
+> => {
+	try {
+		const response: AxiosResponse<{
+			data: IQueryAutocompleteResponse;
+		}> = await ApiV3Instance.get(
+			`autocomplete/aggregate_attributes?${createQueryParams({
+				aggregateOperator,
+				searchText,
+				dataSource,
+			})}`,
+		);
+
+		const payload: BaseAutocompleteData[] =
+			response.data.data.attributeKeys?.map((item) => ({ ...item, id: uuid() })) ||
+			[];
+
+		return {
+			statusCode: 200,
+			error: null,
+			message: response.statusText,
+			payload: { attributeKeys: payload },
+		};
+	} catch (e) {
+		return ErrorResponseHandler(e as AxiosError);
+	}
+};

frontend/src/api/queryBuilder/getAttributeKeys.ts

@@ -0,0 +1,48 @@
+import { ApiV3Instance } from 'api';
+import { ErrorResponseHandler } from 'api/ErrorResponseHandler';
+import { AxiosError, AxiosResponse } from 'axios';
+import createQueryParams from 'lib/createQueryParams';
+import { ErrorResponse, SuccessResponse } from 'types/api';
+// ** Types
+import { IGetAttributeKeysPayload } from 'types/api/queryBuilder/getAttributeKeys';
+import {
+	BaseAutocompleteData,
+	IQueryAutocompleteResponse,
+} from 'types/api/queryBuilder/queryAutocompleteResponse';
+import { v4 as uuid } from 'uuid';
+
+export const getAggregateKeys = async ({
+	aggregateOperator,
+	searchText,
+	dataSource,
+	aggregateAttribute,
+	tagType,
+}: IGetAttributeKeysPayload): Promise<
+	SuccessResponse<IQueryAutocompleteResponse> | ErrorResponse
+> => {
+	try {
+		const response: AxiosResponse<{
+			data: IQueryAutocompleteResponse;
+		}> = await ApiV3Instance.get(
+			`autocomplete/attribute_keys?${createQueryParams({
+				aggregateOperator,
+				searchText,
+				dataSource,
+				aggregateAttribute,
+			})}&tagType=${tagType}`,
+		);
+
+		const payload: BaseAutocompleteData[] =
+			response.data.data.attributeKeys?.map((item) => ({ ...item, id: uuid() })) ||
+			[];
+
+		return {
+			statusCode: 200,
+			error: null,
+			message: response.statusText,
+			payload: { attributeKeys: payload },
+		};
+	} catch (e) {
+		return ErrorResponseHandler(e as AxiosError);
+	}
+};