Merge branch 'variable-update-queue' into dashboard-variable-test-cases

* feat: added API limiting to reduce unneccesary api call for dashboard variables

* feat: fixed dropdown open triggering the api calls for single-select and misc

.github/workflows/build.yaml

@@ -3,6 +3,7 @@ name: build-pipeline
 on:
   pull_request:
     branches:
+      - develop
       - main
       - release/v*
 

.github/workflows/docs.yml

@@ -3,7 +3,7 @@ name: "Update PR labels and Block PR until related docs are shipped for the feat
 on:
   pull_request:
     branches:
-      - main
+      - develop
     types: [opened, edited, labeled, unlabeled]
 
 permissions:

.github/workflows/e2e-k3s.yaml

@@ -42,7 +42,7 @@ jobs:
           kubectl create ns sample-application
 
           # apply hotrod k8s manifest file
-          kubectl -n sample-application apply -f https://raw.githubusercontent.com/SigNoz/signoz/main/sample-apps/hotrod/hotrod.yaml
+          kubectl -n sample-application apply -f https://raw.githubusercontent.com/SigNoz/signoz/develop/sample-apps/hotrod/hotrod.yaml
 
           # wait for all deployments in sample-application namespace to be READY
           kubectl -n sample-application get deploy --output name | xargs -r -n1 -t kubectl -n sample-application rollout status --timeout=300s

.github/workflows/jest-coverage-changes.yml

@@ -2,8 +2,7 @@ name: Jest Coverage - changed files
 
 on:
   pull_request:
-    branches:
-      - main
+    branches: develop
 
 jobs:
   build:
@@ -12,7 +11,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: "refs/heads/main"
+          ref: "refs/heads/develop"
           token: ${{ secrets.GITHUB_TOKEN }} # Provide the GitHub token for authentication
 
       - name: Fetch branch

.github/workflows/push.yaml

@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - develop
     tags:
       - v*
 

.github/workflows/sonar.yml

@@ -3,6 +3,7 @@ on:
   pull_request:
     branches:
       - main
+      - develop
     paths:
       - 'frontend/**'
 defaults:

.github/workflows/staging-deployment.yaml

@@ -1,12 +1,12 @@
 name: staging-deployment
-# Trigger deployment only on push to main branch
+# Trigger deployment only on push to develop branch
 on:
   push:
     branches:
-      - main
+      - develop
 jobs:
   deploy:
-    name: Deploy latest main branch to staging
+    name: Deploy latest develop branch to staging
     runs-on: ubuntu-latest
     environment: staging
     permissions:

.github/workflows/testing-deployment.yaml

@@ -44,7 +44,7 @@ jobs:
           git add .
           git stash push -m "stashed on $(date --iso-8601=seconds)"
           git fetch origin
-          git checkout main
+          git checkout develop
           git pull
           # This is added to include the scenerio when new commit in PR is force-pushed
           git branch -D ${GITHUB_BRANCH}

CONTRIBUTING.md

@@ -339,7 +339,7 @@ to make SigNoz UI available at [localhost:3301](http://localhost:3301)
 **5.1.1 To install the HotROD sample app:**
 
 ```bash
-curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-install.sh \
+curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-install.sh \
   | HELM_RELEASE=my-release SIGNOZ_NAMESPACE=platform bash
 ```
 
@@ -362,7 +362,7 @@ kubectl -n sample-application run strzal --image=djbingham/curl \
 **5.1.4 To delete the HotROD sample app:**
 
 ```bash
-curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-delete.sh \
+curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-delete.sh \
   | HOTROD_NAMESPACE=sample-application bash
 ```
 

deploy/README.md

@@ -58,7 +58,7 @@ from the HotROD application, you should see the data generated from hotrod in Si
 ```sh
 kubectl create ns sample-application
 
-kubectl -n sample-application apply -f https://raw.githubusercontent.com/SigNoz/signoz/main/sample-apps/hotrod/hotrod.yaml
+kubectl -n sample-application apply -f https://raw.githubusercontent.com/SigNoz/signoz/develop/sample-apps/hotrod/hotrod.yaml
 ```
 
 To generate load:

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -146,12 +146,11 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.64.0
+    image: signoz/query-service:0.61.0
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true",
-        "--use-trace-new-schema=true"
+        "--use-logs-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -187,7 +186,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.64.0
+    image: signoz/frontend:0.61.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -200,7 +199,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.111.16
+    image: signoz/signoz-otel-collector:0.111.14
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -238,7 +237,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.111.16
+      image: signoz/signoz-schema-migrator:0.111.14
       deploy:
         restart_policy:
           condition: on-failure

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -110,7 +110,6 @@ exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/signoz_traces
     low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
-    use_new_schema: true
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -69,7 +69,7 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
     container_name: otel-migrator
     command:
       - "sync" 
@@ -86,7 +86,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.111.16
+    image: signoz/signoz-otel-collector:0.111.14
     command:
       [
         "--config=/etc/otel-collector-config.yaml",

deploy/docker/clickhouse-setup/docker-compose-local.yaml

@@ -25,8 +25,7 @@ services:
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true",
-        "--use-trace-new-schema=true"
+        "--use-logs-new-schema=true"
       ]
     ports:
       - "6060:6060"

deploy/docker/clickhouse-setup/docker-compose-minimal.yaml

@@ -162,13 +162,12 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
     container_name: signoz-query-service
     command:
       [
         "-config=/root/config/prometheus.yml",
-        "--use-logs-new-schema=true",
-        "--use-trace-new-schema=true"
+        "--use-logs-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -202,7 +201,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -214,7 +213,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator-sync:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
     container_name: otel-migrator-sync
     command:
       - "sync"
@@ -229,7 +228,7 @@ services:
       #   condition: service_healthy
 
   otel-collector-migrator-async:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
     container_name: otel-migrator-async
     command:
       - "async"
@@ -246,7 +245,7 @@ services:
       #   condition: service_healthy
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/docker-compose.testing.yaml

@@ -167,14 +167,13 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.64.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
     container_name: signoz-query-service
     command:
       [
         "-config=/root/config/prometheus.yml",
         "-gateway-url=https://api.staging.signoz.cloud",
-        "--use-logs-new-schema=true",
-        "--use-trace-new-schema=true"
+        "--use-logs-new-schema=true"
       ]
     # ports:
     #   - "6060:6060"     # pprof port
@@ -209,7 +208,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.64.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -221,7 +220,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -235,7 +234,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -119,7 +119,6 @@ exporters:
   clickhousetraces:
     datasource: tcp://clickhouse:9000/signoz_traces
     low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
-    use_new_schema: true
   clickhousemetricswrite:
     endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:

frontend/.husky/commit-msg

@@ -13,3 +13,8 @@ if [ "$branch" = "main" ]; then
   echo "${color_red}${bold}You can't commit directly to the main branch${reset}"
   exit 1
 fi
+
+if [ "$branch" = "develop" ]; then
+  echo "${color_red}${bold}You can't commit directly to the develop branch${reset}"
+  exit 1
+fi

frontend/src/container/NewDashboard/DashboardVariablesSelection/DashboardVariableSelection.tsx

@@ -1,9 +1,15 @@
 import { Row } from 'antd';
-import { isNull } from 'lodash-es';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
-import { memo, useEffect, useState } from 'react';
+import { memo, useEffect, useRef, useState } from 'react';
 import { IDashboardVariable } from 'types/api/dashboard/getAll';
 
+import {
+	buildDependencies,
+	buildDependencyGraph,
+	buildParentDependencyGraph,
+	onUpdateVariableNode,
+	VariableGraph,
+} from './util';
 import VariableItem from './VariableItem';
 
 function DashboardVariableSelection(): JSX.Element | null {
@@ -21,6 +27,12 @@ function DashboardVariableSelection(): JSX.Element | null {
 
 	const [variablesTableData, setVariablesTableData] = useState<any>([]);
 
+	const [dependencyData, setDependencyData] = useState<{
+		order: string[];
+		graph: VariableGraph;
+		parentDependencyGraph: VariableGraph;
+	} | null>(null);
+
 	useEffect(() => {
 		if (variables) {
 			const tableRowData = [];
@@ -43,35 +55,28 @@ function DashboardVariableSelection(): JSX.Element | null {
 		}
 	}, [variables]);
 
-	const onVarChanged = (name: string): void => {
-		/**
-		 * this function takes care of adding the dependent variables to current update queue and removing
-		 * the updated variable name from the queue
-		 */
-		const dependentVariables = variablesTableData
-			?.map((variable: any) => {
-				if (variable.type === 'QUERY') {
-					const re = new RegExp(`\\{\\{\\s*?\\.${name}\\s*?\\}\\}`); // regex for `{{.var}}`
-					const queryValue = variable.queryValue || '';
-					const dependVarReMatch = queryValue.match(re);
-					if (dependVarReMatch !== null && dependVarReMatch.length > 0) {
-						return variable.name;
-					}
-				}
-				return null;
-			})
-			.filter((val: string | null) => !isNull(val));
-		setVariablesToGetUpdated((prev) => [
-			...prev.filter((v) => v !== name),
-			...dependentVariables,
-		]);
-	};
+	const initializationRef = useRef(false);
+
+	useEffect(() => {
+		if (variablesTableData.length > 0 && !initializationRef.current) {
+			const depGrp = buildDependencies(variablesTableData);
+			const { order, graph } = buildDependencyGraph(depGrp);
+			const parentDependencyGraph = buildParentDependencyGraph(graph);
+			setDependencyData({
+				order,
+				graph,
+				parentDependencyGraph,
+			});
+			initializationRef.current = true;
+		}
+	}, [variablesTableData]);
 
 	const onValueUpdate = (
 		name: string,
 		id: string,
 		value: IDashboardVariable['selectedValue'],
 		allSelected: boolean,
+		isMountedCall?: boolean,
 		// eslint-disable-next-line sonarjs/cognitive-complexity
 	): void => {
 		if (id) {
@@ -111,7 +116,18 @@ function DashboardVariableSelection(): JSX.Element | null {
 				});
 			}
 
-			onVarChanged(name);
+			if (dependencyData && !isMountedCall) {
+				const updatedVariables: string[] = [];
+				onUpdateVariableNode(
+					name,
+					dependencyData.graph,
+					dependencyData.order,
+					(node) => updatedVariables.push(node),
+				);
+				setVariablesToGetUpdated(updatedVariables.filter((v) => v !== name));
+			} else if (isMountedCall) {
+				setVariablesToGetUpdated((prev) => prev.filter((v) => v !== name));
+			}
 		}
 	};
 
@@ -139,6 +155,7 @@ function DashboardVariableSelection(): JSX.Element | null {
 						onValueUpdate={onValueUpdate}
 						variablesToGetUpdated={variablesToGetUpdated}
 						setVariablesToGetUpdated={setVariablesToGetUpdated}
+						dependencyData={dependencyData}
 					/>
 				))}
 		</Row>

frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx

@@ -24,7 +24,7 @@ import { commaValuesParser } from 'lib/dashbaordVariables/customCommaValuesParse
 import sortValues from 'lib/dashbaordVariables/sortVariableValues';
 import { debounce, isArray, isString } from 'lodash-es';
 import map from 'lodash-es/map';
-import { ChangeEvent, memo, useEffect, useMemo, useState } from 'react';
+import { ChangeEvent, memo, useEffect, useMemo, useRef, useState } from 'react';
 import { useQuery } from 'react-query';
 import { useSelector } from 'react-redux';
 import { AppState } from 'store/reducers';
@@ -35,12 +35,15 @@ import { popupContainer } from 'utils/selectPopupContainer';
 
 import { variablePropsToPayloadVariables } from '../utils';
 import { SelectItemStyle } from './styles';
-import { areArraysEqual } from './util';
+import {
+	areArraysEqual,
+	checkAPIInvocation,
+	onUpdateVariableNode,
+	VariableGraph,
+} from './util';
 
 const ALL_SELECT_VALUE = '__ALL__';
 
-const variableRegexPattern = /\{\{\s*?\.([^\s}]+)\s*?\}\}/g;
-
 enum ToggleTagValue {
 	Only = 'Only',
 	All = 'All',
@@ -54,9 +57,15 @@ interface VariableItemProps {
 		id: string,
 		arg1: IDashboardVariable['selectedValue'],
 		allSelected: boolean,
+		isMountedCall?: boolean,
 	) => void;
 	variablesToGetUpdated: string[];
 	setVariablesToGetUpdated: React.Dispatch<React.SetStateAction<string[]>>;
+	dependencyData: {
+		order: string[];
+		graph: VariableGraph;
+		parentDependencyGraph: VariableGraph;
+	} | null;
 }
 
 const getSelectValue = (
@@ -79,6 +88,7 @@ function VariableItem({
 	onValueUpdate,
 	variablesToGetUpdated,
 	setVariablesToGetUpdated,
+	dependencyData,
 }: VariableItemProps): JSX.Element {
 	const [optionsData, setOptionsData] = useState<(string | number | boolean)[]>(
 		[],
@@ -88,60 +98,29 @@ function VariableItem({
 		(state) => state.globalTime,
 	);
 
+	// logic to detect if its a rerender or a new render/mount
+	const isMounted = useRef(false);
+
 	useEffect(() => {
-		if (variableData.allSelected && variableData.type === 'QUERY') {
-			setVariablesToGetUpdated((prev) => {
-				const variablesQueue = [...prev.filter((v) => v !== variableData.name)];
-				if (variableData.name) {
-					variablesQueue.push(variableData.name);
-				}
-				return variablesQueue;
-			});
+		isMounted.current = true;
+	}, []);
+
+	const validVariableUpdate = (): boolean => {
+		if (!variableData.name) {
+			return false;
 		}
-		// eslint-disable-next-line react-hooks/exhaustive-deps
-	}, [minTime, maxTime]);
+		if (!isMounted.current) {
+			// variableData.name is present as the top element or next in the queue - variablesToGetUpdated
+			return Boolean(
+				variablesToGetUpdated.length &&
+					variablesToGetUpdated[0] === variableData.name,
+			);
+		}
+		return variablesToGetUpdated.includes(variableData.name);
+	};
 
 	const [errorMessage, setErrorMessage] = useState<null | string>(null);
 
-	const getDependentVariables = (queryValue: string): string[] => {
-		const matches = queryValue.match(variableRegexPattern);
-
-		// Extract variable names from the matches array without {{ . }}
-		return matches
-			? matches.map((match) => match.replace(variableRegexPattern, '$1'))
-			: [];
-	};
-
-	const getQueryKey = (variableData: IDashboardVariable): string[] => {
-		let dependentVariablesStr = '';
-
-		const dependentVariables = getDependentVariables(
-			variableData.queryValue || '',
-		);
-
-		const variableName = variableData.name || '';
-
-		dependentVariables?.forEach((element) => {
-			const [, variable] =
-				Object.entries(existingVariables).find(
-					([, value]) => value.name === element,
-				) || [];
-
-			dependentVariablesStr += `${element}${variable?.selectedValue}`;
-		});
-
-		const variableKey = dependentVariablesStr.replace(/\s/g, '');
-
-		// added this time dependency for variables query as API respects the passed time range now
-		return [
-			REACT_QUERY_KEY.DASHBOARD_BY_ID,
-			variableName,
-			variableKey,
-			`${minTime}`,
-			`${maxTime}`,
-		];
-	};
-
 	// eslint-disable-next-line sonarjs/cognitive-complexity
 	const getOptions = (variablesRes: VariableResponseProps | null): void => {
 		if (variablesRes && variableData.type === 'QUERY') {
@@ -184,9 +163,7 @@ function VariableItem({
 						if (
 							variableData.type === 'QUERY' &&
 							variableData.name &&
-							(variablesToGetUpdated.includes(variableData.name) ||
-								valueNotInList ||
-								variableData.allSelected)
+							(validVariableUpdate() || valueNotInList || variableData.allSelected)
 						) {
 							let value = variableData.selectedValue;
 							let allSelected = false;
@@ -200,7 +177,16 @@ function VariableItem({
 							}
 
 							if (variableData && variableData?.name && variableData?.id) {
-								onValueUpdate(variableData.name, variableData.id, value, allSelected);
+								onValueUpdate(
+									variableData.name,
+									variableData.id,
+									value,
+									allSelected,
+									isMounted.current,
+								);
+								setVariablesToGetUpdated((prev) =>
+									prev.filter((name) => name !== variableData.name),
+								);
 							}
 						}
 
@@ -224,36 +210,75 @@ function VariableItem({
 		}
 	};
 
-	const { isLoading } = useQuery(getQueryKey(variableData), {
-		enabled: variableData && variableData.type === 'QUERY',
-		queryFn: () =>
-			dashboardVariablesQuery({
-				query: variableData.queryValue || '',
-				variables: variablePropsToPayloadVariables(existingVariables),
-			}),
-		refetchOnWindowFocus: false,
-		onSuccess: (response) => {
-			getOptions(response.payload);
-		},
-		onError: (error: {
-			details: {
-				error: string;
-			};
-		}) => {
-			const { details } = error;
-
-			if (details.error) {
-				let message = details.error;
-				if (details.error.includes('Syntax error:')) {
-					message =
-						'Please make sure query is valid and dependent variables are selected';
+	const { isLoading } = useQuery(
+		[
+			REACT_QUERY_KEY.DASHBOARD_BY_ID,
+			variableData.name || '',
+			`${minTime}`,
+			`${maxTime}`,
+		],
+		{
+			enabled:
+				variableData &&
+				variableData.type === 'QUERY' &&
+				checkAPIInvocation(
+					variablesToGetUpdated,
+					variableData,
+					dependencyData?.parentDependencyGraph,
+				),
+			queryFn: () =>
+				dashboardVariablesQuery({
+					query: variableData.queryValue || '',
+					variables: variablePropsToPayloadVariables(existingVariables),
+				}),
+			refetchOnWindowFocus: false,
+			onSuccess: (response) => {
+				getOptions(response.payload);
+				if (
+					dependencyData?.parentDependencyGraph[variableData.name || ''].length === 0
+				) {
+					const updatedVariables: string[] = [];
+					onUpdateVariableNode(
+						variableData.name || '',
+						dependencyData.graph,
+						dependencyData.order,
+						(node) => updatedVariables.push(node),
+					);
+					setVariablesToGetUpdated((prev) => [
+						...prev,
+						...updatedVariables.filter((v) => v !== variableData.name),
+					]);
 				}
-				setErrorMessage(message);
-			}
+			},
+			onError: (error: {
+				details: {
+					error: string;
+				};
+			}) => {
+				const { details } = error;
+
+				if (details.error) {
+					let message = details.error;
+					if (details.error.includes('Syntax error:')) {
+						message =
+							'Please make sure query is valid and dependent variables are selected';
+					}
+					setErrorMessage(message);
+				}
+			},
 		},
-	});
+	);
 
 	const handleChange = (value: string | string[]): void => {
+		// if value is equal to selected value then return
+		if (
+			value === variableData.selectedValue ||
+			(Array.isArray(value) &&
+				Array.isArray(variableData.selectedValue) &&
+				areArraysEqual(value, variableData.selectedValue))
+		) {
+			return;
+		}
 		if (variableData.name) {
 			if (
 				value === ALL_SELECT_VALUE ||

frontend/src/container/NewDashboard/DashboardVariablesSelection/__test__/dashboardVariables.test.tsx

@@ -0,0 +1,241 @@
+import {
+	buildDependencies,
+	buildDependencyGraph,
+	buildParentDependencyGraph,
+	checkAPIInvocation,
+	onUpdateVariableNode,
+	VariableGraph,
+} from '../util';
+import {
+	buildDependenciesMock,
+	buildGraphMock,
+	checkAPIInvocationMock,
+	onUpdateVariableNodeMock,
+} from './mock';
+
+describe('dashboardVariables - utilities and processors', () => {
+	describe('onUpdateVariableNode', () => {
+		const { graph, topologicalOrder } = onUpdateVariableNodeMock;
+		const testCases = [
+			{
+				scenario: 'root element',
+				nodeToUpdate: 'deployment_environment',
+				expected: [
+					'deployment_environment',
+					'service_name',
+					'endpoint',
+					'http_status_code',
+				],
+			},
+			{
+				scenario: 'middle child',
+				nodeToUpdate: 'k8s_node_name',
+				expected: ['k8s_node_name', 'k8s_namespace_name'],
+			},
+			{
+				scenario: 'leaf element',
+				nodeToUpdate: 'http_status_code',
+				expected: ['http_status_code'],
+			},
+			{
+				scenario: 'node not in graph',
+				nodeToUpdate: 'unknown',
+				expected: [],
+			},
+			{
+				scenario: 'node not in topological order',
+				nodeToUpdate: 'unknown',
+				expected: [],
+			},
+		];
+
+		test.each(testCases)(
+			'should update variable node when $scenario',
+			({ nodeToUpdate, expected }) => {
+				const updatedVariables: string[] = [];
+				const callback = (node: string): void => {
+					updatedVariables.push(node);
+				};
+
+				onUpdateVariableNode(nodeToUpdate, graph, topologicalOrder, callback);
+
+				expect(updatedVariables).toEqual(expected);
+			},
+		);
+
+		it('should return empty array when topological order is empty', () => {
+			const updatedVariables: string[] = [];
+			onUpdateVariableNode('http_status_code', graph, [], (node) =>
+				updatedVariables.push(node),
+			);
+			expect(updatedVariables).toEqual([]);
+		});
+	});
+
+	describe('checkAPIInvocation', () => {
+		const {
+			variablesToGetUpdated,
+			variableData,
+			parentDependencyGraph,
+		} = checkAPIInvocationMock;
+
+		const mockRootElement = {
+			name: 'deployment_environment',
+			key: '036a47cd-9ffc-47de-9f27-0329198964a8',
+			id: '036a47cd-9ffc-47de-9f27-0329198964a8',
+			modificationUUID: '5f71b591-f583-497c-839d-6a1590c3f60f',
+			selectedValue: 'production',
+			type: 'QUERY',
+			// ... other properties omitted for brevity
+		} as any;
+
+		describe('edge cases', () => {
+			it('should return false when variableData is empty', () => {
+				expect(
+					checkAPIInvocation(
+						variablesToGetUpdated,
+						variableData,
+						parentDependencyGraph,
+					),
+				).toBeFalsy();
+			});
+
+			it('should return true when parentDependencyGraph is empty', () => {
+				expect(
+					checkAPIInvocation(variablesToGetUpdated, variableData, {}),
+				).toBeTruthy();
+			});
+		});
+
+		describe('variable sequences', () => {
+			it('should return true for valid sequence', () => {
+				expect(
+					checkAPIInvocation(
+						['k8s_node_name', 'k8s_namespace_name'],
+						variableData,
+						parentDependencyGraph,
+					),
+				).toBeTruthy();
+			});
+
+			it('should return false for invalid sequence', () => {
+				expect(
+					checkAPIInvocation(
+						['k8s_cluster_name', 'k8s_node_name', 'k8s_namespace_name'],
+						variableData,
+						parentDependencyGraph,
+					),
+				).toBeFalsy();
+			});
+
+			it('should return false when variableData is not in sequence', () => {
+				expect(
+					checkAPIInvocation(
+						['deployment_environment', 'service_name', 'endpoint'],
+						variableData,
+						parentDependencyGraph,
+					),
+				).toBeFalsy();
+			});
+		});
+
+		describe('root element behavior', () => {
+			it('should return true for valid root element sequence', () => {
+				expect(
+					checkAPIInvocation(
+						[
+							'deployment_environment',
+							'service_name',
+							'endpoint',
+							'http_status_code',
+						],
+						mockRootElement,
+						parentDependencyGraph,
+					),
+				).toBeTruthy();
+			});
+
+			it('should return true for empty variablesToGetUpdated array', () => {
+				expect(
+					checkAPIInvocation([], mockRootElement, parentDependencyGraph),
+				).toBeTruthy();
+			});
+		});
+	});
+
+	describe('Graph Building Utilities', () => {
+		const { graph } = buildGraphMock;
+		const { variables } = buildDependenciesMock;
+
+		describe('buildParentDependencyGraph', () => {
+			it('should build parent dependency graph with correct relationships', () => {
+				const expected = {
+					deployment_environment: [],
+					service_name: ['deployment_environment'],
+					endpoint: ['deployment_environment', 'service_name'],
+					http_status_code: ['endpoint'],
+					k8s_cluster_name: [],
+					k8s_node_name: ['k8s_cluster_name'],
+					k8s_namespace_name: ['k8s_cluster_name', 'k8s_node_name'],
+					environment: [],
+				};
+
+				expect(buildParentDependencyGraph(graph)).toEqual(expected);
+			});
+
+			it('should handle empty graph', () => {
+				expect(buildParentDependencyGraph({})).toEqual({});
+			});
+		});
+
+		describe('buildDependencyGraph', () => {
+			it('should build complete dependency graph with correct structure and order', () => {
+				const expected = {
+					graph: {
+						deployment_environment: ['service_name', 'endpoint'],
+						service_name: ['endpoint'],
+						endpoint: ['http_status_code'],
+						http_status_code: [],
+						k8s_cluster_name: ['k8s_node_name', 'k8s_namespace_name'],
+						k8s_node_name: ['k8s_namespace_name'],
+						k8s_namespace_name: [],
+						environment: [],
+					},
+					order: [
+						'deployment_environment',
+						'k8s_cluster_name',
+						'environment',
+						'service_name',
+						'k8s_node_name',
+						'endpoint',
+						'k8s_namespace_name',
+						'http_status_code',
+					],
+				};
+
+				expect(buildDependencyGraph(graph)).toEqual(expected);
+			});
+		});
+
+		describe('buildDependencies', () => {
+			it('should build dependency map from variables array', () => {
+				const expected: VariableGraph = {
+					deployment_environment: ['service_name', 'endpoint'],
+					service_name: ['endpoint'],
+					endpoint: ['http_status_code'],
+					http_status_code: [],
+					k8s_cluster_name: ['k8s_node_name', 'k8s_namespace_name'],
+					k8s_node_name: ['k8s_namespace_name'],
+					k8s_namespace_name: [],
+					environment: [],
+				};
+
+				expect(buildDependencies(variables)).toEqual(expected);
+			});
+
+			it('should handle empty variables array', () => {
+				expect(buildDependencies([])).toEqual({});
+			});
+		});
+	});
+});

frontend/src/container/NewDashboard/DashboardVariablesSelection/__test__/mock.ts

@@ -0,0 +1,251 @@
+/* eslint-disable sonarjs/no-duplicate-string */
+export const checkAPIInvocationMock = {
+	variablesToGetUpdated: [],
+	variableData: {
+		name: 'k8s_node_name',
+		key: '4d71d385-beaf-4434-8dbf-c62be68049fc',
+		allSelected: false,
+		customValue: '',
+		description: '',
+		id: '4d71d385-beaf-4434-8dbf-c62be68049fc',
+		modificationUUID: '77233d3c-96d7-4ccb-aa9d-11b04d563068',
+		multiSelect: false,
+		order: 6,
+		queryValue:
+			"SELECT JSONExtractString(labels, 'k8s_node_name') AS k8s_node_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'k8s_node_cpu_time' AND JSONExtractString(labels, 'k8s_cluster_name') = {{.k8s_cluster_name}}\nGROUP BY k8s_node_name",
+		selectedValue: 'gke-signoz-saas-si-consumer-bsc-e2sd4-a6d430fa-gvm2',
+		showALLOption: false,
+		sort: 'DISABLED',
+		textboxValue: '',
+		type: 'QUERY',
+	},
+	parentDependencyGraph: {
+		deployment_environment: [],
+		service_name: ['deployment_environment'],
+		endpoint: ['deployment_environment', 'service_name'],
+		http_status_code: ['endpoint'],
+		k8s_cluster_name: [],
+		environment: [],
+		k8s_node_name: ['k8s_cluster_name'],
+		k8s_namespace_name: ['k8s_cluster_name', 'k8s_node_name'],
+	},
+} as any;
+
+export const onUpdateVariableNodeMock = {
+	nodeToUpdate: 'deployment_environment',
+	graph: {
+		deployment_environment: ['service_name', 'endpoint'],
+		service_name: ['endpoint'],
+		endpoint: ['http_status_code'],
+		http_status_code: [],
+		k8s_cluster_name: ['k8s_node_name', 'k8s_namespace_name'],
+		environment: [],
+		k8s_node_name: ['k8s_namespace_name'],
+		k8s_namespace_name: [],
+	},
+	topologicalOrder: [
+		'deployment_environment',
+		'k8s_cluster_name',
+		'environment',
+		'service_name',
+		'k8s_node_name',
+		'endpoint',
+		'k8s_namespace_name',
+		'http_status_code',
+	],
+	callback: jest.fn(),
+};
+
+export const buildGraphMock = {
+	graph: {
+		deployment_environment: ['service_name', 'endpoint'],
+		service_name: ['endpoint'],
+		endpoint: ['http_status_code'],
+		http_status_code: [],
+		k8s_cluster_name: ['k8s_node_name', 'k8s_namespace_name'],
+		environment: [],
+		k8s_node_name: ['k8s_namespace_name'],
+		k8s_namespace_name: [],
+	},
+};
+
+export const buildDependenciesMock = {
+	variables: [
+		{
+			key: '036a47cd-9ffc-47de-9f27-0329198964a8',
+			name: 'deployment_environment',
+			allSelected: false,
+			customValue: '',
+			description: '',
+			id: '036a47cd-9ffc-47de-9f27-0329198964a8',
+			modificationUUID: '5f71b591-f583-497c-839d-6a1590c3f60f',
+			multiSelect: false,
+			order: 0,
+			queryValue:
+				"SELECT DISTINCT JSONExtractString(labels, 'deployment_environment') AS deployment_environment\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'signoz_calls_total'",
+			selectedValue: 'production',
+			showALLOption: false,
+			sort: 'DISABLED',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: 'eed5c917-1860-4c7e-bf6d-a05b97bafbc9',
+			name: 'service_name',
+			allSelected: true,
+			customValue: '',
+			description: '',
+			id: 'eed5c917-1860-4c7e-bf6d-a05b97bafbc9',
+			modificationUUID: '85db928b-ac9b-4e9f-b274-791112102fdf',
+			multiSelect: true,
+			order: 1,
+			queryValue:
+				"SELECT DISTINCT JSONExtractString(labels, 'service_name') FROM signoz_metrics.distributed_time_series_v4_1day\n WHERE metric_name = 'signoz_calls_total' and JSONExtractString(labels, 'deployment_environment') = {{.deployment_environment}}",
+			selectedValue: ['otelgateway'],
+			showALLOption: true,
+			sort: 'ASC',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: '4022d3c1-e845-4952-8984-78f25f575c7a',
+			name: 'endpoint',
+			allSelected: true,
+			customValue: '',
+			description: '',
+			id: '4022d3c1-e845-4952-8984-78f25f575c7a',
+			modificationUUID: 'c0107fa1-ebb7-4dd3-aa9d-6ba08ecc594d',
+			multiSelect: true,
+			order: 2,
+			queryValue:
+				"SELECT DISTINCT JSONExtractString(labels, 'operation') FROM signoz_metrics.distributed_time_series_v4_1day\n WHERE metric_name = 'signoz_calls_total' AND JSONExtractString(labels, 'service_name') IN {{.service_name}} and JSONExtractString(labels, 'deployment_environment') = {{.deployment_environment}}",
+			selectedValue: [
+				'//v1/traces',
+				'/logs/heroku',
+				'/logs/json',
+				'/logs/vector',
+				'/v1/logs',
+				'/v1/metrics',
+				'/v1/traces',
+				'SELECT',
+				'exporter/signozkafka/logs',
+				'exporter/signozkafka/metrics',
+				'exporter/signozkafka/traces',
+				'extension/signozkeyauth/Authenticate',
+				'get',
+				'hmget',
+				'opentelemetry.proto.collector.logs.v1.LogsService/Export',
+				'opentelemetry.proto.collector.metrics.v1.MetricsService/Export',
+				'opentelemetry.proto.collector.trace.v1.TraceService/Export',
+				'processor/signozlimiter/LogsProcessed',
+				'processor/signozlimiter/MetricsProcessed',
+				'processor/signozlimiter/TracesProcessed',
+				'receiver/otlp/LogsReceived',
+				'receiver/otlp/MetricsReceived',
+				'receiver/otlp/TraceDataReceived',
+				'receiver/signozhttplog/heroku/LogsReceived',
+				'receiver/signozhttplog/json/LogsReceived',
+				'receiver/signozhttplog/vector/LogsReceived',
+				'redis.dial',
+				'redis.pipeline eval',
+				'sadd',
+				'set',
+				'sismember',
+			],
+			showALLOption: true,
+			sort: 'ASC',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: '5e8a3cd9-3cd9-42df-a76c-79471a0f75bd',
+			name: 'http_status_code',
+			customValue: '',
+			description: '',
+			id: '5e8a3cd9-3cd9-42df-a76c-79471a0f75bd',
+			modificationUUID: '9a4021cc-a80a-4f15-8899-78892b763ca7',
+			multiSelect: true,
+			order: 3,
+			queryValue:
+				"SELECT DISTINCT JSONExtractString(labels, 'http_status_code') FROM signoz_metrics.distributed_time_series_v4_1day\n WHERE metric_name = 'signoz_calls_total' AND JSONExtractString(labels, 'operation') IN {{.endpoint}}",
+			showALLOption: true,
+			sort: 'ASC',
+			textboxValue: '',
+			type: 'QUERY',
+			selectedValue: ['', '200', '301', '400', '401', '405', '415', '429'],
+			allSelected: true,
+		},
+		{
+			key: '48e9aa64-05ca-41c2-a1bd-6c8aeca659f1',
+			name: 'k8s_cluster_name',
+			allSelected: false,
+			customValue: 'test-1,\ntest-2,\ntest-3',
+			description: '',
+			id: '48e9aa64-05ca-41c2-a1bd-6c8aeca659f1',
+			modificationUUID: '44722322-368c-4613-bb7f-d0b12867d57a',
+			multiSelect: false,
+			order: 4,
+			queryValue:
+				"SELECT JSONExtractString(labels, 'k8s_cluster_name') AS k8s_cluster_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'k8s_node_cpu_time'\nGROUP BY k8s_cluster_name",
+			selectedValue: 'saasmonitor-cluster',
+			showALLOption: false,
+			sort: 'DISABLED',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: '3ea18ba2-30cf-4220-b03b-720b5eaf35f8',
+			name: 'environment',
+			allSelected: false,
+			customValue: '',
+			description: '',
+			id: '3ea18ba2-30cf-4220-b03b-720b5eaf35f8',
+			modificationUUID: '9f76cb06-1b9f-460f-a174-0b210bb3cf93',
+			multiSelect: false,
+			order: 5,
+			queryValue:
+				"SELECT DISTINCT JSONExtractString(labels, 'deployment_environment') AS environment\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'signoz_calls_total'",
+			selectedValue: 'production',
+			showALLOption: false,
+			sort: 'DISABLED',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: '4d71d385-beaf-4434-8dbf-c62be68049fc',
+			name: 'k8s_node_name',
+			allSelected: false,
+			customValue: '',
+			description: '',
+			id: '4d71d385-beaf-4434-8dbf-c62be68049fc',
+			modificationUUID: '77233d3c-96d7-4ccb-aa9d-11b04d563068',
+			multiSelect: false,
+			order: 6,
+			queryValue:
+				"SELECT JSONExtractString(labels, 'k8s_node_name') AS k8s_node_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'k8s_node_cpu_time' AND JSONExtractString(labels, 'k8s_cluster_name') = {{.k8s_cluster_name}}\nGROUP BY k8s_node_name",
+			selectedValue: 'gke-signoz-saas-si-consumer-bsc-e2sd4-a6d430fa-gvm2',
+			showALLOption: false,
+			sort: 'DISABLED',
+			textboxValue: '',
+			type: 'QUERY',
+		},
+		{
+			key: '937ecbae-b24b-4d6d-8cc4-5d5b8d53569b',
+			name: 'k8s_namespace_name',
+			customValue: '',
+			description: '',
+			id: '937ecbae-b24b-4d6d-8cc4-5d5b8d53569b',
+			modificationUUID: '8ad2442d-8b4d-4c64-848e-af847d1d0eec',
+			multiSelect: false,
+			order: 7,
+			queryValue:
+				"SELECT JSONExtractString(labels, 'k8s_namespace_name') AS k8s_namespace_name\nFROM signoz_metrics.distributed_time_series_v4_1day\nWHERE metric_name = 'k8s_pod_cpu_time' AND JSONExtractString(labels, 'k8s_cluster_name') = {{.k8s_cluster_name}} AND JSONExtractString(labels, 'k8s_node_name') IN {{.k8s_node_name}}\nGROUP BY k8s_namespace_name",
+			showALLOption: false,
+			sort: 'DISABLED',
+			textboxValue: '',
+			type: 'QUERY',
+			selectedValue: 'saasmonitor',
+			allSelected: false,
+		},
+	] as any,
+};

frontend/src/container/NewDashboard/DashboardVariablesSelection/util.ts

@@ -1,3 +1,4 @@
+import { isEmpty } from 'lodash-es';
 import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
 
 export function areArraysEqual(
@@ -29,3 +30,159 @@ export const convertVariablesToDbFormat = (
 		result[id] = obj;
 		return result;
 	}, {});
+
+const getDependentVariables = (queryValue: string): string[] => {
+	const variableRegexPattern = /\{\{\s*?\.([^\s}]+)\s*?\}\}/g;
+
+	const matches = queryValue.match(variableRegexPattern);
+
+	// Extract variable names from the matches array without {{ . }}
+	return matches
+		? matches.map((match) => match.replace(variableRegexPattern, '$1'))
+		: [];
+};
+export type VariableGraph = Record<string, string[]>;
+
+export const buildDependencies = (
+	variables: IDashboardVariable[],
+): VariableGraph => {
+	console.log('buildDependencies', variables);
+	const graph: VariableGraph = {};
+
+	// Initialize empty arrays for all variables first
+	variables.forEach((variable) => {
+		if (variable.name) {
+			graph[variable.name] = [];
+		}
+	});
+
+	// For each QUERY variable, add it as a dependent to its referenced variables
+	variables.forEach((variable) => {
+		if (variable.type === 'QUERY' && variable.name) {
+			const dependentVariables = getDependentVariables(variable.queryValue || '');
+
+			// For each referenced variable, add the current query as a dependent
+			dependentVariables.forEach((referencedVar) => {
+				if (graph[referencedVar]) {
+					graph[referencedVar].push(variable.name as string);
+				} else {
+					graph[referencedVar] = [variable.name as string];
+				}
+			});
+		}
+	});
+
+	return graph;
+};
+
+// Function to build the dependency graph
+export const buildDependencyGraph = (
+	dependencies: VariableGraph,
+): { order: string[]; graph: VariableGraph } => {
+	const inDegree: Record<string, number> = {};
+	const adjList: VariableGraph = {};
+
+	// Initialize in-degree and adjacency list
+	Object.keys(dependencies).forEach((node) => {
+		if (!inDegree[node]) inDegree[node] = 0;
+		if (!adjList[node]) adjList[node] = [];
+		dependencies[node].forEach((child) => {
+			if (!inDegree[child]) inDegree[child] = 0;
+			inDegree[child]++;
+			adjList[node].push(child);
+		});
+	});
+
+	// Topological sort using Kahn's Algorithm
+	const queue: string[] = Object.keys(inDegree).filter(
+		(node) => inDegree[node] === 0,
+	);
+	const topologicalOrder: string[] = [];
+
+	while (queue.length > 0) {
+		const current = queue.shift();
+		if (current === undefined) {
+			break;
+		}
+		topologicalOrder.push(current);
+
+		adjList[current].forEach((neighbor) => {
+			inDegree[neighbor]--;
+			if (inDegree[neighbor] === 0) queue.push(neighbor);
+		});
+	}
+
+	if (topologicalOrder.length !== Object.keys(dependencies).length) {
+		throw new Error('Cycle detected in the dependency graph!');
+	}
+
+	return { order: topologicalOrder, graph: adjList };
+};
+
+export const onUpdateVariableNode = (
+	nodeToUpdate: string,
+	graph: VariableGraph,
+	topologicalOrder: string[],
+	callback: (node: string) => void,
+): void => {
+	const visited = new Set<string>();
+
+	// Start processing from the node to update
+	topologicalOrder.forEach((node) => {
+		if (node === nodeToUpdate || visited.has(node)) {
+			visited.add(node);
+			callback(node);
+			(graph[node] || []).forEach((child) => {
+				visited.add(child);
+			});
+		}
+	});
+};
+
+export const buildParentDependencyGraph = (
+	graph: VariableGraph,
+): VariableGraph => {
+	const parentGraph: VariableGraph = {};
+
+	// Initialize empty arrays for all nodes
+	Object.keys(graph).forEach((node) => {
+		parentGraph[node] = [];
+	});
+
+	// For each node and its children in the original graph
+	Object.entries(graph).forEach(([node, children]) => {
+		// For each child, add the current node as its parent
+		children.forEach((child) => {
+			parentGraph[child].push(node);
+		});
+	});
+
+	return parentGraph;
+};
+
+export const checkAPIInvocation = (
+	variablesToGetUpdated: string[],
+	variableData: IDashboardVariable,
+	parentDependencyGraph?: VariableGraph,
+): boolean => {
+	if (isEmpty(variableData.name)) {
+		return false;
+	}
+
+	if (isEmpty(parentDependencyGraph)) {
+		return true;
+	}
+
+	// if no dependency then true
+	const haveDependency =
+		parentDependencyGraph?.[variableData.name || '']?.length > 0;
+	if (!haveDependency) {
+		return true;
+	}
+
+	// if variable is in the list and has dependency then check if its the top element in the queue then true else false
+	return (
+		variablesToGetUpdated.length > 0 &&
+		variablesToGetUpdated[0] === variableData.name
+	);
+};

frontend/src/pages/MessagingQueues/MQDetails/MQDetails.tsx

@@ -24,13 +24,13 @@ const MQServiceDetailTypePerView = (
 	producerLatencyOption: ProducerLatencyOptions,
 ): Record<string, MessagingQueueServiceDetailType[]> => ({
 	[MessagingQueuesViewType.consumerLag.value]: [
-		MessagingQueueServiceDetailType.ProducerDetails,
 		MessagingQueueServiceDetailType.ConsumerDetails,
+		MessagingQueueServiceDetailType.ProducerDetails,
 		MessagingQueueServiceDetailType.NetworkLatency,
 	],
 	[MessagingQueuesViewType.partitionLatency.value]: [
-		MessagingQueueServiceDetailType.ProducerDetails,
 		MessagingQueueServiceDetailType.ConsumerDetails,
+		MessagingQueueServiceDetailType.ProducerDetails,
 	],
 	[MessagingQueuesViewType.producerLatency.value]: [
 		producerLatencyOption === ProducerLatencyOptions.Consumers
@@ -122,7 +122,7 @@ function MessagingQueuesDetails({
 	producerLatencyOption: ProducerLatencyOptions;
 }): JSX.Element {
 	const [currentTab, setCurrentTab] = useState<MessagingQueueServiceDetailType>(
-		MessagingQueueServiceDetailType.ProducerDetails,
+		MessagingQueueServiceDetailType.ConsumerDetails,
 	);
 
 	useEffect(() => {

frontend/src/pages/MessagingQueues/MessagingQueuesUtils.ts

@@ -179,13 +179,10 @@ export const convertToNanoseconds = (timestamp: number): bigint =>
 export const getStartAndEndTimesInMilliseconds = (
 	timestamp: number,
 ): { start: number; end: number } => {
-	const FIVE_MINUTES_IN_MILLISECONDS = 5 * 60 * 1000; // 300,000 milliseconds
+	const FIVE_MINUTES_IN_MILLISECONDS = 5 * 60 * 1000; // 5 minutes in milliseconds - check with Shivanshu once
 
-	const pointInTime = Math.floor(timestamp * 1000);
-
-	// Convert timestamp to milliseconds and floor it
-	const start = Math.floor(pointInTime - FIVE_MINUTES_IN_MILLISECONDS);
-	const end = Math.floor(pointInTime + FIVE_MINUTES_IN_MILLISECONDS);
+	const start = Math.floor(timestamp);
+	const end = Math.floor(start + FIVE_MINUTES_IN_MILLISECONDS);
 
 	return { start, end };
 };
@@ -314,8 +311,8 @@ export const getMetaDataAndAPIPerView = (
 	return {
 		[MessagingQueuesViewType.consumerLag.value]: {
 			tableApiPayload: {
-				start: (selectedTimelineQuery?.start || 0) * 1e6,
-				end: (selectedTimelineQuery?.end || 0) * 1e6,
+				start: (selectedTimelineQuery?.start || 0) * 1e9,
+				end: (selectedTimelineQuery?.end || 0) * 1e9,
 				variables: {
 					partition: selectedTimelineQuery?.partition,
 					topic: selectedTimelineQuery?.topic,

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.111.16
+	github.com/SigNoz/signoz-otel-collector v0.111.14
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3

go.sum

@@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
 github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
-github.com/SigNoz/signoz-otel-collector v0.111.16 h1:535uKH5Oux+35EsI+L3C6pnAP/Ye0PTCbVizXoL+VqE=
-github.com/SigNoz/signoz-otel-collector v0.111.16/go.mod h1:HJ4m0LY1MPsuZmuRF7Ixb+bY8rxgRzI0VXzOedESsjg=
+github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
+github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

pkg/query-service/app/clickhouseReader/reader.go

@@ -2694,8 +2694,8 @@ func (r *ClickHouseReader) GetTagsInfoInLastHeartBeatInterval(ctx context.Contex
 }
 
 // remove this after sometime
-func removeUnderscoreDuplicateFields(fields []model.Field) []model.Field {
-	lookup := map[string]model.Field{}
+func removeUnderscoreDuplicateFields(fields []model.LogField) []model.LogField {
+	lookup := map[string]model.LogField{}
 	for _, v := range fields {
 		lookup[v.Name+v.DataType] = v
 	}
@@ -2706,7 +2706,7 @@ func removeUnderscoreDuplicateFields(fields []model.Field) []model.Field {
 		}
 	}
 
-	updatedFields := []model.Field{}
+	updatedFields := []model.LogField{}
 	for _, v := range lookup {
 		updatedFields = append(updatedFields, v)
 	}
@@ -2717,11 +2717,11 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
 	// response will contain top level fields from the otel log model
 	response := model.GetFieldsResponse{
 		Selected:    constants.StaticSelectedLogFields,
-		Interesting: []model.Field{},
+		Interesting: []model.LogField{},
 	}
 
 	// get attribute keys
-	attributes := []model.Field{}
+	attributes := []model.LogField{}
 	query := fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsAttributeKeys)
 	err := r.db.Select(ctx, &attributes, query)
 	if err != nil {
@@ -2729,7 +2729,7 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
 	}
 
 	// get resource keys
-	resources := []model.Field{}
+	resources := []model.LogField{}
 	query = fmt.Sprintf("SELECT DISTINCT name, datatype from %s.%s group by name, datatype", r.logsDB, r.logsResourceKeys)
 	err = r.db.Select(ctx, &resources, query)
 	if err != nil {
@@ -2753,11 +2753,9 @@ func (r *ClickHouseReader) GetLogFields(ctx context.Context) (*model.GetFieldsRe
 	return &response, nil
 }
 
-func (r *ClickHouseReader) extractSelectedAndInterestingFields(tableStatement string, overrideFieldType string, fields *[]model.Field, response *model.GetFieldsResponse) {
+func (r *ClickHouseReader) extractSelectedAndInterestingFields(tableStatement string, fieldType string, fields *[]model.LogField, response *model.GetFieldsResponse) {
 	for _, field := range *fields {
-		if overrideFieldType != "" {
-			field.Type = overrideFieldType
-		}
+		field.Type = fieldType
 		// all static fields are assumed to be selected as we don't allow changing them
 		if isColumn(r.useLogsNewSchema, tableStatement, field.Type, field.Name, field.DataType) {
 			response.Selected = append(response.Selected, field)
@@ -2947,165 +2945,6 @@ func (r *ClickHouseReader) UpdateLogField(ctx context.Context, field *model.Upda
 	return nil
 }
 
-func (r *ClickHouseReader) GetTraceFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError) {
-	// response will contain top level fields from the otel trace model
-	response := model.GetFieldsResponse{
-		Selected:    []model.Field{},
-		Interesting: []model.Field{},
-	}
-
-	// get the top level selected fields
-	for _, field := range constants.NewStaticFieldsTraces {
-		if (v3.AttributeKey{} == field) {
-			continue
-		}
-		response.Selected = append(response.Selected, model.Field{
-			Name:     field.Key,
-			DataType: field.DataType.String(),
-			Type:     constants.Static,
-		})
-	}
-
-	// get attribute keys
-	attributes := []model.Field{}
-	query := fmt.Sprintf("SELECT tagKey, tagType, dataType from %s.%s group by tagKey, tagType, dataType", r.TraceDB, r.spanAttributesKeysTable)
-	rows, err := r.db.Query(ctx, query)
-	if err != nil {
-		return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal}
-	}
-	defer rows.Close()
-
-	var tagKey string
-	var dataType string
-	var tagType string
-	for rows.Next() {
-		if err := rows.Scan(&tagKey, &tagType, &dataType); err != nil {
-			return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal}
-		}
-		attributes = append(attributes, model.Field{
-			Name:     tagKey,
-			DataType: dataType,
-			Type:     tagType,
-		})
-	}
-
-	statements := []model.ShowCreateTableStatement{}
-	query = fmt.Sprintf("SHOW CREATE TABLE %s.%s", r.TraceDB, r.traceLocalTableName)
-	err = r.db.Select(ctx, &statements, query)
-	if err != nil {
-		return nil, &model.ApiError{Err: err, Typ: model.ErrorInternal}
-	}
-
-	r.extractSelectedAndInterestingFields(statements[0].Statement, "", &attributes, &response)
-
-	return &response, nil
-
-}
-
-func (r *ClickHouseReader) UpdateTraceField(ctx context.Context, field *model.UpdateField) *model.ApiError {
-	if !field.Selected {
-		return model.ForbiddenError(errors.New("removing a selected field is not allowed, please reach out to support."))
-	}
-
-	// name of the materialized column
-	colname := utils.GetClickhouseColumnNameV2(field.Type, field.DataType, field.Name)
-
-	field.DataType = strings.ToLower(field.DataType)
-
-	// dataType and chDataType of the materialized column
-	var dataTypeMap = map[string]string{
-		"string":  "string",
-		"bool":    "bool",
-		"int64":   "number",
-		"float64": "number",
-	}
-	var chDataTypeMap = map[string]string{
-		"string":  "String",
-		"bool":    "Bool",
-		"int64":   "Float64",
-		"float64": "Float64",
-	}
-	chDataType := chDataTypeMap[field.DataType]
-	dataType := dataTypeMap[field.DataType]
-
-	// typeName: tag => attributes, resource => resources
-	typeName := field.Type
-	if field.Type == string(v3.AttributeKeyTypeTag) {
-		typeName = constants.Attributes
-	} else if field.Type == string(v3.AttributeKeyTypeResource) {
-		typeName = constants.Resources
-	}
-
-	attrColName := fmt.Sprintf("%s_%s", typeName, dataType)
-	for _, table := range []string{r.traceLocalTableName, r.traceTableName} {
-		q := "ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s` %s DEFAULT %s['%s'] CODEC(ZSTD(1))"
-		query := fmt.Sprintf(q,
-			r.TraceDB, table,
-			r.cluster,
-			colname, chDataType,
-			attrColName,
-			field.Name,
-		)
-		err := r.db.Exec(ctx, query)
-		if err != nil {
-			return &model.ApiError{Err: err, Typ: model.ErrorInternal}
-		}
-
-		query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS `%s_exists` bool DEFAULT if(mapContains(%s, '%s') != 0, true, false) CODEC(ZSTD(1))",
-			r.TraceDB, table,
-			r.cluster,
-			colname,
-			attrColName,
-			field.Name,
-		)
-		err = r.db.Exec(ctx, query)
-		if err != nil {
-			return &model.ApiError{Err: err, Typ: model.ErrorInternal}
-		}
-	}
-
-	// create the index
-	if strings.ToLower(field.DataType) == "bool" {
-		// there is no point in creating index for bool attributes as the cardinality is just 2
-		return nil
-	}
-
-	if field.IndexType == "" {
-		field.IndexType = constants.DefaultLogSkipIndexType
-	}
-	if field.IndexGranularity == 0 {
-		field.IndexGranularity = constants.DefaultLogSkipIndexGranularity
-	}
-	query := fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS `%s_idx` (`%s`) TYPE %s  GRANULARITY %d",
-		r.TraceDB, r.traceLocalTableName,
-		r.cluster,
-		colname,
-		colname,
-		field.IndexType,
-		field.IndexGranularity,
-	)
-	err := r.db.Exec(ctx, query)
-	if err != nil {
-		return &model.ApiError{Err: err, Typ: model.ErrorInternal}
-	}
-
-	// add a default minmax index for numbers
-	if dataType == "number" {
-		query = fmt.Sprintf("ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS `%s_minmax_idx` (`%s`) TYPE minmax  GRANULARITY 1",
-			r.TraceDB, r.traceLocalTableName,
-			r.cluster,
-			colname,
-			colname,
-		)
-		err = r.db.Exec(ctx, query)
-		if err != nil {
-			return &model.ApiError{Err: err, Typ: model.ErrorInternal}
-		}
-	}
-
-	return nil
-}
-
 func (r *ClickHouseReader) GetLogs(ctx context.Context, params *model.LogsFilterParams) (*[]model.SignozLog, *model.ApiError) {
 	response := []model.SignozLog{}
 	fields, apiErr := r.GetLogFields(ctx)

pkg/query-service/app/http_handler.go

@@ -125,8 +125,6 @@ type APIHandler struct {
 	daemonsetsRepo   *inframetrics.DaemonSetsRepo
 	statefulsetsRepo *inframetrics.StatefulSetsRepo
 	jobsRepo         *inframetrics.JobsRepo
-
-	pvcsRepo *inframetrics.PvcsRepo
 }
 
 type APIHandlerOpts struct {
@@ -210,7 +208,6 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
 	daemonsetsRepo := inframetrics.NewDaemonSetsRepo(opts.Reader, querierv2)
 	statefulsetsRepo := inframetrics.NewStatefulSetsRepo(opts.Reader, querierv2)
 	jobsRepo := inframetrics.NewJobsRepo(opts.Reader, querierv2)
-	pvcsRepo := inframetrics.NewPvcsRepo(opts.Reader, querierv2)
 
 	aH := &APIHandler{
 		reader:                        opts.Reader,
@@ -240,7 +237,6 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) {
 		daemonsetsRepo:                daemonsetsRepo,
 		statefulsetsRepo:              statefulsetsRepo,
 		jobsRepo:                      jobsRepo,
-		pvcsRepo:                      pvcsRepo,
 	}
 
 	logsQueryBuilder := logsv3.PrepareLogsQuery
@@ -412,11 +408,6 @@ func (aH *APIHandler) RegisterInfraMetricsRoutes(router *mux.Router, am *AuthMid
 	podsSubRouter.HandleFunc("/attribute_values", am.ViewAccess(aH.getPodAttributeValues)).Methods(http.MethodGet)
 	podsSubRouter.HandleFunc("/list", am.ViewAccess(aH.getPodList)).Methods(http.MethodPost)
 
-	pvcsSubRouter := router.PathPrefix("/api/v1/pvcs").Subrouter()
-	pvcsSubRouter.HandleFunc("/attribute_keys", am.ViewAccess(aH.getPvcAttributeKeys)).Methods(http.MethodGet)
-	pvcsSubRouter.HandleFunc("/attribute_values", am.ViewAccess(aH.getPvcAttributeValues)).Methods(http.MethodGet)
-	pvcsSubRouter.HandleFunc("/list", am.ViewAccess(aH.getPvcList)).Methods(http.MethodPost)
-
 	nodesSubRouter := router.PathPrefix("/api/v1/nodes").Subrouter()
 	nodesSubRouter.HandleFunc("/attribute_keys", am.ViewAccess(aH.getNodeAttributeKeys)).Methods(http.MethodGet)
 	nodesSubRouter.HandleFunc("/attribute_values", am.ViewAccess(aH.getNodeAttributeValues)).Methods(http.MethodGet)
@@ -536,9 +527,6 @@ func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) {
 	router.HandleFunc("/api/v1/settings/ingestion_key", am.AdminAccess(aH.insertIngestionKey)).Methods(http.MethodPost)
 	router.HandleFunc("/api/v1/settings/ingestion_key", am.ViewAccess(aH.getIngestionKeys)).Methods(http.MethodGet)
 
-	router.HandleFunc("/api/v2/traces/fields", am.ViewAccess(aH.traceFields)).Methods(http.MethodGet)
-	router.HandleFunc("/api/v2/traces/fields", am.EditAccess(aH.updateTraceField)).Methods(http.MethodPost)
-
 	router.HandleFunc("/api/v1/version", am.OpenAccess(aH.getVersion)).Methods(http.MethodGet)
 	router.HandleFunc("/api/v1/featureFlags", am.OpenAccess(aH.getFeatureFlags)).Methods(http.MethodGet)
 	router.HandleFunc("/api/v1/configs", am.OpenAccess(aH.getConfigs)).Methods(http.MethodGet)
@@ -4087,9 +4075,10 @@ func (aH *APIHandler) CreateLogsPipeline(w http.ResponseWriter, r *http.Request)
 			zap.L().Warn("found no pipelines in the http request, this will delete all the pipelines")
 		}
 
-		validationErr := aH.LogsParsingPipelineController.ValidatePipelines(ctx, postable)
-		if validationErr != nil {
-			return nil, validationErr
+		for _, p := range postable {
+			if err := p.IsValid(); err != nil {
+				return nil, model.BadRequestStr(err.Error())
+			}
 		}
 
 		return aH.LogsParsingPipelineController.ApplyPipelines(ctx, postable)
@@ -4904,35 +4893,3 @@ func (aH *APIHandler) QueryRangeV4(w http.ResponseWriter, r *http.Request) {
 
 	aH.queryRangeV4(r.Context(), queryRangeParams, w, r)
 }
-
-func (aH *APIHandler) traceFields(w http.ResponseWriter, r *http.Request) {
-	fields, apiErr := aH.reader.GetTraceFields(r.Context())
-	if apiErr != nil {
-		RespondError(w, apiErr, "failed to fetch fields from the db")
-		return
-	}
-	aH.WriteJSON(w, r, fields)
-}
-
-func (aH *APIHandler) updateTraceField(w http.ResponseWriter, r *http.Request) {
-	field := model.UpdateField{}
-	if err := json.NewDecoder(r.Body).Decode(&field); err != nil {
-		apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err}
-		RespondError(w, apiErr, "failed to decode payload")
-		return
-	}
-
-	err := logs.ValidateUpdateFieldPayloadV2(&field)
-	if err != nil {
-		apiErr := &model.ApiError{Typ: model.ErrorBadData, Err: err}
-		RespondError(w, apiErr, "incorrect payload")
-		return
-	}
-
-	apiErr := aH.reader.UpdateTraceField(r.Context(), &field)
-	if apiErr != nil {
-		RespondError(w, apiErr, "failed to update field in the db")
-		return
-	}
-	aH.WriteJSON(w, r, field)
-}

pkg/query-service/app/infra.go

@@ -544,56 +544,3 @@ func (aH *APIHandler) getJobList(w http.ResponseWriter, r *http.Request) {
 
 	aH.Respond(w, jobList)
 }
-
-func (aH *APIHandler) getPvcList(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	req := model.VolumeListRequest{}
-
-	err := json.NewDecoder(r.Body).Decode(&req)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	pvcList, err := aH.pvcsRepo.GetPvcList(ctx, req)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	aH.Respond(w, pvcList)
-}
-
-func (aH *APIHandler) getPvcAttributeKeys(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	req, err := parseFilterAttributeKeyRequest(r)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	keys, err := aH.pvcsRepo.GetPvcAttributeKeys(ctx, *req)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	aH.Respond(w, keys)
-}
-
-func (aH *APIHandler) getPvcAttributeValues(w http.ResponseWriter, r *http.Request) {
-	ctx := r.Context()
-	req, err := parseFilterAttributeValueRequest(r)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	values, err := aH.pvcsRepo.GetPvcAttributeValues(ctx, *req)
-	if err != nil {
-		RespondError(w, &model.ApiError{Typ: model.ErrorInternal, Err: err}, nil)
-		return
-	}
-
-	aH.Respond(w, values)
-}

pkg/query-service/app/inframetrics/common.go

@@ -89,10 +89,6 @@ func getParamsForTopJobs(req model.JobListRequest) (int64, string, string) {
 	return getParamsForTopItems(req.Start, req.End)
 }
 
-func getParamsForTopVolumes(req model.VolumeListRequest) (int64, string, string) {
-	return getParamsForTopItems(req.Start, req.End)
-}
-
 // TODO(srikanthccv): remove this
 // What is happening here?
 // The `PrepareTimeseriesFilterQuery` uses the local time series table for sub-query because each fingerprint

pkg/query-service/app/inframetrics/namespaces.go

@@ -23,11 +23,10 @@ var (
 	}
 
 	queryNamesForNamespaces = map[string][]string{
-		"cpu":       {"A"},
-		"memory":    {"D"},
-		"pod_phase": {"H", "I", "J", "K"},
+		"cpu":    {"A"},
+		"memory": {"D"},
 	}
-	namespaceQueryNames = []string{"A", "D", "H", "I", "J", "K"}
+	namespaceQueryNames = []string{"A", "D"}
 
 	attributesKeysForNamespaces = []v3.AttributeKey{
 		{Key: "k8s_namespace_name"},
@@ -308,19 +307,6 @@ func (p *NamespacesRepo) GetNamespaceList(ctx context.Context, req model.Namespa
 				record.MemoryUsage = memory
 			}
 
-			if pending, ok := row.Data["H"].(float64); ok {
-				record.CountByPhase.Pending = int(pending)
-			}
-			if running, ok := row.Data["I"].(float64); ok {
-				record.CountByPhase.Running = int(running)
-			}
-			if succeeded, ok := row.Data["J"].(float64); ok {
-				record.CountByPhase.Succeeded = int(succeeded)
-			}
-			if failed, ok := row.Data["K"].(float64); ok {
-				record.CountByPhase.Failed = int(failed)
-			}
-
 			record.Meta = map[string]string{}
 			if _, ok := namespaceAttrs[record.NamespaceName]; ok {
 				record.Meta = namespaceAttrs[record.NamespaceName]

pkg/query-service/app/inframetrics/nodes.go

@@ -17,7 +17,7 @@ import (
 var (
 	metricToUseForNodes = "k8s_node_cpu_utilization"
 
-	nodeAttrsToEnrich = []string{"k8s_node_name", "k8s_node_uid", "k8s_cluster_name"}
+	nodeAttrsToEnrich = []string{"k8s_node_name", "k8s_node_uid"}
 
 	k8sNodeUIDAttrKey = "k8s_node_uid"
 
@@ -27,14 +27,13 @@ var (
 		"memory":             {"C"},
 		"memory_allocatable": {"D"},
 	}
-	nodeQueryNames = []string{"A", "B", "C", "D", "E", "F"}
+	nodeQueryNames = []string{"A", "B", "C", "D"}
 
 	metricNamesForNodes = map[string]string{
 		"cpu":                "k8s_node_cpu_utilization",
 		"cpu_allocatable":    "k8s_node_allocatable_cpu",
 		"memory":             "k8s_node_memory_usage",
 		"memory_allocatable": "k8s_node_allocatable_memory",
-		"node_condition":     "k8s_node_condition_ready",
 	}
 )
 
@@ -326,14 +325,6 @@ func (p *NodesRepo) GetNodeList(ctx context.Context, req model.NodeListRequest)
 				record.NodeMemoryAllocatable = memory
 			}
 
-			if ready, ok := row.Data["E"].(float64); ok {
-				record.CountByCondition.Ready = int(ready)
-			}
-
-			if notReady, ok := row.Data["F"].(float64); ok {
-				record.CountByCondition.NotReady = int(notReady)
-			}
-
 			record.Meta = map[string]string{}
 			if _, ok := nodeAttrs[record.NodeUID]; ok {
 				record.Meta = nodeAttrs[record.NodeUID]

pkg/query-service/app/inframetrics/nodes_query.go

@@ -109,74 +109,6 @@ var NodesTableListQuery = v3.QueryRangeParamsV3{
 				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
-			// node conditions - Ready
-			"E": {
-				QueryName:  "E",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForNodes["node_condition"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    1,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sNodeUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "E",
-				ReduceTo:         v3.ReduceToOperatorAvg,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-			// node conditions - NotReady
-			"F": {
-				QueryName:  "F",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForNodes["node_condition"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    0,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sNodeUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "F",
-				ReduceTo:         v3.ReduceToOperatorAvg,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
 		},
 		PanelType: v3.PanelTypeTable,
 		QueryType: v3.QueryTypeBuilder,

pkg/query-service/app/inframetrics/pods.go

@@ -27,7 +27,6 @@ var (
 		"k8s_daemonset_name",
 		"k8s_job_name",
 		"k8s_cronjob_name",
-		"k8s_cluster_name",
 	}
 
 	k8sPodUIDAttrKey = "k8s_pod_uid"
@@ -40,9 +39,8 @@ var (
 		"memory_request": {"E", "D"},
 		"memory_limit":   {"F", "D"},
 		"restarts":       {"G", "A"},
-		"pod_phase":      {"H", "I", "J", "K"},
 	}
-	podQueryNames = []string{"A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K"}
+	podQueryNames = []string{"A", "B", "C", "D", "E", "F", "G"}
 
 	metricNamesForPods = map[string]string{
 		"cpu":            "k8s_pod_cpu_utilization",
@@ -52,7 +50,6 @@ var (
 		"memory_request": "k8s_pod_memory_request_utilization",
 		"memory_limit":   "k8s_pod_memory_limit_utilization",
 		"restarts":       "k8s_container_restarts",
-		"pod_phase":      "k8s_pod_phase",
 	}
 )
 
@@ -368,22 +365,6 @@ func (p *PodsRepo) GetPodList(ctx context.Context, req model.PodListRequest) (mo
 				record.RestartCount = int(restarts)
 			}
 
-			if pending, ok := row.Data["H"].(float64); ok {
-				record.CountByPhase.Pending = int(pending)
-			}
-
-			if running, ok := row.Data["I"].(float64); ok {
-				record.CountByPhase.Running = int(running)
-			}
-
-			if succeeded, ok := row.Data["J"].(float64); ok {
-				record.CountByPhase.Succeeded = int(succeeded)
-			}
-
-			if failed, ok := row.Data["K"].(float64); ok {
-				record.CountByPhase.Failed = int(failed)
-			}
-
 			record.Meta = map[string]string{}
 			if _, ok := podAttrs[record.PodUID]; ok {
 				record.Meta = podAttrs[record.PodUID]

pkg/query-service/app/inframetrics/pods_query.go

@@ -54,7 +54,7 @@ var PodsTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "B",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod cpu limit utilization
@@ -80,7 +80,7 @@ var PodsTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "C",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod memory utilization
@@ -132,7 +132,7 @@ var PodsTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "E",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod memory limit utilization
@@ -158,7 +158,7 @@ var PodsTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "F",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			"G": {
@@ -187,142 +187,6 @@ var PodsTableListQuery = v3.QueryRangeParamsV3{
 				Functions:        []v3.Function{{Name: v3.FunctionNameRunningDiff}},
 				Disabled:         false,
 			},
-			// pod phase pending
-			"H": {
-				QueryName:  "H",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForPods["pod_phase"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    1,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPodUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "H",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationCount,
-				Disabled:         false,
-			},
-			// pod phase running
-			"I": {
-				QueryName:  "I",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForPods["pod_phase"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    2,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPodUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "I",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationCount,
-				Disabled:         false,
-			},
-			// pod phase succeeded
-			"J": {
-				QueryName:  "J",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForPods["pod_phase"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    3,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPodUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "J",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationCount,
-				Disabled:         false,
-			},
-			// pod phase failed
-			"K": {
-				QueryName:  "K",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForPods["pod_phase"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key: "__value",
-							},
-							Operator: v3.FilterOperatorEqual,
-							Value:    4,
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPodUIDAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "K",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAnyLast,
-				SpaceAggregation: v3.SpaceAggregationCount,
-				Disabled:         false,
-			},
 		},
 		PanelType: v3.PanelTypeTable,
 		QueryType: v3.QueryTypeBuilder,

pkg/query-service/app/inframetrics/pvcs.go

@@ -1,378 +0,0 @@
-package inframetrics
-
-import (
-	"context"
-	"math"
-	"sort"
-
-	"go.signoz.io/signoz/pkg/query-service/app/metrics/v4/helpers"
-	"go.signoz.io/signoz/pkg/query-service/common"
-	"go.signoz.io/signoz/pkg/query-service/interfaces"
-	"go.signoz.io/signoz/pkg/query-service/model"
-	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
-	"go.signoz.io/signoz/pkg/query-service/postprocess"
-	"golang.org/x/exp/slices"
-)
-
-var (
-	metricToUseForVolumes = "k8s_volume_available"
-
-	volumeAttrsToEnrich = []string{
-		"k8s_pod_uid",
-		"k8s_pod_name",
-		"k8s_namespace_name",
-		"k8s_node_name",
-		"k8s_statefulset_name",
-		"k8s_cluster_name",
-		"k8s_persistentvolumeclaim_name",
-	}
-
-	k8sPersistentVolumeClaimNameAttrKey = "k8s_persistentvolumeclaim_name"
-
-	queryNamesForVolumes = map[string][]string{
-		"available":   {"A"},
-		"capacity":    {"B", "A"},
-		"usage":       {"F1", "B", "A"},
-		"inodes":      {"C", "A"},
-		"inodes_free": {"D", "A"},
-		"inodes_used": {"E", "A"},
-	}
-
-	volumeQueryNames = []string{"A", "B", "C", "D", "E", "F1"}
-
-	metricNamesForVolumes = map[string]string{
-		"available":   "k8s_volume_available",
-		"capacity":    "k8s_volume_capacity",
-		"inodes":      "k8s_volume_inodes",
-		"inodes_free": "k8s_volume_inodes_free",
-		"inodes_used": "k8s_volume_inodes_used",
-	}
-)
-
-type PvcsRepo struct {
-	reader    interfaces.Reader
-	querierV2 interfaces.Querier
-}
-
-func NewPvcsRepo(reader interfaces.Reader, querierV2 interfaces.Querier) *PvcsRepo {
-	return &PvcsRepo{reader: reader, querierV2: querierV2}
-}
-
-func (p *PvcsRepo) GetPvcAttributeKeys(ctx context.Context, req v3.FilterAttributeKeyRequest) (*v3.FilterAttributeKeyResponse, error) {
-	req.DataSource = v3.DataSourceMetrics
-	req.AggregateAttribute = metricToUseForVolumes
-	if req.Limit == 0 {
-		req.Limit = 50
-	}
-
-	attributeKeysResponse, err := p.reader.GetMetricAttributeKeys(ctx, &req)
-	if err != nil {
-		return nil, err
-	}
-
-	// TODO(srikanthccv): only return resource attributes when we have a way to
-	// distinguish between resource attributes and other attributes.
-	filteredKeys := []v3.AttributeKey{}
-	for _, key := range attributeKeysResponse.AttributeKeys {
-		if slices.Contains(pointAttrsToIgnore, key.Key) {
-			continue
-		}
-		filteredKeys = append(filteredKeys, key)
-	}
-
-	return &v3.FilterAttributeKeyResponse{AttributeKeys: filteredKeys}, nil
-}
-
-func (p *PvcsRepo) GetPvcAttributeValues(ctx context.Context, req v3.FilterAttributeValueRequest) (*v3.FilterAttributeValueResponse, error) {
-	req.DataSource = v3.DataSourceMetrics
-	req.AggregateAttribute = metricToUseForVolumes
-	if req.Limit == 0 {
-		req.Limit = 50
-	}
-
-	attributeValuesResponse, err := p.reader.GetMetricAttributeValues(ctx, &req)
-	if err != nil {
-		return nil, err
-	}
-	return attributeValuesResponse, nil
-}
-
-func (p *PvcsRepo) getMetadataAttributes(ctx context.Context, req model.VolumeListRequest) (map[string]map[string]string, error) {
-	volumeAttrs := map[string]map[string]string{}
-
-	for _, key := range volumeAttrsToEnrich {
-		hasKey := false
-		for _, groupByKey := range req.GroupBy {
-			if groupByKey.Key == key {
-				hasKey = true
-				break
-			}
-		}
-		if !hasKey {
-			req.GroupBy = append(req.GroupBy, v3.AttributeKey{Key: key})
-		}
-	}
-
-	mq := v3.BuilderQuery{
-		DataSource: v3.DataSourceMetrics,
-		AggregateAttribute: v3.AttributeKey{
-			Key:      metricToUseForVolumes,
-			DataType: v3.AttributeKeyDataTypeFloat64,
-		},
-		Temporality: v3.Unspecified,
-		GroupBy:     req.GroupBy,
-	}
-
-	query, err := helpers.PrepareTimeseriesFilterQuery(req.Start, req.End, &mq)
-	if err != nil {
-		return nil, err
-	}
-
-	query = localQueryToDistributedQuery(query)
-
-	attrsListResponse, err := p.reader.GetListResultV3(ctx, query)
-	if err != nil {
-		return nil, err
-	}
-
-	for _, row := range attrsListResponse {
-		stringData := map[string]string{}
-		for key, value := range row.Data {
-			if str, ok := value.(string); ok {
-				stringData[key] = str
-			} else if strPtr, ok := value.(*string); ok {
-				stringData[key] = *strPtr
-			}
-		}
-
-		volumeName := stringData[k8sPersistentVolumeClaimNameAttrKey]
-		if _, ok := volumeAttrs[volumeName]; !ok {
-			volumeAttrs[volumeName] = map[string]string{}
-		}
-
-		for _, key := range req.GroupBy {
-			volumeAttrs[volumeName][key.Key] = stringData[key.Key]
-		}
-	}
-
-	return volumeAttrs, nil
-}
-
-func (p *PvcsRepo) getTopVolumeGroups(ctx context.Context, req model.VolumeListRequest, q *v3.QueryRangeParamsV3) ([]map[string]string, []map[string]string, error) {
-	step, timeSeriesTableName, samplesTableName := getParamsForTopVolumes(req)
-
-	queryNames := queryNamesForVolumes[req.OrderBy.ColumnName]
-	topVolumeGroupsQueryRangeParams := &v3.QueryRangeParamsV3{
-		Start: req.Start,
-		End:   req.End,
-		Step:  step,
-		CompositeQuery: &v3.CompositeQuery{
-			BuilderQueries: map[string]*v3.BuilderQuery{},
-			QueryType:      v3.QueryTypeBuilder,
-			PanelType:      v3.PanelTypeTable,
-		},
-	}
-
-	for _, queryName := range queryNames {
-		query := q.CompositeQuery.BuilderQueries[queryName].Clone()
-		query.StepInterval = step
-		query.MetricTableHints = &v3.MetricTableHints{
-			TimeSeriesTableName: timeSeriesTableName,
-			SamplesTableName:    samplesTableName,
-		}
-		if req.Filters != nil && len(req.Filters.Items) > 0 {
-			if query.Filters == nil {
-				query.Filters = &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}
-			}
-			query.Filters.Items = append(query.Filters.Items, req.Filters.Items...)
-		}
-		topVolumeGroupsQueryRangeParams.CompositeQuery.BuilderQueries[queryName] = query
-	}
-
-	queryResponse, _, err := p.querierV2.QueryRange(ctx, topVolumeGroupsQueryRangeParams)
-	if err != nil {
-		return nil, nil, err
-	}
-	formattedResponse, err := postprocess.PostProcessResult(queryResponse, topVolumeGroupsQueryRangeParams)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	if len(formattedResponse) == 0 || len(formattedResponse[0].Series) == 0 {
-		return nil, nil, nil
-	}
-
-	if req.OrderBy.Order == v3.DirectionDesc {
-		sort.Slice(formattedResponse[0].Series, func(i, j int) bool {
-			return formattedResponse[0].Series[i].Points[0].Value > formattedResponse[0].Series[j].Points[0].Value
-		})
-	} else {
-		sort.Slice(formattedResponse[0].Series, func(i, j int) bool {
-			return formattedResponse[0].Series[i].Points[0].Value < formattedResponse[0].Series[j].Points[0].Value
-		})
-	}
-
-	limit := math.Min(float64(req.Offset+req.Limit), float64(len(formattedResponse[0].Series)))
-
-	paginatedTopVolumeGroupsSeries := formattedResponse[0].Series[req.Offset:int(limit)]
-
-	topVolumeGroups := []map[string]string{}
-	for _, series := range paginatedTopVolumeGroupsSeries {
-		topVolumeGroups = append(topVolumeGroups, series.Labels)
-	}
-	allVolumeGroups := []map[string]string{}
-	for _, series := range formattedResponse[0].Series {
-		allVolumeGroups = append(allVolumeGroups, series.Labels)
-	}
-
-	return topVolumeGroups, allVolumeGroups, nil
-}
-
-func (p *PvcsRepo) GetPvcList(ctx context.Context, req model.VolumeListRequest) (model.VolumeListResponse, error) {
-	resp := model.VolumeListResponse{}
-
-	if req.Limit == 0 {
-		req.Limit = 10
-	}
-
-	if req.OrderBy == nil {
-		req.OrderBy = &v3.OrderBy{ColumnName: "usage", Order: v3.DirectionDesc}
-	}
-
-	if req.GroupBy == nil {
-		req.GroupBy = []v3.AttributeKey{{Key: k8sPersistentVolumeClaimNameAttrKey}}
-		resp.Type = model.ResponseTypeList
-	} else {
-		resp.Type = model.ResponseTypeGroupedList
-	}
-
-	step := int64(math.Max(float64(common.MinAllowedStepInterval(req.Start, req.End)), 60))
-
-	query := PvcsTableListQuery.Clone()
-
-	query.Start = req.Start
-	query.End = req.End
-	query.Step = step
-
-	for _, query := range query.CompositeQuery.BuilderQueries {
-		query.StepInterval = step
-		if req.Filters != nil && len(req.Filters.Items) > 0 {
-			if query.Filters == nil {
-				query.Filters = &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}}
-			}
-			query.Filters.Items = append(query.Filters.Items, req.Filters.Items...)
-		}
-		query.GroupBy = req.GroupBy
-	}
-
-	volumeAttrs, err := p.getMetadataAttributes(ctx, req)
-	if err != nil {
-		return resp, err
-	}
-
-	topVolumeGroups, allVolumeGroups, err := p.getTopVolumeGroups(ctx, req, query)
-	if err != nil {
-		return resp, err
-	}
-
-	groupFilters := map[string][]string{}
-	for _, topVolumeGroup := range topVolumeGroups {
-		for k, v := range topVolumeGroup {
-			groupFilters[k] = append(groupFilters[k], v)
-		}
-	}
-
-	for groupKey, groupValues := range groupFilters {
-		hasGroupFilter := false
-		if req.Filters != nil && len(req.Filters.Items) > 0 {
-			for _, filter := range req.Filters.Items {
-				if filter.Key.Key == groupKey {
-					hasGroupFilter = true
-					break
-				}
-			}
-		}
-
-		if !hasGroupFilter {
-			for _, query := range query.CompositeQuery.BuilderQueries {
-				query.Filters.Items = append(query.Filters.Items, v3.FilterItem{
-					Key:      v3.AttributeKey{Key: groupKey},
-					Value:    groupValues,
-					Operator: v3.FilterOperatorIn,
-				})
-			}
-		}
-	}
-
-	queryResponse, _, err := p.querierV2.QueryRange(ctx, query)
-	if err != nil {
-		return resp, err
-	}
-
-	formattedResponse, err := postprocess.PostProcessResult(queryResponse, query)
-	if err != nil {
-		return resp, err
-	}
-
-	records := []model.VolumeListRecord{}
-
-	for _, result := range formattedResponse {
-		for _, row := range result.Table.Rows {
-
-			record := model.VolumeListRecord{
-				VolumeUsage:      -1,
-				VolumeAvailable:  -1,
-				VolumeCapacity:   -1,
-				VolumeInodes:     -1,
-				VolumeInodesFree: -1,
-				VolumeInodesUsed: -1,
-				Meta:             map[string]string{},
-			}
-
-			if volumeName, ok := row.Data[k8sPersistentVolumeClaimNameAttrKey].(string); ok {
-				record.PersistentVolumeClaimName = volumeName
-			}
-
-			if volumeAvailable, ok := row.Data["A"].(float64); ok {
-				record.VolumeAvailable = volumeAvailable
-			}
-			if volumeCapacity, ok := row.Data["B"].(float64); ok {
-				record.VolumeCapacity = volumeCapacity
-			}
-
-			if volumeInodes, ok := row.Data["C"].(float64); ok {
-				record.VolumeInodes = volumeInodes
-			}
-
-			if volumeInodesFree, ok := row.Data["D"].(float64); ok {
-				record.VolumeInodesFree = volumeInodesFree
-			}
-
-			if volumeInodesUsed, ok := row.Data["E"].(float64); ok {
-				record.VolumeInodesUsed = volumeInodesUsed
-			}
-
-			record.VolumeUsage = record.VolumeCapacity - record.VolumeAvailable
-
-			record.Meta = map[string]string{}
-			if _, ok := volumeAttrs[record.PersistentVolumeClaimName]; ok {
-				record.Meta = volumeAttrs[record.PersistentVolumeClaimName]
-			}
-
-			for k, v := range row.Data {
-				if slices.Contains(volumeQueryNames, k) {
-					continue
-				}
-				if labelValue, ok := v.(string); ok {
-					record.Meta[k] = labelValue
-				}
-			}
-
-			records = append(records, record)
-		}
-	}
-	resp.Total = len(allVolumeGroups)
-	resp.Records = records
-
-	return resp, nil
-}

pkg/query-service/app/inframetrics/pvcs_query.go

@@ -1,204 +0,0 @@
-package inframetrics
-
-import v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
-
-var PvcsTableListQuery = v3.QueryRangeParamsV3{
-	CompositeQuery: &v3.CompositeQuery{
-		BuilderQueries: map[string]*v3.BuilderQuery{
-			// k8s.volume.available
-			"A": {
-				QueryName:  "A",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForVolumes["available"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key:      k8sPersistentVolumeClaimNameAttrKey,
-								DataType: v3.AttributeKeyDataTypeString,
-								Type:     v3.AttributeKeyTypeResource,
-							},
-							Operator: v3.FilterOperatorNotEqual,
-							Value:    "",
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPersistentVolumeClaimNameAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "A",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-			// k8s.volume.capacity
-			"B": {
-				QueryName:  "B",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForVolumes["capacity"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key:      k8sPersistentVolumeClaimNameAttrKey,
-								DataType: v3.AttributeKeyDataTypeString,
-								Type:     v3.AttributeKeyTypeResource,
-							},
-							Operator: v3.FilterOperatorNotEqual,
-							Value:    "",
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPersistentVolumeClaimNameAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "B",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-			"F1": {
-				QueryName:  "F1",
-				DataSource: v3.DataSourceMetrics,
-				Expression: "B - A",
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items:    []v3.FilterItem{},
-				},
-				ReduceTo: v3.ReduceToOperatorLast,
-			},
-			// k8s.volume.inodes
-			"C": {
-				QueryName:  "C",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForVolumes["inodes"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key:      k8sPersistentVolumeClaimNameAttrKey,
-								DataType: v3.AttributeKeyDataTypeString,
-								Type:     v3.AttributeKeyTypeResource,
-							},
-							Operator: v3.FilterOperatorNotEqual,
-							Value:    "",
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPersistentVolumeClaimNameAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "C",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-			// k8s.volume.inodes_free
-			"D": {
-				QueryName:  "D",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForVolumes["inodes_free"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key:      k8sPersistentVolumeClaimNameAttrKey,
-								DataType: v3.AttributeKeyDataTypeString,
-								Type:     v3.AttributeKeyTypeResource,
-							},
-							Operator: v3.FilterOperatorNotEqual,
-							Value:    "",
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPersistentVolumeClaimNameAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "D",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-			// k8s.volume.inodes_used
-			"E": {
-				QueryName:  "E",
-				DataSource: v3.DataSourceMetrics,
-				AggregateAttribute: v3.AttributeKey{
-					Key:      metricNamesForVolumes["inodes_used"],
-					DataType: v3.AttributeKeyDataTypeFloat64,
-				},
-				Temporality: v3.Unspecified,
-				Filters: &v3.FilterSet{
-					Operator: "AND",
-					Items: []v3.FilterItem{
-						{
-							Key: v3.AttributeKey{
-								Key:      k8sPersistentVolumeClaimNameAttrKey,
-								DataType: v3.AttributeKeyDataTypeString,
-								Type:     v3.AttributeKeyTypeResource,
-							},
-							Operator: v3.FilterOperatorNotEqual,
-							Value:    "",
-						},
-					},
-				},
-				GroupBy: []v3.AttributeKey{
-					{
-						Key:      k8sPersistentVolumeClaimNameAttrKey,
-						DataType: v3.AttributeKeyDataTypeString,
-						Type:     v3.AttributeKeyTypeResource,
-					},
-				},
-				Expression:       "E",
-				ReduceTo:         v3.ReduceToOperatorLast,
-				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationSum,
-				Disabled:         false,
-			},
-		},
-		PanelType: v3.PanelTypeTable,
-		QueryType: v3.QueryTypeBuilder,
-	},
-	Version:      "v4",
-	FormatForWeb: true,
-}

pkg/query-service/app/inframetrics/workload_query.go

@@ -4,13 +4,13 @@ import v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 
 var (
 	metricNamesForWorkloads = map[string]string{
-		"cpu":            "k8s_pod_cpu_utilization",
-		"cpu_request":    "k8s_pod_cpu_request_utilization",
-		"cpu_limit":      "k8s_pod_cpu_limit_utilization",
-		"memory":         "k8s_pod_memory_usage",
-		"memory_request": "k8s_pod_memory_request_utilization",
-		"memory_limit":   "k8s_pod_memory_limit_utilization",
-		"restarts":       "k8s_container_restarts",
+		"cpu":          "k8s_pod_cpu_utilization",
+		"cpu_req":      "k8s_pod_cpu_request_utilization",
+		"cpu_limit":    "k8s_pod_cpu_limit_utilization",
+		"memory":       "k8s_pod_memory_usage",
+		"memory_req":   "k8s_pod_memory_request_utilization",
+		"memory_limit": "k8s_pod_memory_limit_utilization",
+		"restarts":     "k8s_container_restarts",
 	}
 )
 
@@ -54,7 +54,7 @@ var WorkloadTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "B",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod cpu limit utilization
@@ -74,7 +74,7 @@ var WorkloadTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "C",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod memory utilization
@@ -114,7 +114,7 @@ var WorkloadTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "E",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			// pod memory limit utilization
@@ -134,7 +134,7 @@ var WorkloadTableListQuery = v3.QueryRangeParamsV3{
 				Expression:       "F",
 				ReduceTo:         v3.ReduceToOperatorAvg,
 				TimeAggregation:  v3.TimeAggregationAvg,
-				SpaceAggregation: v3.SpaceAggregationAvg,
+				SpaceAggregation: v3.SpaceAggregationSum,
 				Disabled:         false,
 			},
 			"G": {

pkg/query-service/app/integrations/messagingQueues/kafka/sql.go

@@ -6,32 +6,28 @@ import (
 
 func generateConsumerSQL(start, end int64, topic, partition, consumerGroup, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH consumer_query AS (
     SELECT
-        resource_string_service$$name,
+        serviceName,
         quantile(0.99)(durationNano) / 1000000 AS p99,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count,
-        avg(CASE WHEN has(attributes_number, 'messaging.message.body.size') THEN attributes_number['messaging.message.body.size'] ELSE NULL END) AS avg_msg_size
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count,
+        avg(CASE WHEN has(numberTagMap, 'messaging.message.body.size') THEN numberTagMap['messaging.message.body.size'] ELSE NULL END) AS avg_msg_size
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 5
-        AND attribute_string_messaging$$system = '%s'
-        AND attributes_string['messaging.destination.name'] = '%s'
-        AND attributes_string['messaging.destination.partition.id'] = '%s'
-        AND attributes_string['messaging.kafka.consumer.group'] = '%s'
-    GROUP BY resource_string_service$$name
+        AND msgSystem = '%s'
+        AND stringTagMap['messaging.destination.name'] = '%s'
+        AND stringTagMap['messaging.destination.partition.id'] = '%s'
+        AND stringTagMap['messaging.kafka.consumer.group'] = '%s'
+    GROUP BY serviceName
 )
 
 SELECT
-    resource_string_service$$name AS service_name,
+    serviceName AS service_name,
     p99,
     COALESCE((error_count * 100.0) / total_requests, 0) AS error_rate,
     COALESCE(total_requests / %d, 0) AS throughput,
@@ -39,31 +35,27 @@ SELECT
 FROM
     consumer_query
 ORDER BY
-    resource_string_service$$name;
-`, start, end, tsBucketStart, tsBucketEnd, queueType, topic, partition, consumerGroup, timeRange)
+    serviceName;
+`, start, end, queueType, topic, partition, consumerGroup, timeRange)
 	return query
 }
 
 // S1 landing
 func generatePartitionLatencySQL(start, end int64, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH partition_query AS (
     SELECT
         quantile(0.99)(durationNano) / 1000000 AS p99,
         count(*) AS total_requests,
-        attributes_string['messaging.destination.name'] AS topic,
-		attributes_string['messaging.destination.partition.id'] AS partition
-    FROM signoz_traces.distributed_signoz_index_v3
+        stringTagMap['messaging.destination.name'] AS topic,
+		stringTagMap['messaging.destination.partition.id'] AS partition
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 4
-        AND attribute_string_messaging$$system = '%s'
+        AND msgSystem = '%s'
     GROUP BY topic, partition
 )
 
@@ -76,39 +68,35 @@ FROM
     partition_query
 ORDER BY
     topic;
-`, start, end, tsBucketStart, tsBucketEnd, queueType, timeRange)
+`, start, end, queueType, timeRange)
 	return query
 }
 
 // S1 consumer
 func generateConsumerPartitionLatencySQL(start, end int64, topic, partition, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH consumer_pl AS (
     SELECT
-        attributes_string['messaging.kafka.consumer.group'] AS consumer_group,
-        resource_string_service$$name,
+        stringTagMap['messaging.kafka.consumer.group'] AS consumer_group,
+        serviceName,
         quantile(0.99)(durationNano) / 1000000 AS p99,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 5
-        AND attribute_string_messaging$$system = '%s'
-        AND attributes_string['messaging.destination.name'] = '%s'
-        AND attributes_string['messaging.destination.partition.id'] = '%s'
-    GROUP BY consumer_group, resource_string_service$$name
+        AND msgSystem = '%s'
+        AND stringTagMap['messaging.destination.name'] = '%s'
+        AND stringTagMap['messaging.destination.partition.id'] = '%s'
+    GROUP BY consumer_group, serviceName
 )
 
 SELECT
     consumer_group,
-    resource_string_service$$name AS service_name,
+    serviceName AS service_name,
     p99,
     COALESCE((error_count * 100.0) / total_requests, 0) AS error_rate,
     COALESCE(total_requests / %d, 0) AS throughput
@@ -116,68 +104,61 @@ FROM
     consumer_pl
 ORDER BY
     consumer_group;
-`, start, end, tsBucketStart, tsBucketEnd, queueType, topic, partition, timeRange)
+`, start, end, queueType, topic, partition, timeRange)
 	return query
 }
 
 // S3, producer overview
 func generateProducerPartitionThroughputSQL(start, end int64, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000 // t, svc, rps, byte*, p99, err
+	// t, svc, rps, byte*, p99, err
 	query := fmt.Sprintf(`
 WITH producer_latency AS (
     SELECT
-		resource_string_service$$name,
+		serviceName,
         quantile(0.99)(durationNano) / 1000000 AS p99,
-		attributes_string['messaging.destination.name'] AS topic,
+		stringTagMap['messaging.destination.name'] AS topic,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 4
-        AND attribute_string_messaging$$system = '%s'
-    GROUP BY topic, resource_string_service$$name
+        AND msgSystem = '%s'
+    GROUP BY topic, serviceName
 )
 
 SELECT
 	topic,
-	resource_string_service$$name AS service_name,
+	serviceName AS service_name,
     p99,
     COALESCE((error_count * 100.0) / total_requests, 0) AS error_rate,
     COALESCE(total_requests / %d, 0) AS throughput
 FROM
     producer_latency
-`, start, end, tsBucketStart, tsBucketEnd, queueType, timeRange)
+`, start, end, queueType, timeRange)
 	return query
 }
 
 // S3, producer topic/service overview
 func generateProducerTopicLatencySQL(start, end int64, topic, service, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH consumer_latency AS (
     SELECT
         quantile(0.99)(durationNano) / 1000000 AS p99,
-		attributes_string['messaging.destination.partition.id'] AS partition,
+		stringTagMap['messaging.destination.partition.id'] AS partition,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 4
-		AND resource_string_service$$name = '%s'
-        AND attribute_string_messaging$$system = '%s'
-		AND attributes_string['messaging.destination.name'] = '%s'
+		AND serviceName = '%s'
+        AND msgSystem = '%s'
+		AND stringTagMap['messaging.destination.name'] = '%s'
     GROUP BY partition
 )
 
@@ -188,38 +169,34 @@ SELECT
     COALESCE(total_requests / %d, 0) AS throughput
 FROM
     consumer_latency
-`, start, end, tsBucketStart, tsBucketEnd, service, queueType, topic, timeRange)
+`, start, end, service, queueType, topic, timeRange)
 	return query
 }
 
 // S3 consumer overview
 func generateConsumerLatencySQL(start, end int64, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH consumer_latency AS (
     SELECT
-        resource_string_service$$name,
-        attributes_string['messaging.destination.name'] AS topic,
+        serviceName,
+        stringTagMap['messaging.destination.name'] AS topic,
         quantile(0.99)(durationNano) / 1000000 AS p99,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count,
-        SUM(attributes_number['messaging.message.body.size']) AS total_bytes
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count,
+        SUM(numberTagMap['messaging.message.body.size']) AS total_bytes
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 5
-        AND attribute_string_messaging$$system = '%s'
-    GROUP BY topic, resource_string_service$$name
+        AND msgSystem = '%s'
+    GROUP BY topic, serviceName
 )
 
 SELECT
     topic,
-    resource_string_service$$name AS service_name,
+    serviceName AS service_name,
     p99,
     COALESCE((error_count * 100.0) / total_requests, 0) AS error_rate,
     COALESCE(total_requests / %d, 0) AS ingestion_rate,
@@ -228,32 +205,28 @@ FROM
     consumer_latency
 ORDER BY
     topic;
-`, start, end, tsBucketStart, tsBucketEnd, queueType, timeRange, timeRange)
+`, start, end, queueType, timeRange, timeRange)
 	return query
 }
 
 // S3 consumer topic/service
 func generateConsumerServiceLatencySQL(start, end int64, topic, service, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH consumer_latency AS (
     SELECT
         quantile(0.99)(durationNano) / 1000000 AS p99,
-		attributes_string['messaging.destination.partition.id'] AS partition,
+		stringTagMap['messaging.destination.partition.id'] AS partition,
         COUNT(*) AS total_requests,
-        sumIf(1, status_code = 2) AS error_count
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 5
-		AND resource_string_service$$name = '%s'
-        AND attribute_string_messaging$$system = '%s'
-		AND attributes_string['messaging.destination.name'] = '%s'
+		AND serviceName = '%s'
+        AND msgSystem = '%s'
+		AND stringTagMap['messaging.destination.name'] = '%s'
     GROUP BY partition
 )
 
@@ -264,7 +237,7 @@ SELECT
     COALESCE(total_requests / %d, 0) AS throughput
 FROM
     consumer_latency
-`, start, end, tsBucketStart, tsBucketEnd, service, queueType, topic, timeRange)
+`, start, end, service, queueType, topic, timeRange)
 	return query
 }
 
@@ -273,26 +246,26 @@ func generateProducerConsumerEvalSQL(start, end int64, queueType string, evalTim
 	query := fmt.Sprintf(`
 WITH trace_data AS (
     SELECT
-        p.resource_string_service$$name AS producer_service,
-        c.resource_string_service$$name AS consumer_service,
-        p.trace_id,
+        p.serviceName AS producer_service,
+        c.serviceName AS consumer_service,
+        p.traceID,
         p.timestamp AS producer_timestamp,
         c.timestamp AS consumer_timestamp,
         p.durationNano AS durationNano,
         (toUnixTimestamp64Nano(c.timestamp) - toUnixTimestamp64Nano(p.timestamp)) + p.durationNano AS time_difference
     FROM
-        signoz_traces.distributed_signoz_index_v3 p
+        signoz_traces.distributed_signoz_index_v2 p
     INNER JOIN
-        signoz_traces.distributed_signoz_index_v3 c
-            ON p.trace_id = c.trace_id
-            AND c.parent_span_id = p.span_id
+        signoz_traces.distributed_signoz_index_v2 c
+            ON p.traceID = c.traceID
+            AND c.parentSpanID = p.spanID
     WHERE
         p.kind = 4
         AND c.kind = 5
         AND toUnixTimestamp64Nano(p.timestamp) BETWEEN '%d' AND '%d'
         AND toUnixTimestamp64Nano(c.timestamp) BETWEEN '%d' AND '%d'
-        AND c.attribute_string_messaging$$system = '%s'
-        AND p.attribute_string_messaging$$system = '%s'
+        AND c.msgSystem = '%s'
+        AND p.msgSystem = '%s'
 )
 
 SELECT
@@ -305,7 +278,7 @@ SELECT
         arrayMap(x -> x.1,
             arraySort(
                 x -> -x.2,
-                groupArrayIf((trace_id, time_difference), time_difference > '%d')
+                groupArrayIf((traceID, time_difference), time_difference > '%d')
             )
         ),
         1, 10
@@ -320,107 +293,91 @@ GROUP BY
 
 func generateProducerSQL(start, end int64, topic, partition, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 WITH producer_query AS (
     SELECT
-        resource_string_service$$name,
+        serviceName,
         quantile(0.99)(durationNano) / 1000000 AS p99,
         count(*) AS total_count,
-        sumIf(1, status_code = 2) AS error_count
-    FROM signoz_traces.distributed_signoz_index_v3
+        sumIf(1, statusCode = 2) AS error_count
+    FROM signoz_traces.distributed_signoz_index_v2
     WHERE
         timestamp >= '%d'
         AND timestamp <= '%d'
-        AND ts_bucket_start >=  '%d'
-        AND ts_bucket_start <= '%d'
         AND kind = 4
-        AND attribute_string_messaging$$system = '%s'
-        AND attributes_string['messaging.destination.name'] = '%s'
-        AND attributes_string['messaging.destination.partition.id'] = '%s'
-    GROUP BY resource_string_service$$name
+        AND msgSystem = '%s'
+        AND stringTagMap['messaging.destination.name'] = '%s'
+        AND stringTagMap['messaging.destination.partition.id'] = '%s'
+    GROUP BY serviceName
 )
 
 SELECT
-    resource_string_service$$name AS service_name,
+    serviceName AS service_name,
     p99,
     COALESCE((error_count * 100.0) / total_count, 0) AS error_percentage,
     COALESCE(total_count / %d, 0) AS throughput
 FROM
     producer_query
 ORDER BY
-    resource_string_service$$name;
-`, start, end, tsBucketStart, tsBucketEnd, queueType, topic, partition, timeRange)
+    serviceName;
+`, start, end, queueType, topic, partition, timeRange)
 	return query
 }
 
 func generateNetworkLatencyThroughputSQL(start, end int64, consumerGroup, partitionID, queueType string) string {
 	timeRange := (end - start) / 1000000000
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 SELECT
-    attributes_string['messaging.client_id'] AS client_id,
-	resources_string['service.instance.id'] AS service_instance_id,
-    resource_string_service$$name AS service_name,
+    stringTagMap['messaging.client_id'] AS client_id,
+	stringTagMap['service.instance.id'] AS service_instance_id,
+    serviceName AS service_name,
     count(*) / %d AS throughput
-FROM signoz_traces.distributed_signoz_index_v3
+FROM signoz_traces.distributed_signoz_index_v2
 WHERE
     timestamp >= '%d'
     AND timestamp <= '%d'
-    AND ts_bucket_start >=  '%d'
-    AND ts_bucket_start <= '%d'
     AND kind = 5
-    AND attribute_string_messaging$$system = '%s' 
-    AND attributes_string['messaging.kafka.consumer.group'] = '%s'
-    AND attributes_string['messaging.destination.partition.id'] = '%s'
+    AND msgSystem = '%s' 
+    AND stringTagMap['messaging.kafka.consumer.group'] = '%s'
+    AND stringTagMap['messaging.destination.partition.id'] = '%s'
 GROUP BY service_name, client_id, service_instance_id
 ORDER BY throughput DESC
-`, timeRange, start, end, tsBucketStart, tsBucketEnd, queueType, consumerGroup, partitionID)
+`, timeRange, start, end, queueType, consumerGroup, partitionID)
 	return query
 }
 
 func onboardProducersSQL(start, end int64, queueType string) string {
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 SELECT 
     COUNT(*) = 0 AS entries,
-    COUNT(IF(attribute_string_messaging$$system = '%s', 1, NULL)) = 0 AS queue,
+    COUNT(IF(msgSystem = '%s', 1, NULL)) = 0 AS queue,
     COUNT(IF(kind = 4, 1, NULL)) = 0 AS kind,
-    COUNT(IF(has(attributes_string, 'messaging.destination.name'), 1, NULL)) = 0 AS destination,
-    COUNT(IF(has(attributes_string, 'messaging.destination.partition.id'), 1, NULL)) = 0 AS partition
+    COUNT(IF(has(stringTagMap, 'messaging.destination.name'), 1, NULL)) = 0 AS destination,
+    COUNT(IF(has(stringTagMap, 'messaging.destination.partition.id'), 1, NULL)) = 0 AS partition
 FROM 
-    signoz_traces.distributed_signoz_index_v3
+    signoz_traces.distributed_signoz_index_v2
 WHERE 
     timestamp >= '%d'
-    AND timestamp <= '%d'
-    AND ts_bucket_start >=  '%d'
-    AND ts_bucket_start <= '%d';`, queueType, start, end, tsBucketStart, tsBucketEnd)
+    AND timestamp <= '%d';`, queueType, start, end)
 	return query
 }
 
 func onboardConsumerSQL(start, end int64, queueType string) string {
-	tsBucketStart := (start / 1000000000) - 1800
-	tsBucketEnd := end / 1000000000
 	query := fmt.Sprintf(`
 SELECT  
     COUNT(*) = 0 AS entries,
-    COUNT(IF(attribute_string_messaging$$system = '%s', 1, NULL)) = 0 AS queue,
+    COUNT(IF(msgSystem = '%s', 1, NULL)) = 0 AS queue,
     COUNT(IF(kind = 5, 1, NULL)) = 0 AS kind,
-    COUNT(resource_string_service$$name) = 0 AS svc,
-    COUNT(IF(has(attributes_string, 'messaging.destination.name'), 1, NULL)) = 0 AS destination,
-    COUNT(IF(has(attributes_string, 'messaging.destination.partition.id'), 1, NULL)) = 0 AS partition,
-    COUNT(IF(has(attributes_string, 'messaging.kafka.consumer.group'), 1, NULL)) = 0 AS cgroup,
-    COUNT(IF(has(attributes_number, 'messaging.message.body.size'), 1, NULL)) = 0 AS bodysize,
-    COUNT(IF(has(attributes_string, 'messaging.client_id'), 1, NULL)) = 0 AS clientid,
-    COUNT(IF(has(resources_string, 'service.instance.id'), 1, NULL)) = 0 AS instanceid
-FROM signoz_traces.distributed_signoz_index_v3
+    COUNT(serviceName) = 0 AS svc,
+    COUNT(IF(has(stringTagMap, 'messaging.destination.name'), 1, NULL)) = 0 AS destination,
+    COUNT(IF(has(stringTagMap, 'messaging.destination.partition.id'), 1, NULL)) = 0 AS partition,
+    COUNT(IF(has(stringTagMap, 'messaging.kafka.consumer.group'), 1, NULL)) = 0 AS cgroup,
+    COUNT(IF(has(numberTagMap, 'messaging.message.body.size'), 1, NULL)) = 0 AS bodysize,
+    COUNT(IF(has(stringTagMap, 'messaging.client_id'), 1, NULL)) = 0 AS clientid,
+    COUNT(IF(has(stringTagMap, 'service.instance.id'), 1, NULL)) = 0 AS instanceid
+FROM signoz_traces.distributed_signoz_index_v2
 WHERE 
     timestamp >= '%d'
-    AND timestamp <= '%d'
-    AND ts_bucket_start >=  '%d'
-    AND ts_bucket_start <= '%d'	;`, queueType, start, end, tsBucketStart, tsBucketEnd)
+    AND timestamp <= '%d';`, queueType, start, end)
 	return query
 }

pkg/query-service/app/logparsingpipeline/controller.go

@@ -94,45 +94,6 @@ func (ic *LogParsingPipelineController) ApplyPipelines(
 	return ic.GetPipelinesByVersion(ctx, cfg.Version)
 }
 
-func (ic *LogParsingPipelineController) ValidatePipelines(
-	ctx context.Context,
-	postedPipelines []PostablePipeline,
-) *model.ApiError {
-	for _, p := range postedPipelines {
-		if err := p.IsValid(); err != nil {
-			return model.BadRequestStr(err.Error())
-		}
-	}
-
-	// Also run a collector simulation to ensure config is fit
-	// for e2e use with a collector
-	pipelines := []Pipeline{}
-	for _, pp := range postedPipelines {
-		pipelines = append(pipelines, Pipeline{
-			Id:          uuid.New().String(),
-			OrderId:     pp.OrderId,
-			Enabled:     pp.Enabled,
-			Name:        pp.Name,
-			Alias:       pp.Alias,
-			Description: &pp.Description,
-			Filter:      pp.Filter,
-			Config:      pp.Config,
-		})
-	}
-
-	sampleLogs := []model.SignozLog{{Body: ""}}
-	_, _, simulationErr := SimulatePipelinesProcessing(
-		ctx, pipelines, sampleLogs,
-	)
-	if simulationErr != nil {
-		return model.BadRequest(fmt.Errorf(
-			"invalid pipelines config: %w", simulationErr.ToError(),
-		))
-	}
-
-	return nil
-}
-
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(

pkg/query-service/app/logs/parser.go

@@ -228,8 +228,8 @@ func parseColumn(s string) (*string, error) {
 	return &colName, nil
 }
 
-func arrayToMap(fields []model.Field) map[string]model.Field {
-	res := map[string]model.Field{}
+func arrayToMap(fields []model.LogField) map[string]model.LogField {
+	res := map[string]model.LogField{}
 	for _, field := range fields {
 		res[field.Name] = field
 	}
@@ -251,7 +251,7 @@ func replaceInterestingFields(allFields *model.GetFieldsResponse, queryTokens []
 	return queryTokens, nil
 }
 
-func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]model.Field, interestingFieldLookup map[string]model.Field) (string, error) {
+func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]model.LogField, interestingFieldLookup map[string]model.LogField) (string, error) {
 	op := strings.TrimSpace(operatorRegex.FindString(queryToken))
 	opLower := strings.ToLower(op)
 
@@ -283,7 +283,7 @@ func replaceFieldInToken(queryToken string, selectedFieldsLookup map[string]mode
 			}
 		} else {
 			// creating the query token here as we have the metadata
-			field := model.Field{}
+			field := model.LogField{}
 
 			if sfield, ok := selectedFieldsLookup[sqlColName]; ok {
 				field = sfield

pkg/query-service/app/logs/parser_test.go

@@ -238,14 +238,14 @@ func TestParseColumn(t *testing.T) {
 func TestReplaceInterestingFields(t *testing.T) {
 	queryTokens := []string{"id.userid IN (100) ", "and id_key >= 50 ", `AND body ILIKE '%searchstring%'`}
 	allFields := model.GetFieldsResponse{
-		Selected: []model.Field{
+		Selected: []model.LogField{
 			{
 				Name:     "id_key",
 				DataType: "int64",
 				Type:     "attributes",
 			},
 		},
-		Interesting: []model.Field{
+		Interesting: []model.LogField{
 			{
 				Name:     "id.userid",
 				DataType: "int64",
@@ -326,7 +326,7 @@ func TestCheckIfPrevousPaginateAndModifyOrder(t *testing.T) {
 }
 
 var generateSQLQueryFields = model.GetFieldsResponse{
-	Selected: []model.Field{
+	Selected: []model.LogField{
 		{
 			Name:     "field1",
 			DataType: "int64",
@@ -348,7 +348,7 @@ var generateSQLQueryFields = model.GetFieldsResponse{
 			Type:     "static",
 		},
 	},
-	Interesting: []model.Field{
+	Interesting: []model.LogField{
 		{
 			Name:     "FielD1",
 			DataType: "int64",

pkg/query-service/app/logs/validator.go

@@ -6,7 +6,6 @@ import (
 
 	"go.signoz.io/signoz/pkg/query-service/constants"
 	"go.signoz.io/signoz/pkg/query-service/model"
-	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 )
 
 func ValidateUpdateFieldPayload(field *model.UpdateField) error {
@@ -39,36 +38,3 @@ func ValidateUpdateFieldPayload(field *model.UpdateField) error {
 	}
 	return nil
 }
-
-func ValidateUpdateFieldPayloadV2(field *model.UpdateField) error {
-	if field.Name == "" {
-		return fmt.Errorf("name cannot be empty")
-	}
-	if field.Type == "" {
-		return fmt.Errorf("type cannot be empty")
-	}
-	if field.DataType == "" {
-		return fmt.Errorf("dataType cannot be empty")
-	}
-
-	// the logs api uses the old names i.e attributes and resources while traces use tag and attribute.
-	// update log api to use tag and attribute.
-	matched, err := regexp.MatchString(fmt.Sprintf("^(%s|%s)$", v3.AttributeKeyTypeTag, v3.AttributeKeyTypeResource), field.Type)
-	if err != nil {
-		return err
-	}
-	if !matched {
-		return fmt.Errorf("type %s not supported", field.Type)
-	}
-
-	if field.IndexType != "" {
-		matched, err := regexp.MatchString(`^(minmax|set\([0-9]\)|bloom_filter\((0?.?[0-9]+|1)\)|tokenbf_v1\([0-9]+,[0-9]+,[0-9]+\)|ngrambf_v1\([0-9]+,[0-9]+,[0-9]+,[0-9]+\))$`, field.IndexType)
-		if err != nil {
-			return err
-		}
-		if !matched {
-			return fmt.Errorf("index type %s not supported", field.IndexType)
-		}
-	}
-	return nil
-}

pkg/query-service/app/metrics/query_builder.go

@@ -5,73 +5,9 @@ import (
 	"reflect"
 	"strings"
 
-	v3 "go.signoz.io/signoz/pkg/query-service/model/v3"
 	"go.uber.org/zap"
 )
 
-func AddMetricValueFilter(mq *v3.BuilderQuery) *v3.MetricValueFilter {
-
-	var metricValueFilter *v3.MetricValueFilter = nil
-
-	if mq != nil && mq.Filters != nil && mq.Filters.Items != nil {
-		for _, item := range mq.Filters.Items {
-			if item.Key.Key == "__value" {
-				switch v := item.Value.(type) {
-				case float64:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: v,
-					}
-				case float32:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case int:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case int8:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case int16:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case int32:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case int64:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case uint:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case uint8:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case uint16:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case uint32:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				case uint64:
-					metricValueFilter = &v3.MetricValueFilter{
-						Value: float64(v),
-					}
-				}
-			}
-		}
-	}
-	return metricValueFilter
-}
-
 // FormattedValue formats the value to be used in clickhouse query
 func FormattedValue(v interface{}) string {
 	switch x := v.(type) {

pkg/query-service/app/metrics/v3/query_builder.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 	"time"
 
-	"go.signoz.io/signoz/pkg/query-service/app/metrics"
 	"go.signoz.io/signoz/pkg/query-service/app/metrics/v4/helpers"
 	"go.signoz.io/signoz/pkg/query-service/common"
 	"go.signoz.io/signoz/pkg/query-service/constants"
@@ -336,10 +335,6 @@ func PrepareMetricQuery(start, end int64, queryType v3.QueryType, panelType v3.P
 
 	start, end = common.AdjustedMetricTimeRange(start, end, mq.StepInterval, *mq)
 
-	if valFilter := metrics.AddMetricValueFilter(mq); valFilter != nil {
-		mq.MetricValueFilter = valFilter
-	}
-
 	// if the aggregate operator is a histogram quantile, and user has not forgotten
 	// the le tag in the group by then add the le tag to the group by
 	if mq.AggregateOperator == v3.AggregateOperatorHistQuant50 ||

pkg/query-service/app/metrics/v4/cumulative/table.go

@@ -20,16 +20,11 @@ func PrepareMetricQueryCumulativeTable(start, end, step int64, mq *v3.BuilderQue
 	orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
 	selectLabels := helpers.GroupByAttributeKeyTags(mq.GroupBy...)
 
-	valueFilter := " WHERE isNaN(per_series_value) = 0"
-	if mq.MetricValueFilter != nil {
-		valueFilter += fmt.Sprintf(" AND per_series_value = %f", mq.MetricValueFilter.Value)
-	}
-
 	queryTmpl :=
 		"SELECT %s," +
 			" %s as value" +
 			" FROM (%s)" +
-			valueFilter +
+			" WHERE isNaN(per_series_value) = 0" +
 			" GROUP BY %s" +
 			" ORDER BY %s"
 

pkg/query-service/app/metrics/v4/cumulative/timeseries.go

@@ -190,16 +190,11 @@ func PrepareMetricQueryCumulativeTimeSeries(start, end, step int64, mq *v3.Build
 	orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
 	selectLabels := helpers.GroupByAttributeKeyTags(mq.GroupBy...)
 
-	valueFilter := " WHERE isNaN(per_series_value) = 0"
-	if mq.MetricValueFilter != nil {
-		valueFilter += fmt.Sprintf(" AND per_series_value = %f", mq.MetricValueFilter.Value)
-	}
-
 	queryTmpl :=
 		"SELECT %s," +
 			" %s as value" +
 			" FROM (%s)" +
-			valueFilter +
+			" WHERE isNaN(per_series_value) = 0" +
 			" GROUP BY %s" +
 			" ORDER BY %s"
 

pkg/query-service/app/metrics/v4/delta/table.go

@@ -25,16 +25,11 @@ func PrepareMetricQueryDeltaTable(start, end, step int64, mq *v3.BuilderQuery) (
 	orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
 	selectLabels := helpers.GroupByAttributeKeyTags(mq.GroupBy...)
 
-	valueFilter := " WHERE isNaN(per_series_value) = 0"
-	if mq.MetricValueFilter != nil {
-		valueFilter += fmt.Sprintf(" AND per_series_value = %f", mq.MetricValueFilter.Value)
-	}
-
 	queryTmpl :=
 		"SELECT %s," +
 			" %s as value" +
 			" FROM (%s)" +
-			valueFilter +
+			" WHERE isNaN(per_series_value) = 0" +
 			" GROUP BY %s" +
 			" ORDER BY %s"
 

pkg/query-service/app/metrics/v4/delta/timeseries.go

@@ -142,16 +142,11 @@ func PrepareMetricQueryDeltaTimeSeries(start, end, step int64, mq *v3.BuilderQue
 	orderBy := helpers.OrderByAttributeKeyTags(mq.OrderBy, mq.GroupBy)
 	selectLabels := helpers.GroupByAttributeKeyTags(mq.GroupBy...)
 
-	valueFilter := " WHERE isNaN(per_series_value) = 0"
-	if mq.MetricValueFilter != nil {
-		valueFilter += fmt.Sprintf(" AND per_series_value = %f", mq.MetricValueFilter.Value)
-	}
-
 	queryTmpl :=
 		"SELECT %s," +
 			" %s as value" +
 			" FROM (%s)" +
-			valueFilter +
+			" WHERE isNaN(per_series_value) = 0" +
 			" GROUP BY %s" +
 			" ORDER BY %s"
 

pkg/query-service/app/metrics/v4/helpers/sub_query.go

@@ -270,10 +270,6 @@ func PrepareTimeseriesFilterQuery(start, end int64, mq *v3.BuilderQuery) (string
 
 	if fs != nil && len(fs.Items) != 0 {
 		for _, item := range fs.Items {
-			if item.Key.Key == "__value" {
-				continue
-			}
-
 			toFormat := item.Value
 			op := v3.FilterOperator(strings.ToLower(strings.TrimSpace(string(item.Operator))))
 			if op == v3.FilterOperatorContains || op == v3.FilterOperatorNotContains {

pkg/query-service/app/metrics/v4/query_builder.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"time"
 
-	"go.signoz.io/signoz/pkg/query-service/app/metrics"
 	metricsV3 "go.signoz.io/signoz/pkg/query-service/app/metrics/v3"
 	"go.signoz.io/signoz/pkg/query-service/app/metrics/v4/cumulative"
 	"go.signoz.io/signoz/pkg/query-service/app/metrics/v4/delta"
@@ -20,9 +19,6 @@ import (
 // step is in seconds
 func PrepareMetricQuery(start, end int64, queryType v3.QueryType, panelType v3.PanelType, mq *v3.BuilderQuery, options metricsV3.Options) (string, error) {
 
-	if valFilter := metrics.AddMetricValueFilter(mq); valFilter != nil {
-		mq.MetricValueFilter = valFilter
-	}
 	start, end = common.AdjustedMetricTimeRange(start, end, mq.StepInterval, *mq)
 
 	var quantile float64

pkg/query-service/app/querier/helper.go

@@ -190,7 +190,7 @@ func (q *querier) runBuilderQuery(
 				ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil}
 				return
 			}
-			query = strings.Replace(placeholderQuery, "#LIMIT_PLACEHOLDER", limitQuery, 1)
+			query = fmt.Sprintf(placeholderQuery, limitQuery)
 		} else {
 			query, err = tracesQueryBuilder(
 				start,

pkg/query-service/app/querier/v2/helper.go

@@ -190,7 +190,7 @@ func (q *querier) runBuilderQuery(
 				ch <- channelResult{Err: err, Name: queryName, Query: limitQuery, Series: nil}
 				return
 			}
-			query = strings.Replace(placeholderQuery, "#LIMIT_PLACEHOLDER", limitQuery, 1)
+			query = fmt.Sprintf(placeholderQuery, limitQuery)
 		} else {
 			query, err = tracesQueryBuilder(
 				start,

pkg/query-service/app/traces/v4/enrich.go

@@ -34,8 +34,8 @@ func enrichKeyWithMetadata(key v3.AttributeKey, keys map[string]v3.AttributeKey)
 		return v
 	}
 
-	for _, tkey := range utils.GenerateEnrichmentKeys(key) {
-		if val, ok := keys[tkey]; ok {
+	for _, key := range utils.GenerateEnrichmentKeys(key) {
+		if val, ok := keys[key]; ok {
 			return val
 		}
 	}

pkg/query-service/app/traces/v4/query_builder.go

@@ -74,19 +74,6 @@ func getSelectLabels(groupBy []v3.AttributeKey) string {
 	return strings.Join(labels, ",")
 }
 
-// TODO(nitya): use the _exists columns as well in the future similar to logs
-func existsSubQueryForFixedColumn(key v3.AttributeKey, op v3.FilterOperator) (string, error) {
-	if key.DataType == v3.AttributeKeyDataTypeString {
-		if op == v3.FilterOperatorExists {
-			return fmt.Sprintf("%s %s ''", getColumnName(key), tracesOperatorMappingV3[v3.FilterOperatorNotEqual]), nil
-		} else {
-			return fmt.Sprintf("%s %s ''", getColumnName(key), tracesOperatorMappingV3[v3.FilterOperatorEqual]), nil
-		}
-	} else {
-		return "", fmt.Errorf("unsupported operation, exists and not exists can only be applied on custom attributes or string type columns")
-	}
-}
-
 func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) {
 	var conditions []string
 
@@ -123,7 +110,7 @@ func buildTracesFilterQuery(fs *v3.FilterSet) (string, error) {
 					conditions = append(conditions, fmt.Sprintf(operator, columnName, fmtVal))
 				case v3.FilterOperatorExists, v3.FilterOperatorNotExists:
 					if item.Key.IsColumn {
-						subQuery, err := existsSubQueryForFixedColumn(item.Key, item.Operator)
+						subQuery, err := tracesV3.ExistsSubQueryForFixedColumn(item.Key, item.Operator)
 						if err != nil {
 							return "", err
 						}
@@ -325,7 +312,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
 	}
 
 	if options.GraphLimitQtype == constants.SecondQueryGraphLimit {
-		filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "#LIMIT_PLACEHOLDER)"
+		filterSubQuery = filterSubQuery + " AND " + fmt.Sprintf("(%s) GLOBAL IN (", tracesV3.GetSelectKeys(mq.AggregateOperator, mq.GroupBy)) + "%s)"
 	}
 
 	switch mq.AggregateOperator {
@@ -363,7 +350,7 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, panelType v3.
 	case v3.AggregateOperatorCount:
 		if mq.AggregateAttribute.Key != "" {
 			if mq.AggregateAttribute.IsColumn {
-				subQuery, err := existsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists)
+				subQuery, err := tracesV3.ExistsSubQueryForFixedColumn(mq.AggregateAttribute, v3.FilterOperatorExists)
 				if err == nil {
 					filterSubQuery = fmt.Sprintf("%s AND %s", filterSubQuery, subQuery)
 				}

pkg/query-service/app/traces/v4/query_builder_test.go

@@ -265,11 +265,9 @@ func Test_buildTracesFilterQuery(t *testing.T) {
 					{Key: v3.AttributeKey{Key: "isDone", DataType: v3.AttributeKeyDataTypeBool, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorNotExists},
 					{Key: v3.AttributeKey{Key: "host1", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag}, Operator: v3.FilterOperatorNotExists},
 					{Key: v3.AttributeKey{Key: "path", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists},
-					{Key: v3.AttributeKey{Key: "http_url", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists},
-					{Key: v3.AttributeKey{Key: "http.route", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag, IsColumn: true}, Operator: v3.FilterOperatorNotExists},
 				}},
 			},
-			want: "mapContains(attributes_string, 'host') AND mapContains(attributes_number, 'duration') AND NOT mapContains(attributes_bool, 'isDone') AND NOT mapContains(attributes_string, 'host1') AND `attribute_string_path` = '' AND http_url = '' AND `attribute_string_http$$route` = ''",
+			want: "mapContains(attributes_string, 'host') AND mapContains(attributes_number, 'duration') AND NOT mapContains(attributes_bool, 'isDone') AND NOT mapContains(attributes_string, 'host1') AND path = ''",
 		},
 	}
 	for _, tt := range tests {
@@ -685,7 +683,7 @@ func TestPrepareTracesQuery(t *testing.T) {
 				},
 			},
 			want: "SELECT  attributes_string['function'] as `function`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 where " +
-				"(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND mapContains(attributes_string, 'function') AND (`function`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `function` order by value DESC",
+				"(timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND mapContains(attributes_string, 'function') AND (`function`) GLOBAL IN (%s) group by `function` order by value DESC",
 		},
 		{
 			name: "test with limit with resources- first",
@@ -768,7 +766,7 @@ func TestPrepareTracesQuery(t *testing.T) {
 			want: "SELECT  `attribute_string_function` as `function`, serviceName as `serviceName`, toFloat64(count(distinct(name))) as value from signoz_traces.distributed_signoz_index_v3 " +
 				"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000') AND (ts_bucket_start >= 1680064560 AND ts_bucket_start <= 1680066458) AND attributes_number['line'] = 100 " +
 				"AND (resource_fingerprint GLOBAL IN (SELECT fingerprint FROM signoz_traces.distributed_traces_v3_resource WHERE (seen_at_ts_bucket_start >= 1680064560) AND (seen_at_ts_bucket_start <= 1680066458) " +
-				"AND simpleJSONExtractString(labels, 'hostname') = 'server1' AND labels like '%hostname%server1%')) AND (`function`,`serviceName`) GLOBAL IN (#LIMIT_PLACEHOLDER) group by `function`,`serviceName` order by value DESC",
+				"AND simpleJSONExtractString(labels, 'hostname') = 'server1' AND labels like '%hostname%server1%')) AND (`function`,`serviceName`) GLOBAL IN (%s) group by `function`,`serviceName` order by value DESC",
 		},
 	}
 

pkg/query-service/constants/constants.go

@@ -290,7 +290,7 @@ const (
 	UINT8                 = "Uint8"
 )
 
-var StaticSelectedLogFields = []model.Field{
+var StaticSelectedLogFields = []model.LogField{
 	{
 		Name:     "timestamp",
 		DataType: UINT32,

pkg/query-service/interfaces/interface.go

@@ -109,10 +109,6 @@ type Reader interface {
 	SubscribeToQueryProgress(queryId string) (<-chan model.QueryProgress, func(), *model.ApiError)
 
 	GetCountOfThings(ctx context.Context, query string) (uint64, error)
-
-	//trace
-	GetTraceFields(ctx context.Context) (*model.GetFieldsResponse, *model.ApiError)
-	UpdateTraceField(ctx context.Context, field *model.UpdateField) *model.ApiError
 }
 
 type Querier interface {

pkg/query-service/model/infra.go

@@ -151,20 +151,13 @@ type NodeListResponse struct {
 	Total   int              `json:"total"`
 }
 
-type NodeCountByCondition struct {
-	Ready    int `json:"ready"`
-	NotReady int `json:"notReady"`
-	Unknown  int `json:"unknown"`
-}
-
 type NodeListRecord struct {
-	NodeUID               string               `json:"nodeUID,omitempty"`
-	NodeCPUUsage          float64              `json:"nodeCPUUsage"`
-	NodeCPUAllocatable    float64              `json:"nodeCPUAllocatable"`
-	NodeMemoryUsage       float64              `json:"nodeMemoryUsage"`
-	NodeMemoryAllocatable float64              `json:"nodeMemoryAllocatable"`
-	CountByCondition      NodeCountByCondition `json:"countByCondition"`
-	Meta                  map[string]string    `json:"meta"`
+	NodeUID               string            `json:"nodeUID,omitempty"`
+	NodeCPUUsage          float64           `json:"nodeCPUUsage"`
+	NodeCPUAllocatable    float64           `json:"nodeCPUAllocatable"`
+	NodeMemoryUsage       float64           `json:"nodeMemoryUsage"`
+	NodeMemoryAllocatable float64           `json:"nodeMemoryAllocatable"`
+	Meta                  map[string]string `json:"meta"`
 }
 
 type NamespaceListRequest struct {
@@ -187,7 +180,6 @@ type NamespaceListRecord struct {
 	NamespaceName string            `json:"namespaceName"`
 	CPUUsage      float64           `json:"cpuUsage"`
 	MemoryUsage   float64           `json:"memoryUsage"`
-	CountByPhase  PodCountByPhase   `json:"countByPhase"`
 	Meta          map[string]string `json:"meta"`
 }
 
@@ -337,30 +329,3 @@ type JobListRecord struct {
 	SuccessfulPods        int               `json:"successfulPods"`
 	Meta                  map[string]string `json:"meta"`
 }
-
-type VolumeListRequest struct {
-	Start   int64             `json:"start"` // epoch time in ms
-	End     int64             `json:"end"`   // epoch time in ms
-	Filters *v3.FilterSet     `json:"filters"`
-	GroupBy []v3.AttributeKey `json:"groupBy"`
-	OrderBy *v3.OrderBy       `json:"orderBy"`
-	Offset  int               `json:"offset"`
-	Limit   int               `json:"limit"`
-}
-
-type VolumeListResponse struct {
-	Type    ResponseType       `json:"type"`
-	Records []VolumeListRecord `json:"records"`
-	Total   int                `json:"total"`
-}
-
-type VolumeListRecord struct {
-	PersistentVolumeClaimName string            `json:"persistentVolumeClaimName"`
-	VolumeAvailable           float64           `json:"volumeAvailable"`
-	VolumeCapacity            float64           `json:"volumeCapacity"`
-	VolumeInodes              float64           `json:"volumeInodes"`
-	VolumeInodesFree          float64           `json:"volumeInodesFree"`
-	VolumeInodesUsed          float64           `json:"volumeInodesUsed"`
-	VolumeUsage               float64           `json:"volumeUsage"`
-	Meta                      map[string]string `json:"meta"`
-}

pkg/query-service/model/response.go

@@ -509,15 +509,15 @@ type ShowCreateTableStatement struct {
 	Statement string `json:"statement" ch:"statement"`
 }
 
-type Field struct {
+type LogField struct {
 	Name     string `json:"name" ch:"name"`
 	DataType string `json:"dataType" ch:"datatype"`
 	Type     string `json:"type"`
 }
 
 type GetFieldsResponse struct {
-	Selected    []Field `json:"selected"`
-	Interesting []Field `json:"interesting"`
+	Selected    []LogField `json:"selected"`
+	Interesting []LogField `json:"interesting"`
 }
 
 // Represents a log record in query service requests and responses.

pkg/query-service/model/v3/v3.go

@@ -770,19 +770,6 @@ type MetricTableHints struct {
 	SamplesTableName    string
 }
 
-type MetricValueFilter struct {
-	Value float64
-}
-
-func (m *MetricValueFilter) Clone() *MetricValueFilter {
-	if m == nil {
-		return nil
-	}
-	return &MetricValueFilter{
-		Value: m.Value,
-	}
-}
-
 type BuilderQuery struct {
 	QueryName            string            `json:"queryName"`
 	StepInterval         int64             `json:"stepInterval"`
@@ -808,8 +795,7 @@ type BuilderQuery struct {
 	ShiftBy              int64
 	IsAnomaly            bool
 	QueriesUsedInFormula []string
-	MetricTableHints     *MetricTableHints  `json:"-"`
-	MetricValueFilter    *MetricValueFilter `json:"-"`
+	MetricTableHints     *MetricTableHints `json:"-"`
 }
 
 func (b *BuilderQuery) SetShiftByFromFunc() {
@@ -873,7 +859,6 @@ func (b *BuilderQuery) Clone() *BuilderQuery {
 		ShiftBy:              b.ShiftBy,
 		IsAnomaly:            b.IsAnomaly,
 		QueriesUsedInFormula: b.QueriesUsedInFormula,
-		MetricValueFilter:    b.MetricValueFilter.Clone(),
 	}
 }
 

pkg/query-service/tests/integration/logparsingpipeline_test.go

@@ -350,27 +350,6 @@ func TestLogPipelinesValidation(t *testing.T) {
 				},
 			},
 			ExpectedResponseStatusCode: 400,
-		}, {
-			Name: "Invalid from field path",
-			Pipeline: logparsingpipeline.PostablePipeline{
-				OrderId: 1,
-				Name:    "pipeline 1",
-				Alias:   "pipeline1",
-				Enabled: true,
-				Filter:  validPipelineFilterSet,
-				Config: []logparsingpipeline.PipelineOperator{
-					{
-						OrderId: 1,
-						ID:      "move",
-						Type:    "move",
-						From:    `attributes.temp_parsed_body."@l"`,
-						To:      "attributes.test",
-						Enabled: true,
-						Name:    "test move",
-					},
-				},
-			},
-			ExpectedResponseStatusCode: 400,
 		},
 	}
 

sample-apps/hotrod/README.md

@@ -5,7 +5,7 @@ Follow the steps in this section to install a sample application named HotR.O.D,
 ```console
 kubectl create ns sample-application
 
-kubectl -n sample-application apply -f https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod.yaml
+kubectl -n sample-application apply -f https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod.yaml
 ```
 
 In case, you have installed SigNoz in namespace other than `platform` or selected Helm release name other than `my-release`, follow the steps below:
@@ -15,7 +15,7 @@ export HELM_RELEASE=my-release-2
 export SIGNOZ_NAMESPACE=platform-2
 export HOTROD_NAMESPACE=sample-application-2
 
-curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-install.sh | bash
+curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-install.sh | bash
 ```
 
 To delete sample application:
@@ -23,7 +23,7 @@ To delete sample application:
 ```console
 export HOTROD_NAMESPACE=sample-application-2
 
-curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-delete.sh | bash
+curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-delete.sh | bash
 ```
 
 For testing with local scripts, you can use the following commands:

sample-apps/hotrod/hotrod-delete.sh

@@ -7,7 +7,7 @@ HOTROD_NAMESPACE=${HOTROD_NAMESPACE:-"sample-application"}
 if [[ "${HOTROD_NAMESPACE}" == "default" || "${HOTROD_NAMESPACE}" == "kube-system" || "${HOTROD_NAMESPACE}" == "platform" ]]; then
     echo "Default k8s namespace and SigNoz namespace must not be deleted"
     echo "Deleting components only"
-    kubectl delete --namespace="${HOTROD_NAMESPACE}" -f <(cat hotrod-template.yaml || curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-template.yaml)
+    kubectl delete --namespace="${HOTROD_NAMESPACE}" -f <(cat hotrod-template.yaml || curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-template.yaml)
 else
     echo "Delete HotROD sample app namespace ${HOTROD_NAMESPACE}"
     kubectl delete namespace "${HOTROD_NAMESPACE}"

sample-apps/hotrod/hotrod-install.sh

@@ -37,7 +37,7 @@ kubectl create namespace "$HOTROD_NAMESPACE" --save-config --dry-run -o yaml 2>/
 
 # Setup sample apps into specified namespace
 kubectl apply --namespace="${HOTROD_NAMESPACE}" -f <( \
-    (cat hotrod-template.yaml 2>/dev/null || curl -sL https://github.com/SigNoz/signoz/raw/main/sample-apps/hotrod/hotrod-template.yaml) | \
+    (cat hotrod-template.yaml 2>/dev/null || curl -sL https://github.com/SigNoz/signoz/raw/develop/sample-apps/hotrod/hotrod-template.yaml) | \
         HOTROD_NAMESPACE="${HOTROD_NAMESPACE}" \
         HOTROD_IMAGE="${HOTROD_IMAGE}" \
         LOCUST_IMAGE="${LOCUST_IMAGE}" \