chore(signoz): pin versions: SigNoz 0.63.0, SigNoz OtelCollector 0.111.16

Signed-off-by: Prashant Shahi <prashant@signoz.io>

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -146,7 +146,7 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.61.0
+    image: signoz/query-service:0.63.0
     command:
       [
         "-config=/root/config/prometheus.yml",
@@ -186,7 +186,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.61.0
+    image: signoz/frontend:0.63.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -199,7 +199,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -237,7 +237,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.111.14
+      image: signoz/signoz-schema-migrator:0.111.16
       deploy:
         restart_policy:
           condition: on-failure

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -69,7 +69,7 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator
     command:
       - "sync" 
@@ -86,7 +86,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.111.14
+    image: signoz/signoz-otel-collector:0.111.16
     command:
       [
         "--config=/etc/otel-collector-config.yaml",

deploy/docker/clickhouse-setup/docker-compose-minimal.yaml

@@ -162,7 +162,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.63.0}
     container_name: signoz-query-service
     command:
       [
@@ -201,7 +201,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.63.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -213,7 +213,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator-sync:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator-sync
     command:
       - "sync"
@@ -228,7 +228,7 @@ services:
       #   condition: service_healthy
 
   otel-collector-migrator-async:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator-async
     command:
       - "async"
@@ -245,7 +245,7 @@ services:
       #   condition: service_healthy
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/docker-compose.testing.yaml

@@ -167,7 +167,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.61.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.63.0}
     container_name: signoz-query-service
     command:
       [
@@ -208,7 +208,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.61.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.63.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -220,7 +220,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.111.16}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -234,7 +234,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.14}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.111.16}
     container_name: signoz-otel-collector
     command:
       [

frontend/src/container/NewDashboard/DashboardVariablesSelection/DashboardVariableSelection.tsx

@@ -1,15 +1,9 @@
 import { Row } from 'antd';
+import { isNull } from 'lodash-es';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
-import { memo, useEffect, useRef, useState } from 'react';
+import { memo, useEffect, useState } from 'react';
 import { IDashboardVariable } from 'types/api/dashboard/getAll';
 
-import {
-	buildDependencies,
-	buildDependencyGraph,
-	buildParentDependencyGraph,
-	onUpdateVariableNode,
-	VariableGraph,
-} from './util';
 import VariableItem from './VariableItem';
 
 function DashboardVariableSelection(): JSX.Element | null {
@@ -27,12 +21,6 @@ function DashboardVariableSelection(): JSX.Element | null {
 
 	const [variablesTableData, setVariablesTableData] = useState<any>([]);
 
-	const [dependencyData, setDependencyData] = useState<{
-		order: string[];
-		graph: VariableGraph;
-		parentDependencyGraph: VariableGraph;
-	} | null>(null);
-
 	useEffect(() => {
 		if (variables) {
 			const tableRowData = [];
@@ -55,28 +43,35 @@ function DashboardVariableSelection(): JSX.Element | null {
 		}
 	}, [variables]);
 
-	const initializationRef = useRef(false);
-
-	useEffect(() => {
-		if (variablesTableData.length > 0 && !initializationRef.current) {
-			const depGrp = buildDependencies(variablesTableData);
-			const { order, graph } = buildDependencyGraph(depGrp);
-			const parentDependencyGraph = buildParentDependencyGraph(graph);
-			setDependencyData({
-				order,
-				graph,
-				parentDependencyGraph,
-			});
-			initializationRef.current = true;
-		}
-	}, [variablesTableData]);
+	const onVarChanged = (name: string): void => {
+		/**
+		 * this function takes care of adding the dependent variables to current update queue and removing
+		 * the updated variable name from the queue
+		 */
+		const dependentVariables = variablesTableData
+			?.map((variable: any) => {
+				if (variable.type === 'QUERY') {
+					const re = new RegExp(`\\{\\{\\s*?\\.${name}\\s*?\\}\\}`); // regex for `{{.var}}`
+					const queryValue = variable.queryValue || '';
+					const dependVarReMatch = queryValue.match(re);
+					if (dependVarReMatch !== null && dependVarReMatch.length > 0) {
+						return variable.name;
+					}
+				}
+				return null;
+			})
+			.filter((val: string | null) => !isNull(val));
+		setVariablesToGetUpdated((prev) => [
+			...prev.filter((v) => v !== name),
+			...dependentVariables,
+		]);
+	};
 
 	const onValueUpdate = (
 		name: string,
 		id: string,
 		value: IDashboardVariable['selectedValue'],
 		allSelected: boolean,
-		isMountedCall?: boolean,
 		// eslint-disable-next-line sonarjs/cognitive-complexity
 	): void => {
 		if (id) {
@@ -116,18 +111,7 @@ function DashboardVariableSelection(): JSX.Element | null {
 				});
 			}
 
-			if (dependencyData && !isMountedCall) {
-				const updatedVariables: string[] = [];
-				onUpdateVariableNode(
-					name,
-					dependencyData.graph,
-					dependencyData.order,
-					(node) => updatedVariables.push(node),
-				);
-				setVariablesToGetUpdated(updatedVariables.filter((v) => v !== name));
-			} else if (isMountedCall) {
-				setVariablesToGetUpdated((prev) => prev.filter((v) => v !== name));
-			}
+			onVarChanged(name);
 		}
 	};
 
@@ -155,7 +139,6 @@ function DashboardVariableSelection(): JSX.Element | null {
 						onValueUpdate={onValueUpdate}
 						variablesToGetUpdated={variablesToGetUpdated}
 						setVariablesToGetUpdated={setVariablesToGetUpdated}
-						dependencyData={dependencyData}
 					/>
 				))}
 		</Row>

frontend/src/container/NewDashboard/DashboardVariablesSelection/VariableItem.tsx

@@ -24,7 +24,7 @@ import { commaValuesParser } from 'lib/dashbaordVariables/customCommaValuesParse
 import sortValues from 'lib/dashbaordVariables/sortVariableValues';
 import { debounce, isArray, isString } from 'lodash-es';
 import map from 'lodash-es/map';
-import { ChangeEvent, memo, useEffect, useMemo, useRef, useState } from 'react';
+import { ChangeEvent, memo, useEffect, useMemo, useState } from 'react';
 import { useQuery } from 'react-query';
 import { useSelector } from 'react-redux';
 import { AppState } from 'store/reducers';
@@ -35,15 +35,12 @@ import { popupContainer } from 'utils/selectPopupContainer';
 
 import { variablePropsToPayloadVariables } from '../utils';
 import { SelectItemStyle } from './styles';
-import {
-	areArraysEqual,
-	checkAPIInvocation,
-	onUpdateVariableNode,
-	VariableGraph,
-} from './util';
+import { areArraysEqual } from './util';
 
 const ALL_SELECT_VALUE = '__ALL__';
 
+const variableRegexPattern = /\{\{\s*?\.([^\s}]+)\s*?\}\}/g;
+
 enum ToggleTagValue {
 	Only = 'Only',
 	All = 'All',
@@ -57,15 +54,9 @@ interface VariableItemProps {
 		id: string,
 		arg1: IDashboardVariable['selectedValue'],
 		allSelected: boolean,
-		isMountedCall?: boolean,
 	) => void;
 	variablesToGetUpdated: string[];
 	setVariablesToGetUpdated: React.Dispatch<React.SetStateAction<string[]>>;
-	dependencyData: {
-		order: string[];
-		graph: VariableGraph;
-		parentDependencyGraph: VariableGraph;
-	} | null;
 }
 
 const getSelectValue = (
@@ -88,7 +79,6 @@ function VariableItem({
 	onValueUpdate,
 	variablesToGetUpdated,
 	setVariablesToGetUpdated,
-	dependencyData,
 }: VariableItemProps): JSX.Element {
 	const [optionsData, setOptionsData] = useState<(string | number | boolean)[]>(
 		[],
@@ -98,29 +88,60 @@ function VariableItem({
 		(state) => state.globalTime,
 	);
 
-	// logic to detect if its a rerender or a new render/mount
-	const isMounted = useRef(false);
-
 	useEffect(() => {
-		isMounted.current = true;
-	}, []);
-
-	const validVariableUpdate = (): boolean => {
-		if (!variableData.name) {
-			return false;
+		if (variableData.allSelected && variableData.type === 'QUERY') {
+			setVariablesToGetUpdated((prev) => {
+				const variablesQueue = [...prev.filter((v) => v !== variableData.name)];
+				if (variableData.name) {
+					variablesQueue.push(variableData.name);
+				}
+				return variablesQueue;
+			});
 		}
-		if (!isMounted.current) {
-			// variableData.name is present as the top element or next in the queue - variablesToGetUpdated
-			return Boolean(
-				variablesToGetUpdated.length &&
-					variablesToGetUpdated[0] === variableData.name,
-			);
-		}
-		return variablesToGetUpdated.includes(variableData.name);
-	};
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, [minTime, maxTime]);
 
 	const [errorMessage, setErrorMessage] = useState<null | string>(null);
 
+	const getDependentVariables = (queryValue: string): string[] => {
+		const matches = queryValue.match(variableRegexPattern);
+
+		// Extract variable names from the matches array without {{ . }}
+		return matches
+			? matches.map((match) => match.replace(variableRegexPattern, '$1'))
+			: [];
+	};
+
+	const getQueryKey = (variableData: IDashboardVariable): string[] => {
+		let dependentVariablesStr = '';
+
+		const dependentVariables = getDependentVariables(
+			variableData.queryValue || '',
+		);
+
+		const variableName = variableData.name || '';
+
+		dependentVariables?.forEach((element) => {
+			const [, variable] =
+				Object.entries(existingVariables).find(
+					([, value]) => value.name === element,
+				) || [];
+
+			dependentVariablesStr += `${element}${variable?.selectedValue}`;
+		});
+
+		const variableKey = dependentVariablesStr.replace(/\s/g, '');
+
+		// added this time dependency for variables query as API respects the passed time range now
+		return [
+			REACT_QUERY_KEY.DASHBOARD_BY_ID,
+			variableName,
+			variableKey,
+			`${minTime}`,
+			`${maxTime}`,
+		];
+	};
+
 	// eslint-disable-next-line sonarjs/cognitive-complexity
 	const getOptions = (variablesRes: VariableResponseProps | null): void => {
 		if (variablesRes && variableData.type === 'QUERY') {
@@ -163,7 +184,9 @@ function VariableItem({
 						if (
 							variableData.type === 'QUERY' &&
 							variableData.name &&
-							(validVariableUpdate() || valueNotInList || variableData.allSelected)
+							(variablesToGetUpdated.includes(variableData.name) ||
+								valueNotInList ||
+								variableData.allSelected)
 						) {
 							let value = variableData.selectedValue;
 							let allSelected = false;
@@ -177,16 +200,7 @@ function VariableItem({
 							}
 
 							if (variableData && variableData?.name && variableData?.id) {
-								onValueUpdate(
-									variableData.name,
-									variableData.id,
-									value,
-									allSelected,
-									isMounted.current,
-								);
-								setVariablesToGetUpdated((prev) =>
-									prev.filter((name) => name !== variableData.name),
-								);
+								onValueUpdate(variableData.name, variableData.id, value, allSelected);
 							}
 						}
 
@@ -210,75 +224,36 @@ function VariableItem({
 		}
 	};
 
-	const { isLoading } = useQuery(
-		[
-			REACT_QUERY_KEY.DASHBOARD_BY_ID,
-			variableData.name || '',
-			`${minTime}`,
-			`${maxTime}`,
-		],
-		{
-			enabled:
-				variableData &&
-				variableData.type === 'QUERY' &&
-				checkAPIInvocation(
-					variablesToGetUpdated,
-					variableData,
-					dependencyData?.parentDependencyGraph,
-				),
-			queryFn: () =>
-				dashboardVariablesQuery({
-					query: variableData.queryValue || '',
-					variables: variablePropsToPayloadVariables(existingVariables),
-				}),
-			refetchOnWindowFocus: false,
-			onSuccess: (response) => {
-				getOptions(response.payload);
-				if (
-					dependencyData?.parentDependencyGraph[variableData.name || ''].length === 0
-				) {
-					const updatedVariables: string[] = [];
-					onUpdateVariableNode(
-						variableData.name || '',
-						dependencyData.graph,
-						dependencyData.order,
-						(node) => updatedVariables.push(node),
-					);
-					setVariablesToGetUpdated((prev) => [
-						...prev,
-						...updatedVariables.filter((v) => v !== variableData.name),
-					]);
-				}
-			},
-			onError: (error: {
-				details: {
-					error: string;
-				};
-			}) => {
-				const { details } = error;
-
-				if (details.error) {
-					let message = details.error;
-					if (details.error.includes('Syntax error:')) {
-						message =
-							'Please make sure query is valid and dependent variables are selected';
-					}
-					setErrorMessage(message);
-				}
-			},
+	const { isLoading } = useQuery(getQueryKey(variableData), {
+		enabled: variableData && variableData.type === 'QUERY',
+		queryFn: () =>
+			dashboardVariablesQuery({
+				query: variableData.queryValue || '',
+				variables: variablePropsToPayloadVariables(existingVariables),
+			}),
+		refetchOnWindowFocus: false,
+		onSuccess: (response) => {
+			getOptions(response.payload);
 		},
-	);
+		onError: (error: {
+			details: {
+				error: string;
+			};
+		}) => {
+			const { details } = error;
+
+			if (details.error) {
+				let message = details.error;
+				if (details.error.includes('Syntax error:')) {
+					message =
+						'Please make sure query is valid and dependent variables are selected';
+				}
+				setErrorMessage(message);
+			}
+		},
+	});
 
 	const handleChange = (value: string | string[]): void => {
-		// if value is equal to selected value then return
-		if (
-			value === variableData.selectedValue ||
-			(Array.isArray(value) &&
-				Array.isArray(variableData.selectedValue) &&
-				areArraysEqual(value, variableData.selectedValue))
-		) {
-			return;
-		}
 		if (variableData.name) {
 			if (
 				value === ALL_SELECT_VALUE ||

frontend/src/container/NewDashboard/DashboardVariablesSelection/util.ts

@@ -1,4 +1,3 @@
-import { isEmpty } from 'lodash-es';
 import { Dashboard, IDashboardVariable } from 'types/api/dashboard/getAll';
 
 export function areArraysEqual(
@@ -30,158 +29,3 @@ export const convertVariablesToDbFormat = (
 		result[id] = obj;
 		return result;
 	}, {});
-
-const getDependentVariables = (queryValue: string): string[] => {
-	const variableRegexPattern = /\{\{\s*?\.([^\s}]+)\s*?\}\}/g;
-
-	const matches = queryValue.match(variableRegexPattern);
-
-	// Extract variable names from the matches array without {{ . }}
-	return matches
-		? matches.map((match) => match.replace(variableRegexPattern, '$1'))
-		: [];
-};
-export type VariableGraph = Record<string, string[]>;
-
-export const buildDependencies = (
-	variables: IDashboardVariable[],
-): VariableGraph => {
-	const graph: VariableGraph = {};
-
-	// Initialize empty arrays for all variables first
-	variables.forEach((variable) => {
-		if (variable.name) {
-			graph[variable.name] = [];
-		}
-	});
-
-	// For each QUERY variable, add it as a dependent to its referenced variables
-	variables.forEach((variable) => {
-		if (variable.type === 'QUERY' && variable.name) {
-			const dependentVariables = getDependentVariables(variable.queryValue || '');
-
-			// For each referenced variable, add the current query as a dependent
-			dependentVariables.forEach((referencedVar) => {
-				if (graph[referencedVar]) {
-					graph[referencedVar].push(variable.name as string);
-				} else {
-					graph[referencedVar] = [variable.name as string];
-				}
-			});
-		}
-	});
-
-	return graph;
-};
-
-// Function to build the dependency graph
-export const buildDependencyGraph = (
-	dependencies: VariableGraph,
-): { order: string[]; graph: VariableGraph } => {
-	const inDegree: Record<string, number> = {};
-	const adjList: VariableGraph = {};
-
-	// Initialize in-degree and adjacency list
-	Object.keys(dependencies).forEach((node) => {
-		if (!inDegree[node]) inDegree[node] = 0;
-		if (!adjList[node]) adjList[node] = [];
-		dependencies[node].forEach((child) => {
-			if (!inDegree[child]) inDegree[child] = 0;
-			inDegree[child]++;
-			adjList[node].push(child);
-		});
-	});
-
-	// Topological sort using Kahn's Algorithm
-	const queue: string[] = Object.keys(inDegree).filter(
-		(node) => inDegree[node] === 0,
-	);
-	const topologicalOrder: string[] = [];
-
-	while (queue.length > 0) {
-		const current = queue.shift();
-		if (current === undefined) {
-			break;
-		}
-		topologicalOrder.push(current);
-
-		adjList[current].forEach((neighbor) => {
-			inDegree[neighbor]--;
-			if (inDegree[neighbor] === 0) queue.push(neighbor);
-		});
-	}
-
-	if (topologicalOrder.length !== Object.keys(dependencies).length) {
-		throw new Error('Cycle detected in the dependency graph!');
-	}
-
-	return { order: topologicalOrder, graph: adjList };
-};
-
-export const onUpdateVariableNode = (
-	nodeToUpdate: string,
-	graph: VariableGraph,
-	topologicalOrder: string[],
-	callback: (node: string) => void,
-): void => {
-	const visited = new Set<string>();
-
-	// Start processing from the node to update
-	topologicalOrder.forEach((node) => {
-		if (node === nodeToUpdate || visited.has(node)) {
-			visited.add(node);
-			callback(node);
-			(graph[node] || []).forEach((child) => {
-				visited.add(child);
-			});
-		}
-	});
-};
-
-export const buildParentDependencyGraph = (
-	graph: VariableGraph,
-): VariableGraph => {
-	const parentGraph: VariableGraph = {};
-
-	// Initialize empty arrays for all nodes
-	Object.keys(graph).forEach((node) => {
-		parentGraph[node] = [];
-	});
-
-	// For each node and its children in the original graph
-	Object.entries(graph).forEach(([node, children]) => {
-		// For each child, add the current node as its parent
-		children.forEach((child) => {
-			parentGraph[child].push(node);
-		});
-	});
-
-	return parentGraph;
-};
-
-export const checkAPIInvocation = (
-	variablesToGetUpdated: string[],
-	variableData: IDashboardVariable,
-	parentDependencyGraph?: VariableGraph,
-): boolean => {
-	if (isEmpty(variableData.name)) {
-		return false;
-	}
-
-	if (isEmpty(parentDependencyGraph)) {
-		return true;
-	}
-
-	// if no dependency then true
-	const haveDependency =
-		parentDependencyGraph?.[variableData.name || '']?.length > 0;
-	if (!haveDependency) {
-		return true;
-	}
-
-	// if variable is in the list and has dependency then check if its the top element in the queue then true else false
-	return (
-		variablesToGetUpdated.length > 0 &&
-		variablesToGetUpdated[0] === variableData.name
-	);
-};

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.25.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.111.14
+	github.com/SigNoz/signoz-otel-collector v0.111.16
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3

go.sum

@@ -70,8 +70,8 @@ github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkb
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
 github.com/SigNoz/prometheus v1.12.0 h1:+BXeIHyMOOWWa+xjhJ+x80JFva7r1WzWIfIhQ5PUmIE=
 github.com/SigNoz/prometheus v1.12.0/go.mod h1:EqNM27OwmPfqMUk+E+XG1L9rfDFcyXnzzDrg0EPOfxA=
-github.com/SigNoz/signoz-otel-collector v0.111.14 h1:nvRucNK/TTtZKM3Dsr/UNx+LwkjaGwx0yPlMvGw/4j0=
-github.com/SigNoz/signoz-otel-collector v0.111.14/go.mod h1:vRDT10om89DHybN7SRMlt8IN9+/pgh1D57pNHPr2LM4=
+github.com/SigNoz/signoz-otel-collector v0.111.16 h1:535uKH5Oux+35EsI+L3C6pnAP/Ye0PTCbVizXoL+VqE=
+github.com/SigNoz/signoz-otel-collector v0.111.16/go.mod h1:HJ4m0LY1MPsuZmuRF7Ixb+bY8rxgRzI0VXzOedESsjg=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=

pkg/query-service/app/http_handler.go

@@ -4075,10 +4075,9 @@ func (aH *APIHandler) CreateLogsPipeline(w http.ResponseWriter, r *http.Request)
 			zap.L().Warn("found no pipelines in the http request, this will delete all the pipelines")
 		}
 
-		for _, p := range postable {
-			if err := p.IsValid(); err != nil {
-				return nil, model.BadRequestStr(err.Error())
-			}
+		validationErr := aH.LogsParsingPipelineController.ValidatePipelines(ctx, postable)
+		if validationErr != nil {
+			return nil, validationErr
 		}
 
 		return aH.LogsParsingPipelineController.ApplyPipelines(ctx, postable)

pkg/query-service/app/logparsingpipeline/controller.go

@@ -94,6 +94,45 @@ func (ic *LogParsingPipelineController) ApplyPipelines(
 	return ic.GetPipelinesByVersion(ctx, cfg.Version)
 }
 
+func (ic *LogParsingPipelineController) ValidatePipelines(
+	ctx context.Context,
+	postedPipelines []PostablePipeline,
+) *model.ApiError {
+	for _, p := range postedPipelines {
+		if err := p.IsValid(); err != nil {
+			return model.BadRequestStr(err.Error())
+		}
+	}
+
+	// Also run a collector simulation to ensure config is fit
+	// for e2e use with a collector
+	pipelines := []Pipeline{}
+	for _, pp := range postedPipelines {
+		pipelines = append(pipelines, Pipeline{
+			Id:          uuid.New().String(),
+			OrderId:     pp.OrderId,
+			Enabled:     pp.Enabled,
+			Name:        pp.Name,
+			Alias:       pp.Alias,
+			Description: &pp.Description,
+			Filter:      pp.Filter,
+			Config:      pp.Config,
+		})
+	}
+
+	sampleLogs := []model.SignozLog{{Body: ""}}
+	_, _, simulationErr := SimulatePipelinesProcessing(
+		ctx, pipelines, sampleLogs,
+	)
+	if simulationErr != nil {
+		return model.BadRequest(fmt.Errorf(
+			"invalid pipelines config: %w", simulationErr.ToError(),
+		))
+	}
+
+	return nil
+}
+
 // Returns effective list of pipelines including user created
 // pipelines and pipelines for installed integrations
 func (ic *LogParsingPipelineController) getEffectivePipelinesByVersion(

pkg/query-service/tests/integration/logparsingpipeline_test.go

@@ -350,6 +350,27 @@ func TestLogPipelinesValidation(t *testing.T) {
 				},
 			},
 			ExpectedResponseStatusCode: 400,
+		}, {
+			Name: "Invalid from field path",
+			Pipeline: logparsingpipeline.PostablePipeline{
+				OrderId: 1,
+				Name:    "pipeline 1",
+				Alias:   "pipeline1",
+				Enabled: true,
+				Filter:  validPipelineFilterSet,
+				Config: []logparsingpipeline.PipelineOperator{
+					{
+						OrderId: 1,
+						ID:      "move",
+						Type:    "move",
+						From:    `attributes.temp_parsed_body."@l"`,
+						To:      "attributes.test",
+						Enabled: true,
+						Name:    "test move",
+					},
+				},
+			},
+			ExpectedResponseStatusCode: 400,
 		},
 	}