Merge remote-tracking branch 'origin/develop' into feat/read-in-order

The following vulnerabilities are fixed with an upgrade:
- https://snyk.io/vuln/SNYK-JS-AXIOS-6144788

Co-authored-by: snyk-bot <snyk-bot@snyk.io>
Co-authored-by: Prashant Shahi <prashant@signoz.io>

.github/workflows/jest-coverage-changes.yml

@@ -0,0 +1,31 @@
+name: Jest Coverage - changed files
+
+on:
+  pull_request:
+    branches: develop
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: "refs/heads/develop"
+          token: ${{ secrets.GITHUB_TOKEN }} # Provide the GitHub token for authentication
+
+      - name: Fetch branch
+        run: git fetch origin ${{ github.event.pull_request.head.ref }}
+
+      - run: |
+          git checkout ${{ github.event.pull_request.head.sha }}
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Install dependencies
+        run: cd frontend && npm install -g yarn && yarn
+
+      - name: npm run test:changedsince
+        run: cd frontend && npm run i18n:generate-hash && npm run test:changedsince

deploy/docker-swarm/clickhouse-setup/docker-compose.yaml

@@ -22,7 +22,7 @@ x-clickhouse-defaults: &clickhouse-defaults
         "wget",
         "--spider",
         "-q",
-        "localhost:8123/ping"
+        "0.0.0.0:8123/ping"
       ]
     interval: 30s
     timeout: 5s
@@ -146,7 +146,7 @@ services:
         condition: on-failure
 
   query-service:
-    image: signoz/query-service:0.43.0
+    image: signoz/query-service:0.44.0
     command:
       [
         "-config=/root/config/prometheus.yml",
@@ -186,7 +186,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:0.43.0
+    image: signoz/frontend:0.44.0
     deploy:
       restart_policy:
         condition: on-failure
@@ -199,7 +199,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.88.20
+    image: signoz/signoz-otel-collector:0.88.21
     command:
       [
         "--config=/etc/otel-collector-config.yaml",
@@ -237,7 +237,7 @@ services:
       - query-service
 
   otel-collector-migrator:
-      image: signoz/signoz-schema-migrator:0.88.20
+      image: signoz/signoz-schema-migrator:0.88.21
       deploy:
         restart_policy:
           condition: on-failure

deploy/docker-swarm/clickhouse-setup/otel-collector-config.yaml

@@ -111,18 +111,18 @@ processors:
 
 exporters:
   clickhousetraces:
-    datasource: tcp://clickhouse:9000/?database=signoz_traces
+    datasource: tcp://clickhouse:9000/signoz_traces
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     low_cardinal_exception_grouping: ${LOW_CARDINAL_EXCEPTION_GROUPING}
   clickhousemetricswrite:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
+    endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
   clickhousemetricswrite/prometheus:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
+    endpoint: tcp://clickhouse:9000/signoz_metrics
   # logging: {}
   clickhouselogsexporter:
-    dsn: tcp://clickhouse:9000/
+    dsn: tcp://clickhouse:9000/signoz_logs
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     timeout: 10s
 extensions:

deploy/docker-swarm/clickhouse-setup/prometheus.yml

@@ -22,4 +22,4 @@ rule_files:
 scrape_configs: []
 
 remote_read:
-  - url: tcp://clickhouse:9000/?database=signoz_metrics
+  - url: tcp://clickhouse:9000/signoz_metrics

deploy/docker/clickhouse-setup/docker-compose-core.yaml

@@ -46,7 +46,7 @@ services:
           "wget",
           "--spider",
           "-q",
-          "localhost:8123/ping"
+          "0.0.0.0:8123/ping"
         ]
       interval: 30s
       timeout: 5s
@@ -66,7 +66,7 @@ services:
       - --storage.path=/data
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.20}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.21}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -81,7 +81,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
   otel-collector:
     container_name: signoz-otel-collector
-    image: signoz/signoz-otel-collector:0.88.20
+    image: signoz/signoz-otel-collector:0.88.21
     command:
       [
         "--config=/etc/otel-collector-config.yaml",

deploy/docker/clickhouse-setup/docker-compose.yaml

@@ -21,7 +21,7 @@ x-clickhouse-defaults: &clickhouse-defaults
         "wget",
         "--spider",
         "-q",
-        "localhost:8123/ping"
+        "0.0.0.0:8123/ping"
       ]
     interval: 30s
     timeout: 5s
@@ -164,7 +164,7 @@ services:
   # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
 
   query-service:
-    image: signoz/query-service:${DOCKER_TAG:-0.43.0}
+    image: signoz/query-service:${DOCKER_TAG:-0.44.0}
     container_name: signoz-query-service
     command:
       [
@@ -203,7 +203,7 @@ services:
     <<: *db-depend
 
   frontend:
-    image: signoz/frontend:${DOCKER_TAG:-0.43.0}
+    image: signoz/frontend:${DOCKER_TAG:-0.44.0}
     container_name: signoz-frontend
     restart: on-failure
     depends_on:
@@ -215,7 +215,7 @@ services:
       - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.20}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.21}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -229,7 +229,7 @@ services:
 
 
   otel-collector:
-    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.20}
+    image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.88.21}
     container_name: signoz-otel-collector
     command:
       [

deploy/docker/clickhouse-setup/otel-collector-config.yaml

@@ -122,21 +122,20 @@ extensions:
 
 exporters:
   clickhousetraces:
-    datasource: tcp://clickhouse:9000/?database=signoz_traces
+    datasource: tcp://clickhouse:9000/signoz_traces
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     low_cardinal_exception_grouping: ${LOW_CARDINAL_EXCEPTION_GROUPING}
   clickhousemetricswrite:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
+    endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
   clickhousemetricswrite/prometheus:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
-  # logging: {}
-
+    endpoint: tcp://clickhouse:9000/signoz_metrics
   clickhouselogsexporter:
-    dsn: tcp://clickhouse:9000/
+    dsn: tcp://clickhouse:9000/signoz_logs
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     timeout: 10s
+  # logging: {}
 
 service:
   telemetry:

deploy/docker/clickhouse-setup/prometheus.yml

@@ -22,4 +22,4 @@ rule_files:
 scrape_configs: []
 
 remote_read:
-  - url: tcp://clickhouse:9000/?database=signoz_metrics
+  - url: tcp://clickhouse:9000/signoz_metrics

frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM nginx:1.25.2-alpine
+FROM nginx:1.26-alpine
 
 # Add Maintainer Info
 LABEL maintainer="signoz"

frontend/jest.config.ts

@@ -35,6 +35,14 @@ const config: Config.InitialOptions = {
 			browsers: ['chromium', 'firefox', 'webkit'],
 		},
 	},
+	coverageThreshold: {
+		global: {
+			statements: 80,
+			branches: 65,
+			functions: 80,
+			lines: 80,
+		},
+	},
 };
 
 export default config;

frontend/package.json

@@ -21,7 +21,9 @@
 		"playwright:codegen:local": "playwright codegen http://localhost:3301",
 		"playwright:codegen:local:auth": "yarn playwright:codegen:local --load-storage=tests/auth.json",
 		"husky:configure": "cd .. && husky install frontend/.husky && cd frontend && chmod ug+x .husky/*",
-		"commitlint": "commitlint --edit $1"
+		"commitlint": "commitlint --edit $1",
+		"test": "jest --coverage",
+		"test:changedsince": "jest --changedSince=develop --coverage --silent"
 	},
 	"engines": {
 		"node": ">=16.15.0"
@@ -51,7 +53,7 @@
 		"ansi-to-html": "0.7.2",
 		"antd": "5.11.0",
 		"antd-table-saveas-excel": "2.2.1",
-		"axios": "1.6.2",
+		"axios": "1.6.4",
 		"babel-eslint": "^10.1.0",
 		"babel-jest": "^29.6.4",
 		"babel-loader": "9.1.3",

frontend/src/components/ResizeTable/DynamicColumnTable.tsx

@@ -1,8 +1,9 @@
 /* eslint-disable react/jsx-props-no-spreading */
 import './DynamicColumnTable.syles.scss';
 
-import { Button, Dropdown, MenuProps, Switch } from 'antd';
+import { Button, Dropdown, Flex, MenuProps, Switch } from 'antd';
 import { ColumnsType } from 'antd/lib/table';
+import FacingIssueBtn from 'components/facingIssueBtn/FacingIssueBtn';
 import { SlidersHorizontal } from 'lucide-react';
 import { memo, useEffect, useState } from 'react';
 import { popupContainer } from 'utils/selectPopupContainer';
@@ -20,6 +21,7 @@ function DynamicColumnTable({
 	columns,
 	dynamicColumns,
 	onDragColumn,
+	facingIssueBtn,
 	...restProps
 }: DynamicColumnTableProps): JSX.Element {
 	const [columnsData, setColumnsData] = useState<ColumnsType | undefined>(
@@ -83,19 +85,22 @@ function DynamicColumnTable({
 
 	return (
 		<div className="DynamicColumnTable">
-			{dynamicColumns && (
-				<Dropdown
-					getPopupContainer={popupContainer}
-					menu={{ items }}
-					trigger={['click']}
-				>
-					<Button
-						className="dynamicColumnTable-button filter-btn"
-						size="middle"
-						icon={<SlidersHorizontal size={14} />}
-					/>
-				</Dropdown>
-			)}
+			<Flex justify="flex-end" align="center" gap={8}>
+				{facingIssueBtn && <FacingIssueBtn {...facingIssueBtn} />}
+				{dynamicColumns && (
+					<Dropdown
+						getPopupContainer={popupContainer}
+						menu={{ items }}
+						trigger={['click']}
+					>
+						<Button
+							className="dynamicColumnTable-button filter-btn"
+							size="middle"
+							icon={<SlidersHorizontal size={14} />}
+						/>
+					</Dropdown>
+				)}
+			</Flex>
 
 			<ResizeTable
 				columns={columnsData}

frontend/src/components/ResizeTable/types.ts

@@ -2,6 +2,7 @@
 import { TableProps } from 'antd';
 import { ColumnsType } from 'antd/es/table';
 import { ColumnGroupType, ColumnType } from 'antd/lib/table';
+import { FacingIssueBtnProps } from 'components/facingIssueBtn/FacingIssueBtn';
 
 import { TableDataSource } from './contants';
 
@@ -12,6 +13,7 @@ export interface DynamicColumnTableProps extends TableProps<any> {
 	tablesource: typeof TableDataSource[keyof typeof TableDataSource];
 	dynamicColumns: TableProps<any>['columns'];
 	onDragColumn?: (fromIndex: number, toIndex: number) => void;
+	facingIssueBtn?: FacingIssueBtnProps;
 }
 
 export type GetVisibleColumnsFunction = (

frontend/src/components/facingIssueBtn/FacingIssueBtn.style.scss

@@ -0,0 +1,9 @@
+.facing-issue-button {
+	color: var(--bg-amber-500);
+	border-color: var(--bg-amber-500);
+
+	.ant-btn:hover {
+		color: var(--bg-amber-400) !important;
+		border-color: var(--bg-amber-300) !important;
+	}
+}

frontend/src/components/facingIssueBtn/FacingIssueBtn.tsx

@@ -0,0 +1,57 @@
+import './FacingIssueBtn.style.scss';
+
+import { Button } from 'antd';
+import logEvent from 'api/common/logEvent';
+import cx from 'classnames';
+import { FeatureKeys } from 'constants/features';
+import useFeatureFlags from 'hooks/useFeatureFlag';
+import { defaultTo } from 'lodash-es';
+import { HelpCircle } from 'lucide-react';
+import { isCloudUser } from 'utils/app';
+
+export interface FacingIssueBtnProps {
+	eventName: string;
+	attributes: Record<string, unknown>;
+	message?: string;
+	buttonText?: string;
+	className?: string;
+}
+
+function FacingIssueBtn({
+	attributes,
+	eventName,
+	message = '',
+	buttonText = '',
+	className = '',
+}: FacingIssueBtnProps): JSX.Element | null {
+	const handleFacingIssuesClick = (): void => {
+		logEvent(eventName, attributes);
+
+		if (window.Intercom) {
+			window.Intercom('showNewMessage', defaultTo(message, ''));
+		}
+	};
+
+	const isChatSupportEnabled = useFeatureFlags(FeatureKeys.CHAT_SUPPORT)?.active;
+	const isCloudUserVal = isCloudUser();
+
+	return isCloudUserVal && isChatSupportEnabled ? ( // Note: we would need to move this condition to license based in future
+		<div className="facing-issue-button">
+			<Button
+				className={cx('periscope-btn', 'facing-issue-button', className)}
+				onClick={handleFacingIssuesClick}
+				icon={<HelpCircle size={14} />}
+			>
+				{buttonText || 'Facing issues?'}
+			</Button>
+		</div>
+	) : null;
+}
+
+FacingIssueBtn.defaultProps = {
+	message: '',
+	buttonText: '',
+	className: '',
+};
+
+export default FacingIssueBtn;

frontend/src/container/BillingContainer/BillingContainer.styles.scss

@@ -1,4 +1,5 @@
 .billing-container {
+	margin-bottom: 40px;
 	padding-top: 36px;
 	width: 65%;
 

frontend/src/container/CreateAlertRule/index.tsx

@@ -2,6 +2,7 @@ import { Form, Row } from 'antd';
 import { ENTITY_VERSION_V4 } from 'constants/app';
 import FormAlertRules from 'container/FormAlertRules';
 import { useGetCompositeQueryParam } from 'hooks/queryBuilder/useGetCompositeQueryParam';
+import { isEqual } from 'lodash-es';
 import { useEffect, useState } from 'react';
 import { useLocation } from 'react-router-dom';
 import { AlertTypes } from 'types/api/alerts/alertTypes';
@@ -18,9 +19,7 @@ import SelectAlertType from './SelectAlertType';
 
 function CreateRules(): JSX.Element {
 	const [initValues, setInitValues] = useState<AlertDef | null>(null);
-	const [alertType, setAlertType] = useState<AlertTypes>(
-		AlertTypes.METRICS_BASED_ALERT,
-	);
+	const [alertType, setAlertType] = useState<AlertTypes>();
 
 	const location = useLocation();
 	const queryParams = new URLSearchParams(location.search);
@@ -56,10 +55,10 @@ function CreateRules(): JSX.Element {
 		}
 		const dataSource = compositeQuery?.builder?.queryData[0]?.dataSource;
 
-		const alertType = ALERT_TYPE_VS_SOURCE_MAPPING[dataSource];
+		const alertTypeFromQuery = ALERT_TYPE_VS_SOURCE_MAPPING[dataSource];
 
-		if (alertType) {
-			onSelectType(alertType);
+		if (alertTypeFromQuery && !isEqual(alertType, alertTypeFromQuery)) {
+			onSelectType(alertTypeFromQuery);
 		}
 		// eslint-disable-next-line react-hooks/exhaustive-deps
 	}, [compositeQuery]);

frontend/src/container/FormAlertRules/FormAlertRules.styles.scss

@@ -1,45 +1,50 @@
 .create-alert-modal {
-   .ant-modal-content {
-        background-color: var(--bg-ink-300);
-        .ant-modal-confirm-title {
-            color: var(--bg-vanilla-100);
-        }
+	.ant-modal-content {
+		background-color: var(--bg-ink-300);
+		.ant-modal-confirm-title {
+			color: var(--bg-vanilla-100);
+		}
 
-        .ant-modal-confirm-content {
-            .ant-typography {
-                color: var(--bg-vanilla-100);
-            }
-        }
+		.ant-modal-confirm-content {
+			.ant-typography {
+				color: var(--bg-vanilla-100);
+			}
+		}
 
-        .ant-modal-confirm-btns {
-            button:nth-of-type(1) {
-                background-color: var(--bg-slate-400);
-                border: none;
-                color: var(--bg-vanilla-100);
-            }
-        }
-   } 
+		.ant-modal-confirm-btns {
+			button:nth-of-type(1) {
+				background-color: var(--bg-slate-400);
+				border: none;
+				color: var(--bg-vanilla-100);
+			}
+		}
+	}
 }
 
 .lightMode {
-    .ant-modal-content {
-        background-color: var(--bg-vanilla-100);
-        .ant-modal-confirm-title {
-            color: var(--bg-ink-500);
-        }
+	.ant-modal-content {
+		background-color: var(--bg-vanilla-100);
+		.ant-modal-confirm-title {
+			color: var(--bg-ink-500);
+		}
 
-        .ant-modal-confirm-content {
-            .ant-typography {
-                color: var(--bg-ink-500);
-            }
-        }
+		.ant-modal-confirm-content {
+			.ant-typography {
+				color: var(--bg-ink-500);
+			}
+		}
 
-        .ant-modal-confirm-btns {
-            button:nth-of-type(1) {
-                background-color: var(--bg-vanilla-300);
-                border: none;
-                color: var(--bg-ink-500);
-            }
-        }
-   }  
-}
+		.ant-modal-confirm-btns {
+			button:nth-of-type(1) {
+				background-color: var(--bg-vanilla-300);
+				border: none;
+				color: var(--bg-ink-500);
+			}
+		}
+	}
+}
+
+.facing-issue-btn {
+	margin-top: 20px;
+	width: 100%;
+}

frontend/src/container/FormAlertRules/index.tsx

@@ -11,6 +11,7 @@ import {
 } from 'antd';
 import saveAlertApi from 'api/alerts/save';
 import testAlertApi from 'api/alerts/testAlert';
+import FacingIssueBtn from 'components/facingIssueBtn/FacingIssueBtn';
 import { FeatureKeys } from 'constants/features';
 import { QueryParams } from 'constants/query';
 import { PANEL_TYPES } from 'constants/queryBuilder';
@@ -138,15 +139,21 @@ function FormAlertRules({
 
 	useEffect(() => {
 		// Set selectedQueryName based on the length of queryOptions
-		setAlertDef((def) => ({
-			...def,
-			condition: {
-				...def.condition,
-				selectedQueryName:
-					queryOptions.length > 0 ? String(queryOptions[0].value) : undefined,
-			},
-		}));
-	}, [currentQuery?.queryType, queryOptions]);
+		const selectedQueryName = alertDef?.condition?.selectedQueryName;
+		if (
+			!selectedQueryName ||
+			!queryOptions.some((option) => option.value === selectedQueryName)
+		) {
+			setAlertDef((def) => ({
+				...def,
+				condition: {
+					...def.condition,
+					selectedQueryName:
+						queryOptions.length > 0 ? String(queryOptions[0].value) : undefined,
+				},
+			}));
+		}
+	}, [alertDef, currentQuery?.queryType, queryOptions]);
 
 	const onCancelHandler = useCallback(() => {
 		history.replace(ROUTES.LIST_ALL_ALERT);
@@ -482,6 +489,8 @@ function FormAlertRules({
 		alertDef?.broadcastToAll ||
 		(alertDef.preferredChannels && alertDef.preferredChannels.length > 0);
 
+	const isRuleCreated = !ruleId || ruleId === 0;
+
 	return (
 		<>
 			{Element}
@@ -563,6 +572,30 @@ function FormAlertRules({
 				</StyledLeftContainer>
 				<Col flex="1 1 300px">
 					<UserGuide queryType={currentQuery.queryType} />
+					<FacingIssueBtn
+						attributes={{
+							alert: alertDef?.alert,
+							alertType: alertDef?.alertType,
+							id: ruleId,
+							ruleType: alertDef?.ruleType,
+							state: (alertDef as any)?.state,
+							panelType,
+							screen: isRuleCreated ? 'Edit Alert' : 'New Alert',
+						}}
+						className="facing-issue-btn"
+						eventName="Alert: Facing Issues in alert"
+						buttonText="Facing Issues in alert"
+						message={`Hi Team,
+
+I am facing issues configuring alerts in SigNoz. Here are my alert rule details
+
+Name: ${alertDef?.alert || ''}
+Alert Type: ${alertDef?.alertType || ''}
+State: ${(alertDef as any)?.state || ''}
+Alert Id: ${ruleId}
+
+Thanks`}
+					/>
 				</Col>
 			</PanelContainer>
 		</>

frontend/src/container/GridCardLayout/GridCardLayout.tsx

@@ -1,7 +1,8 @@
 import './GridCardLayout.styles.scss';
 
 import { PlusOutlined } from '@ant-design/icons';
-import { Tooltip } from 'antd';
+import { Flex, Tooltip } from 'antd';
+import FacingIssueBtn from 'components/facingIssueBtn/FacingIssueBtn';
 import { SOMETHING_WENT_WRONG } from 'constants/api';
 import { QueryParams } from 'constants/query';
 import { PANEL_TYPES } from 'constants/queryBuilder';
@@ -169,28 +170,47 @@ function GraphLayout({ onAddPanelHandler }: GraphLayoutProps): JSX.Element {
 
 	return (
 		<>
-			<ButtonContainer>
-				<Tooltip title="Open in Full Screen">
-					<Button
-						className="periscope-btn"
-						loading={updateDashboardMutation.isLoading}
-						onClick={handle.enter}
-						icon={<FullscreenIcon size={16} />}
-						disabled={updateDashboardMutation.isLoading}
-					/>
-				</Tooltip>
+			<Flex justify="flex-end" gap={8} align="center">
+				<FacingIssueBtn
+					attributes={{
+						uuid: selectedDashboard?.uuid,
+						title: data?.title,
+						screen: 'Dashboard Details',
+					}}
+					eventName="Dashboard: Facing Issues in dashboard"
+					buttonText="Facing Issues in dashboard"
+					message={`Hi Team,
 
-				{!isDashboardLocked && addPanelPermission && (
-					<Button
-						className="periscope-btn"
-						onClick={onAddPanelHandler}
-						icon={<PlusOutlined />}
-						data-testid="add-panel"
-					>
-						{t('dashboard:add_panel')}
-					</Button>
-				)}
-			</ButtonContainer>
+I am facing issues configuring dashboard in SigNoz. Here are my dashboard details
+
+Name: ${data?.title || ''}
+Dashboard Id: ${selectedDashboard?.uuid || ''}
+
+Thanks`}
+				/>
+				<ButtonContainer>
+					<Tooltip title="Open in Full Screen">
+						<Button
+							className="periscope-btn"
+							loading={updateDashboardMutation.isLoading}
+							onClick={handle.enter}
+							icon={<FullscreenIcon size={16} />}
+							disabled={updateDashboardMutation.isLoading}
+						/>
+					</Tooltip>
+
+					{!isDashboardLocked && addPanelPermission && (
+						<Button
+							className="periscope-btn"
+							onClick={onAddPanelHandler}
+							icon={<PlusOutlined />}
+							data-testid="add-panel"
+						>
+							{t('dashboard:add_panel')}
+						</Button>
+					)}
+				</ButtonContainer>
+			</Flex>
 
 			<FullScreen handle={handle} className="fullscreen-grid-container">
 				<ReactGridLayout

frontend/src/container/ListAlertRules/ListAlert.tsx

@@ -358,6 +358,18 @@ function ListAlert({ allAlertRules, refetch }: ListAlertProps): JSX.Element {
 				pagination={{
 					defaultCurrent: Number(paginationParam) || 1,
 				}}
+				facingIssueBtn={{
+					attributes: {
+						screen: 'Alert list page',
+					},
+					eventName: 'Alert: Facing Issues in alert',
+					buttonText: 'Facing Issues in alert',
+					message: `Hi Team,
+
+I am facing issues with alerts.
+
+Thanks`,
+				}}
 			/>
 		</>
 	);

frontend/src/container/ListOfDashboard/DashboardsList.tsx

@@ -385,6 +385,18 @@ function DashboardsList(): JSX.Element {
 					dataSource={data}
 					onChange={handleChange}
 					showSorterTooltip
+					facingIssueBtn={{
+						attributes: {
+							screen: 'Dashboard list page',
+						},
+						eventName: 'Dashboard: Facing Issues in dashboard',
+						buttonText: 'Facing Issues in dashboard',
+						message: `Hi Team,
+
+I am facing issues with dashboards.
+						
+Thanks`,
+					}}
 				/>
 			</TableContainer>
 		</Card>

frontend/src/container/NewDashboard/ComponentsSlider/index.tsx

@@ -1,146 +1,59 @@
-import { SOMETHING_WENT_WRONG } from 'constants/api';
 import { QueryParams } from 'constants/query';
 import { PANEL_TYPES } from 'constants/queryBuilder';
-import { useUpdateDashboard } from 'hooks/dashboard/useUpdateDashboard';
-import { useNotifications } from 'hooks/useNotifications';
 import createQueryParams from 'lib/createQueryParams';
 import history from 'lib/history';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
 import { LogsAggregatorOperator } from 'types/common/queryBuilder';
 import { v4 as uuid } from 'uuid';
 
-import {
-	listViewInitialLogQuery,
-	listViewInitialTraceQuery,
-	PANEL_TYPES_INITIAL_QUERY,
-} from './constants';
+import { PANEL_TYPES_INITIAL_QUERY } from './constants';
 import menuItems from './menuItems';
 import { Card, Container, Text } from './styles';
 
 function DashboardGraphSlider(): JSX.Element {
-	const {
-		handleToggleDashboardSlider,
-		layouts,
-		selectedDashboard,
-	} = useDashboard();
-
-	const { data } = selectedDashboard || {};
-
-	const { notifications } = useNotifications();
-
-	const updateDashboardMutation = useUpdateDashboard();
+	const { handleToggleDashboardSlider } = useDashboard();
 
 	// eslint-disable-next-line sonarjs/cognitive-complexity
 	const onClickHandler = (name: PANEL_TYPES) => (): void => {
 		const id = uuid();
-
-		updateDashboardMutation.mutateAsync(
-			{
-				uuid: selectedDashboard?.uuid || '',
-				data: {
-					title: data?.title || '',
-					variables: data?.variables || {},
-					description: data?.description || '',
-					name: data?.name || '',
-					tags: data?.tags || [],
-					version: data?.version || 'v3',
-					layout: [
+		handleToggleDashboardSlider(false);
+		const queryParamsLog = {
+			graphType: name,
+			widgetId: id,
+			[QueryParams.compositeQuery]: JSON.stringify({
+				...PANEL_TYPES_INITIAL_QUERY[name],
+				builder: {
+					...PANEL_TYPES_INITIAL_QUERY[name].builder,
+					queryData: [
 						{
-							i: id,
-							w: 6,
-							x: 0,
-							h: 3,
-							y: 0,
-						},
-						...(layouts.filter((layout) => layout.i !== PANEL_TYPES.EMPTY_WIDGET) ||
-							[]),
-					],
-					widgets: [
-						...(data?.widgets || []),
-						{
-							id,
-							title: '',
-							description: '',
-							isStacked: false,
-							nullZeroValues: '',
-							opacity: '',
-							panelTypes: name,
-							query:
-								name === PANEL_TYPES.LIST
-									? listViewInitialLogQuery
-									: PANEL_TYPES_INITIAL_QUERY[name],
-							timePreferance: 'GLOBAL_TIME',
-							softMax: null,
-							softMin: null,
-							selectedLogFields: [
-								{
-									dataType: 'string',
-									type: '',
-									name: 'body',
-								},
-								{
-									dataType: 'string',
-									type: '',
-									name: 'timestamp',
-								},
-							],
-							selectedTracesFields: [
-								...listViewInitialTraceQuery.builder.queryData[0].selectColumns,
-							],
+							...PANEL_TYPES_INITIAL_QUERY[name].builder.queryData[0],
+							aggregateOperator: LogsAggregatorOperator.NOOP,
+							orderBy: [{ columnName: 'timestamp', order: 'desc' }],
+							offset: 0,
+							pageSize: 100,
 						},
 					],
 				},
-			},
-			{
-				onSuccess: (data) => {
-					if (data.payload) {
-						handleToggleDashboardSlider(false);
-						const queryParamsLog = {
-							graphType: name,
-							widgetId: id,
-							[QueryParams.compositeQuery]: JSON.stringify({
-								...PANEL_TYPES_INITIAL_QUERY[name],
-								builder: {
-									...PANEL_TYPES_INITIAL_QUERY[name].builder,
-									queryData: [
-										{
-											...PANEL_TYPES_INITIAL_QUERY[name].builder.queryData[0],
-											aggregateOperator: LogsAggregatorOperator.NOOP,
-											orderBy: [{ columnName: 'timestamp', order: 'desc' }],
-											offset: 0,
-											pageSize: 100,
-										},
-									],
-								},
-							}),
-						};
+			}),
+		};
 
-						const queryParams = {
-							graphType: name,
-							widgetId: id,
-							[QueryParams.compositeQuery]: JSON.stringify(
-								PANEL_TYPES_INITIAL_QUERY[name],
-							),
-						};
+		const queryParams = {
+			graphType: name,
+			widgetId: id,
+			[QueryParams.compositeQuery]: JSON.stringify(
+				PANEL_TYPES_INITIAL_QUERY[name],
+			),
+		};
 
-						if (name === PANEL_TYPES.LIST) {
-							history.push(
-								`${history.location.pathname}/new?${createQueryParams(queryParamsLog)}`,
-							);
-						} else {
-							history.push(
-								`${history.location.pathname}/new?${createQueryParams(queryParams)}`,
-							);
-						}
-					}
-				},
-				onError: () => {
-					notifications.success({
-						message: SOMETHING_WENT_WRONG,
-					});
-				},
-			},
-		);
+		if (name === PANEL_TYPES.LIST) {
+			history.push(
+				`${history.location.pathname}/new?${createQueryParams(queryParamsLog)}`,
+			);
+		} else {
+			history.push(
+				`${history.location.pathname}/new?${createQueryParams(queryParams)}`,
+			);
+		}
 	};
 
 	return (

frontend/src/container/NewWidget/LeftContainer/QuerySection/index.tsx

@@ -4,6 +4,7 @@ import { Button, Tabs, Tooltip, Typography } from 'antd';
 import TextToolTip from 'components/TextToolTip';
 import { PANEL_TYPES } from 'constants/queryBuilder';
 import { QBShortcuts } from 'constants/shortcuts/QBShortcuts';
+import { getDefaultWidgetData } from 'container/NewWidget/utils';
 import { QueryBuilder } from 'container/QueryBuilder';
 import { QueryBuilderProps } from 'container/QueryBuilder/QueryBuilder.interfaces';
 import { useKeyboardHotkeys } from 'hooks/hotkeys/useKeyboardHotkeys';
@@ -11,6 +12,7 @@ import { useQueryBuilder } from 'hooks/queryBuilder/useQueryBuilder';
 import { useShareBuilderUrl } from 'hooks/queryBuilder/useShareBuilderUrl';
 import { updateStepInterval } from 'hooks/queryBuilder/useStepInterval';
 import useUrlQuery from 'hooks/useUrlQuery';
+import { defaultTo } from 'lodash-es';
 import { Atom, Play, Terminal } from 'lucide-react';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
 import {
@@ -55,8 +57,11 @@ function QuerySection({
 
 	const getWidget = useCallback(() => {
 		const widgetId = urlQuery.get('widgetId');
-		return widgets?.find((e) => e.id === widgetId);
-	}, [widgets, urlQuery]);
+		return defaultTo(
+			widgets?.find((e) => e.id === widgetId),
+			getDefaultWidgetData(widgetId || '', selectedGraph),
+		);
+	}, [urlQuery, widgets, selectedGraph]);
 
 	const selectedWidget = getWidget() as Widgets;
 

frontend/src/container/NewWidget/index.tsx

@@ -1,6 +1,7 @@
 /* eslint-disable sonarjs/cognitive-complexity */
 import { LockFilled, WarningOutlined } from '@ant-design/icons';
-import { Button, Modal, Space, Tooltip, Typography } from 'antd';
+import { Button, Flex, Modal, Space, Tooltip, Typography } from 'antd';
+import FacingIssueBtn from 'components/facingIssueBtn/FacingIssueBtn';
 import { SOMETHING_WENT_WRONG } from 'constants/api';
 import { FeatureKeys } from 'constants/features';
 import { QueryParams } from 'constants/query';
@@ -14,6 +15,7 @@ import { MESSAGE, useIsFeatureDisabled } from 'hooks/useFeatureFlag';
 import { useNotifications } from 'hooks/useNotifications';
 import useUrlQuery from 'hooks/useUrlQuery';
 import history from 'lib/history';
+import { defaultTo, isUndefined } from 'lodash-es';
 import { DashboardWidgetPageParams } from 'pages/DashboardWidget';
 import { useDashboard } from 'providers/Dashboard/Dashboard';
 import {
@@ -45,7 +47,11 @@ import {
 	RightContainerWrapper,
 } from './styles';
 import { NewWidgetProps } from './types';
-import { getIsQueryModified, handleQueryChange } from './utils';
+import {
+	getDefaultWidgetData,
+	getIsQueryModified,
+	handleQueryChange,
+} from './utils';
 
 function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 	const {
@@ -80,10 +86,26 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 
 	const { dashboardId } = useParams<DashboardWidgetPageParams>();
 
+	const [isNewDashboard, setIsNewDashboard] = useState<boolean>(false);
+
+	useEffect(() => {
+		const widgetId = query.get('widgetId');
+		const selectedWidget = widgets?.find((e) => e.id === widgetId);
+		const isWidgetNotPresent = isUndefined(selectedWidget);
+		if (isWidgetNotPresent) {
+			setIsNewDashboard(true);
+		}
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+	}, []);
+
 	const getWidget = useCallback(() => {
 		const widgetId = query.get('widgetId');
-		return widgets?.find((e) => e.id === widgetId);
-	}, [query, widgets]);
+		const selectedWidget = widgets?.find((e) => e.id === widgetId);
+		return defaultTo(
+			selectedWidget,
+			getDefaultWidgetData(widgetId || '', selectedGraph),
+		);
+	}, [query, selectedGraph, widgets]);
 
 	const [selectedWidget, setSelectedWidget] = useState(getWidget());
 
@@ -227,6 +249,20 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 			return;
 		}
 
+		const widgetId = query.get('widgetId');
+		let updatedLayout = selectedDashboard.data.layout || [];
+		if (isNewDashboard) {
+			updatedLayout = [
+				{
+					i: widgetId || '',
+					w: 6,
+					x: 0,
+					h: 3,
+					y: 0,
+				},
+				...updatedLayout,
+			];
+		}
 		const dashboard: Dashboard = {
 			...selectedDashboard,
 			uuid: selectedDashboard.uuid,
@@ -254,6 +290,7 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 					},
 					...afterWidgets,
 				],
+				layout: [...updatedLayout],
 			},
 		};
 
@@ -274,6 +311,8 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 		});
 	}, [
 		selectedDashboard,
+		query,
+		isNewDashboard,
 		preWidgets,
 		selectedWidget,
 		selectedTime.enum,
@@ -363,33 +402,55 @@ function NewWidget({ selectedGraph }: NewWidgetProps): JSX.Element {
 
 	return (
 		<Container>
-			<ButtonContainer>
-				{isSaveDisabled && (
-					<Tooltip title={MESSAGE.PANEL}>
+			<Flex justify="space-between" align="center">
+				<FacingIssueBtn
+					attributes={{
+						uuid: selectedDashboard?.uuid,
+						title: selectedDashboard?.data.title,
+						panelType: graphType,
+						widgetId: query.get('widgetId'),
+						queryType: currentQuery.queryType,
+					}}
+					eventName="Dashboard: Facing Issues in dashboard"
+					buttonText="Facing Issues in dashboard"
+					message={`Hi Team,
+
+I am facing issues configuring dashboard in SigNoz. Here are my dashboard details
+				
+Name: ${selectedDashboard?.data.title || ''}
+Panel type: ${graphType}
+Dashboard Id: ${selectedDashboard?.uuid || ''}
+				
+Thanks`}
+				/>
+				<ButtonContainer>
+					{isSaveDisabled && (
+						<Tooltip title={MESSAGE.PANEL}>
+							<Button
+								icon={<LockFilled />}
+								type="primary"
+								disabled={isSaveDisabled}
+								onClick={onSaveDashboard}
+							>
+								Save Changes
+							</Button>
+						</Tooltip>
+					)}
+
+					{!isSaveDisabled && (
 						<Button
-							icon={<LockFilled />}
 							type="primary"
+							data-testid="new-widget-save"
+							loading={updateDashboardMutation.isLoading}
 							disabled={isSaveDisabled}
 							onClick={onSaveDashboard}
 						>
 							Save Changes
 						</Button>
-					</Tooltip>
-				)}
-
-				{!isSaveDisabled && (
-					<Button
-						type="primary"
-						data-testid="new-widget-save"
-						loading={updateDashboardMutation.isLoading}
-						disabled={isSaveDisabled}
-						onClick={onSaveDashboard}
-					>
-						Save Changes
-					</Button>
-				)}
-				<Button onClick={onClickDiscardHandler}>Discard Changes</Button>
-			</ButtonContainer>
+					)}
+					<Button onClick={onClickDiscardHandler}>Discard Changes</Button>
+				</ButtonContainer>
+			</Flex>
 
 			<PanelContainer>
 				<LeftContainerWrapper flex={5}>

frontend/src/container/NewWidget/utils.ts

@@ -3,7 +3,13 @@ import {
 	initialQueryBuilderFormValuesMap,
 	PANEL_TYPES,
 } from 'constants/queryBuilder';
+import {
+	listViewInitialLogQuery,
+	listViewInitialTraceQuery,
+	PANEL_TYPES_INITIAL_QUERY,
+} from 'container/NewDashboard/ComponentsSlider/constants';
 import { isEqual, set, unset } from 'lodash-es';
+import { Widgets } from 'types/api/dashboard/getAll';
 import { IBuilderQuery, Query } from 'types/api/queryBuilder/queryBuilderData';
 import { DataSource } from 'types/common/queryBuilder';
 
@@ -302,3 +308,38 @@ export function handleQueryChange(
 		},
 	};
 }
+
+export const getDefaultWidgetData = (
+	id: string,
+	name: PANEL_TYPES,
+): Widgets => ({
+	id,
+	title: '',
+	description: '',
+	isStacked: false,
+	nullZeroValues: '',
+	opacity: '',
+	panelTypes: name,
+	query:
+		name === PANEL_TYPES.LIST
+			? listViewInitialLogQuery
+			: PANEL_TYPES_INITIAL_QUERY[name],
+	timePreferance: 'GLOBAL_TIME',
+	softMax: null,
+	softMin: null,
+	selectedLogFields: [
+		{
+			dataType: 'string',
+			type: '',
+			name: 'body',
+		},
+		{
+			dataType: 'string',
+			type: '',
+			name: 'timestamp',
+		},
+	],
+	selectedTracesFields: [
+		...listViewInitialTraceQuery.builder.queryData[0].selectColumns,
+	],
+});

frontend/src/container/OnboardingContainer/common/ModuleStepsContainer/ModuleStepsContainer.tsx

@@ -10,6 +10,7 @@ import {
 	LeftCircleOutlined,
 } from '@ant-design/icons';
 import { Button, Space, Steps, Typography } from 'antd';
+import logEvent from 'api/common/logEvent';
 import ROUTES from 'constants/routes';
 import { stepsMap } from 'container/OnboardingContainer/constants/stepsConfig';
 import { DataSourceType } from 'container/OnboardingContainer/Steps/DataSource/DataSource';
@@ -381,11 +382,12 @@ export default function ModuleStepsContainer({
 	};
 
 	const handleFacingIssuesClick = (): void => {
-		trackEvent('Onboarding V2: Facing Issues Sending Data to SigNoz', {
+		logEvent('Onboarding V2: Facing Issues Sending Data to SigNoz', {
 			dataSource: selectedDataSource?.id,
 			framework: selectedFramework,
 			environment: selectedEnvironment,
 			module: activeStep?.module?.id,
+			step: activeStep?.step?.id,
 		});
 
 		const message = `Hi Team,

frontend/src/container/QueryBuilder/components/QBEntityOptions/QBEntityOptions.tsx

@@ -105,8 +105,8 @@ export default function QBEntityOptions({
 								onQueryFunctionsUpdates && (
 									<QueryFunctions
 										query={query}
-										queryFunctions={query.functions}
-										key={query.functions.toString()}
+										queryFunctions={query.functions || []}
+										key={query.functions?.toString()}
 										onChange={onQueryFunctionsUpdates}
 										maxFunctions={isLogsDataSource ? 1 : 3}
 									/>

frontend/src/container/TracesExplorer/ListView/utils.tsx

@@ -24,7 +24,14 @@ export const getTraceLink = (record: RowData): string =>
 export const getListColumns = (
 	selectedColumns: BaseAutocompleteData[],
 ): ColumnsType<RowData> => {
-	const initialColumns: ColumnsType<RowData> = [];
+	const initialColumns: ColumnsType<RowData> = [
+		{
+			dataIndex: 'date',
+			key: 'date',
+			title: 'Timestamp',
+			width: 145,
+		},
+	];
 
 	const columns: ColumnsType<RowData> =
 		selectedColumns.map(({ dataType, key, type }) => ({

frontend/src/pages/DashboardWidget/index.tsx

@@ -49,15 +49,11 @@ function DashboardWidget(): JSX.Element | null {
 		);
 	}
 
-	if (selectedWidget === undefined) {
-		return null;
-	}
-
 	return (
 		<NewWidget
-			yAxisUnit={selectedWidget.yAxisUnit}
+			yAxisUnit={selectedWidget?.yAxisUnit}
 			selectedGraph={selectedGraph}
-			fillSpans={selectedWidget.fillSpans}
+			fillSpans={selectedWidget?.fillSpans}
 		/>
 	);
 }

frontend/src/providers/Dashboard/Dashboard.tsx

@@ -176,8 +176,6 @@ export function DashboardProvider({
 
 		return data;
 	};
-
-	console.log(variablesToGetUpdated);
 	const dashboardResponse = useQuery(
 		[REACT_QUERY_KEY.DASHBOARD_BY_ID, isDashboardPage?.params],
 		{

frontend/yarn.lock

@@ -5813,12 +5813,12 @@ axe-core@^4.6.2:
   resolved "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz"
   integrity sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==
 
-axios@1.6.2:
-  version "1.6.2"
-  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.2.tgz#de67d42c755b571d3e698df1b6504cde9b0ee9f2"
-  integrity sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==
+axios@1.6.4:
+  version "1.6.4"
+  resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.4.tgz#184ee1f63d412caffcf30d2c50982253c3ee86e0"
+  integrity sha512-heJnIs6N4aa1eSthhN9M5ioILu8Wi8vmQW9iHQ9NUvfkJb0lEEDUiIdQNAuBtfUt3FxReaKdpQA5DbmMOqzF/A==
   dependencies:
-    follow-redirects "^1.15.0"
+    follow-redirects "^1.15.4"
     form-data "^4.0.0"
     proxy-from-env "^1.1.0"
 
@@ -9204,10 +9204,10 @@ follow-redirects@^1.0.0, follow-redirects@^1.14.0:
   resolved "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz"
   integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==
 
-follow-redirects@^1.15.0:
-  version "1.15.3"
-  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.3.tgz#fe2f3ef2690afce7e82ed0b44db08165b207123a"
-  integrity sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==
+follow-redirects@^1.15.4:
+  version "1.15.4"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.4.tgz#cdc7d308bf6493126b17ea2191ea0ccf3e535adf"
+  integrity sha512-Cr4D/5wlrb0z9dgERpUL3LrmPKVDsETIJhaCMeDfuFYcqa5bldGV6wBsAN6X/vxlXQtFBMrXdXxdL8CbDTGniw==
 
 fontfaceobserver@2.3.0:
   version "2.3.0"

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.20.0
 	github.com/DATA-DOG/go-sqlmock v1.5.2
 	github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd
-	github.com/SigNoz/signoz-otel-collector v0.88.20
+	github.com/SigNoz/signoz-otel-collector v0.88.21
 	github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974
 	github.com/SigNoz/zap_otlp/zap_otlp_sync v0.0.0-20230822164844-1b861a431974
 	github.com/antonmedv/expr v1.15.3
@@ -46,7 +46,6 @@ require (
 	github.com/russellhaering/goxmldsig v1.2.0
 	github.com/samber/lo v1.38.1
 	github.com/sethvargo/go-password v0.2.0
-	github.com/smartystreets/assertions v1.13.1
 	github.com/smartystreets/goconvey v1.8.1
 	github.com/soheilhy/cmux v0.1.5
 	github.com/srikanthccv/ClickHouse-go-mock v0.7.0
@@ -66,9 +65,9 @@ require (
 	go.opentelemetry.io/otel/sdk v1.23.1
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.19.0
+	golang.org/x/crypto v0.21.0
 	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
-	golang.org/x/net v0.21.0
+	golang.org/x/net v0.23.0
 	golang.org/x/oauth2 v0.16.0
 	google.golang.org/grpc v1.62.0
 	google.golang.org/protobuf v1.33.0
@@ -190,7 +189,7 @@ require (
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	gonum.org/v1/gonum v0.14.0 // indirect
@@ -203,4 +202,4 @@ require (
 	k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect
 )
 
-replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.10.1
+replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.11.0

go.sum

@@ -96,10 +96,10 @@ github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd h1:Bk43AsDYe0fhkbj57eGXx8H3ZJ4zhmQXBnrW523ktj8=
 github.com/SigNoz/govaluate v0.0.0-20240203125216-988004ccc7fd/go.mod h1:nxRcH/OEdM8QxzH37xkGzomr1O0JpYBRS6pwjsWW6Pc=
-github.com/SigNoz/prometheus v1.10.1 h1:2LKRtPDMgSJpgDRDy0GUQiXi+yhDNqcbptuEon4Wpls=
-github.com/SigNoz/prometheus v1.10.1/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww=
-github.com/SigNoz/signoz-otel-collector v0.88.20 h1:saC1unOxkpw4VCKyPsIIUq37vKkQ5fK/eDlnuHMm0UE=
-github.com/SigNoz/signoz-otel-collector v0.88.20/go.mod h1:PThU+A6SgzEotT3ngKN4WVGWW0+eS7F1a2Rnq11aZZA=
+github.com/SigNoz/prometheus v1.11.0 h1:toX7fU2wqY1TnzvPzDglIYx6OxpqrZ0NNlM/H5S5+u8=
+github.com/SigNoz/prometheus v1.11.0/go.mod h1:MffmFu2qFILQrOHehx3D0XjYtaZMVfI+Ppeiv98x4Ww=
+github.com/SigNoz/signoz-otel-collector v0.88.21 h1:9K1FLUncUZh7cPfOLDPuT8itU8LyCufk4QwGp18hK88=
+github.com/SigNoz/signoz-otel-collector v0.88.21/go.mod h1:sT1EM9PFDaOJLbAz5npWpgXK6OhpWJ9PpSwyhHWs9rU=
 github.com/SigNoz/zap_otlp v0.1.0 h1:T7rRcFN87GavY8lDGZj0Z3Xv6OhJA6Pj3I9dNPmqvRc=
 github.com/SigNoz/zap_otlp v0.1.0/go.mod h1:lcHvbDbRgvDnPxo9lDlaL1JK2PyOyouP/C3ynnYIvyo=
 github.com/SigNoz/zap_otlp/zap_otlp_encoder v0.0.0-20230822164844-1b861a431974 h1:PKVgdf83Yw+lZJbFtNGBgqXiXNf3+kOXW2qZ7Ms7OaY=
@@ -777,8 +777,6 @@ github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGB
 github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
 github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
-github.com/smartystreets/assertions v1.13.1 h1:Ef7KhSmjZcK6AVf9YbJdvPYG9avaF0ZxudX+ThRdWfU=
-github.com/smartystreets/assertions v1.13.1/go.mod h1:cXr/IwVfSo/RbCSPhoAPv73p3hlSdrBH/b3SdnW/LMY=
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
@@ -946,8 +944,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1044,8 +1042,8 @@ golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1179,13 +1177,13 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/query-service/app/clickhouseReader/reader.go

@@ -163,12 +163,24 @@ func NewReaderFromClickhouseConnection(
 		os.Exit(1)
 	}
 
+	regex := os.Getenv("ClickHouseOptimizeReadInOrderRegex")
+	var regexCompiled *regexp.Regexp
+	if regex != "" {
+		regexCompiled, err = regexp.Compile(regex)
+		if err != nil {
+			zap.L().Error("Incorrect regex for ClickHouseOptimizeReadInOrderRegex")
+			os.Exit(1)
+		}
+	}
+
 	wrap := clickhouseConnWrapper{
 		conn: db,
 		settings: ClickhouseQuerySettings{
 			MaxExecutionTimeLeaf:                os.Getenv("ClickHouseMaxExecutionTimeLeaf"),
 			TimeoutBeforeCheckingExecutionSpeed: os.Getenv("ClickHouseTimeoutBeforeCheckingExecutionSpeed"),
 			MaxBytesToRead:                      os.Getenv("ClickHouseMaxBytesToRead"),
+			OptimizeReadInOrderRegex:            os.Getenv("ClickHouseOptimizeReadInOrderRegex"),
+			OptimizeReadInOrderRegexCompiled:    regexCompiled,
 		},
 	}
 

pkg/query-service/app/clickhouseReader/wrapper.go

@@ -3,6 +3,7 @@ package clickhouseReader
 import (
 	"context"
 	"encoding/json"
+	"regexp"
 	"strings"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -13,6 +14,8 @@ type ClickhouseQuerySettings struct {
 	MaxExecutionTimeLeaf                string
 	TimeoutBeforeCheckingExecutionSpeed string
 	MaxBytesToRead                      string
+	OptimizeReadInOrderRegex            string
+	OptimizeReadInOrderRegexCompiled    *regexp.Regexp
 }
 
 type clickhouseConnWrapper struct {
@@ -58,6 +61,11 @@ func (c clickhouseConnWrapper) addClickHouseSettings(ctx context.Context, query
 		settings["timeout_before_checking_execution_speed"] = c.settings.TimeoutBeforeCheckingExecutionSpeed
 	}
 
+	// only list queries of
+	if c.settings.OptimizeReadInOrderRegex != "" && c.settings.OptimizeReadInOrderRegexCompiled.Match([]byte(query)) {
+		settings["optimize_read_in_order"] = 0
+	}
+
 	ctx = clickhouse.Context(ctx, clickhouse.WithSettings(settings))
 	return ctx
 }

pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-logs.md

@@ -0,0 +1,125 @@
+### Collect Clickhouse Logs
+
+You can configure Clickhouse logs collection by providing the required collector config to your collector.
+
+#### Create collector config file
+
+Save the following config for collecting clickhouse logs in a file named `clickhouse-logs-collection-config.yaml`
+
+```yaml
+receivers:
+  filelog/clickhouse:
+    include: ["${env:CLICKHOUSE_LOG_FILE}"]
+    operators:
+      # Parse default clickhouse text log format.
+      # See https://github.com/ClickHouse/ClickHouse/blob/master/src/Loggers/OwnPatternFormatter.cpp
+      - type: recombine
+        source_identifier: attributes["log.file.name"]
+        is_first_entry: body matches '^\\d{4}\\.\\d{2}\\.\\d{2}\\s+'
+        combine_field: body
+        overwrite_with: oldest
+      - type: regex_parser
+        parse_from: body
+        if: body matches '^(?P<ts>\\d{4}\\.\\d{2}\\.\\d{2} \\d{2}:\\d{2}:\\d{2}.?[0-9]*)\\s+\\[\\s+(\\x1b.*?m)?(?P<thread_id>\\d*)(\\x1b.*?m)?\\s+\\]\\s+{((\\x1b.*?m)?(?P<query_id>[0-9a-zA-Z-_]*)(\\x1b.*?m)?)?}\\s+<(\\x1b.*?m)?(?P<log_level>\\w*)(\\x1b.*?m)?>\\s+((\\x1b.*?m)?(?P<clickhouse_component>[a-zA-Z0-9_]+)(\\x1b.*?m)?:)?\\s+(?s)(?P<message>.*)$'
+        regex: '^(?P<ts>\d{4}\.\d{2}\.\d{2} \d{2}:\d{2}:\d{2}.?[0-9]*)\s+\[\s+(\x1b.*?m)?(?P<thread_id>\d*)(\x1b.*?m)?\s+\]\s+{((\x1b.*?m)?(?P<query_id>[0-9a-zA-Z-_]*)(\x1b.*?m)?)?}\s+<(\x1b.*?m)?(?P<log_level>\w*)(\x1b.*?m)?>\s+((\x1b.*?m)?(?P<clickhouse_component>[a-zA-Z0-9_]+)(\x1b.*?m)?:)?\s+(?s)(?P<message>.*)$'
+      - type: time_parser
+        if: attributes.ts != nil
+        parse_from: attributes.ts
+        layout_type: gotime
+        layout: 2006.01.02 15:04:05.999999
+        location: ${env:CLICKHOUSE_TIMEZONE}
+      - type: remove
+        if: attributes.ts != nil
+        field: attributes.ts
+      - type: severity_parser
+        if: attributes.log_level != nil
+        parse_from: attributes.log_level
+        overwrite_text: true
+        # For mapping details, see getPriorityName defined in https://github.com/ClickHouse/ClickHouse/blob/master/src/Interpreters/InternalTextLogsQueue.cpp
+        mapping:
+          trace:
+            - Trace
+            - Test
+          debug: Debug
+          info:
+            - Information
+            - Notice
+          warn: Warning
+          error: Error
+          fatal:
+            - Fatal
+            - Critical
+      - type: remove
+        if: attributes.log_level != nil
+        field: attributes.log_level
+      - type: move
+        if: attributes.message != nil
+        from: attributes.message
+        to: body
+      - type: add
+        field: attributes.source
+        value: clickhouse
+
+processors:
+  batch:
+    send_batch_size: 10000
+    send_batch_max_size: 11000
+    timeout: 10s
+
+exporters:
+  # export to SigNoz cloud
+  otlp/clickhouse-logs:
+    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
+    tls:
+      insecure: false
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
+
+  # export to local collector
+  # otlp/clickhouse-logs:
+  #   endpoint: "localhost:4317"
+  #   tls:
+  #     insecure: true
+
+service:
+  pipelines:
+    logs/clickhouse:
+      receivers: [filelog/clickhouse]
+      processors: [batch]
+      exporters: [otlp/clickhouse-logs]
+```
+
+#### Set Environment Variables
+
+Set the following environment variables in your otel-collector environment:
+
+```bash
+
+# path of Clickhouse server log file. must be accessible by the otel collector
+# typically found at /var/log/clickhouse-server/clickhouse-server.log.
+# Log file location can be found in clickhouse server config
+# See https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#logger
+export CLICKHOUSE_LOG_FILE="/var/log/clickhouse-server/server.log"
+
+# Locale of the clickhouse server.
+# Clickhouse logs timestamps in it's locale without TZ info
+# Timezone setting can be found in clickhouse config. For details see https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#timezone
+# Must be a IANA timezone name like Asia/Kolkata. For examples, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+export CLICKHOUSE_TIMEZONE="Etc/UTC"
+
+# region specific SigNoz cloud ingestion endpoint
+export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
+
+# your SigNoz ingestion key
+export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
+
+```
+
+#### Use collector config file
+
+Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector  
+```bash
+--config clickhouse-logs-collection-config.yaml
+```  
+Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.
+

pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-metrics.md

@@ -0,0 +1,82 @@
+### Collect Clickhouse Metrics
+
+You can configure Clickhouse metrics collection by providing the required collector config to your collector.
+
+#### Create collector config file
+
+Save the following config for collecting Clickhouse metrics in a file named `clickhouse-metrics-collection-config.yaml`
+
+```yaml
+receivers:
+  prometheus/clickhouse:
+    config:
+      global:
+        scrape_interval: 60s
+      scrape_configs:
+        - job_name: clickhouse
+          static_configs:
+            - targets:
+                - ${env:CLICKHOUSE_PROM_METRICS_ENDPOINT}
+          metrics_path: ${env:CLICKHOUSE_PROM_METRICS_PATH}
+
+processors:
+  # enriches the data with additional host information
+  # see https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor#resource-detection-processor
+  resourcedetection/system:
+    # add additional detectors if needed
+    detectors: ["system"]
+    system:
+      hostname_sources: ["os"]
+
+exporters:
+  # export to SigNoz cloud
+  otlp/clickhouse:
+    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
+    tls:
+      insecure: false
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
+
+  # export to local collector
+  # otlp/clickhouse:
+  #   endpoint: "localhost:4317"
+  #   tls:
+  #     insecure: true
+
+service:
+  pipelines:
+    metrics/clickhouse:
+      receivers: [prometheus/clickhouse]
+      # note: remove this processor if the collector host is not running on the same host as the clickhouse instance
+      processors: [resourcedetection/system]
+      exporters: [otlp/clickhouse]
+```
+
+#### Set Environment Variables
+
+Set the following environment variables in your otel-collector environment:
+
+```bash
+# Prometheus metrics endpoint on the clickhouse server reachable from the otel collector.
+# You can examine clickhouse server configuration to find it. For details see https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#prometheus
+export CLICKHOUSE_PROM_METRICS_ENDPOINT="clickhouse:9363"
+
+# Prometheus metrics path on the clickhouse server
+# You can examine clickhouse server configuration to find it. For details see https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#prometheus
+export CLICKHOUSE_PROM_METRICS_PATH="/metrics"
+
+# region specific SigNoz cloud ingestion endpoint
+export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
+
+# your SigNoz ingestion key
+export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
+
+```
+
+#### Use collector config file
+
+Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector  
+```bash
+--config clickhouse-metrics-collection-config.yaml
+```  
+Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.

pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/collect-query-logs.md

@@ -0,0 +1,80 @@
+### Collect Clickhouse Query Logs
+
+You can configure collection from system.query_log table in clickhouse by providing the required collector config to your collector.
+
+#### Create collector config file
+
+Save the following config for collecting clickhouse query logs in a file named `clickhouse-query-logs-collection-config.yaml`
+
+```yaml
+receivers:
+  clickhousesystemtablesreceiver/query_log:
+    dsn: "${env:CLICKHOUSE_MONITORING_DSN}"
+    cluster_name: "${env:CLICKHOUSE_CLUSTER_NAME}"
+    query_log_scrape_config:
+      scrape_interval_seconds: ${env:QUERY_LOG_SCRAPE_INTERVAL_SECONDS}
+      min_scrape_delay_seconds: ${env:QUERY_LOG_SCRAPE_DELAY_SECONDS}
+
+exporters:
+  # export to SigNoz cloud
+  otlp/clickhouse-query-logs:
+    endpoint: "${env:OTLP_DESTINATION_ENDPOINT}"
+    tls:
+      insecure: false
+    headers:
+      "signoz-access-token": "${env:SIGNOZ_INGESTION_KEY}"
+
+  # export to local collector
+  # otlp/clickhouse-query-logs:
+  #   endpoint: "localhost:4317"
+  #   tls:
+  #     insecure: true
+
+service:
+  pipelines:
+    logs/clickhouse-query-logs:
+      receivers: [clickhousesystemtablesreceiver/query_log]
+      processors: []
+      exporters: [otlp/clickhouse-query-logs]
+```
+
+#### Set Environment Variables
+
+Set the following environment variables in your otel-collector environment:
+
+```bash
+
+# DSN for connecting to clickhouse with the monitoring user
+# Replace monitoring:<PASSWORD> with `username:password` for your monitoring user
+# Note: The monitoring user must be able to issue select queries on system.query_log table.
+export CLICKHOUSE_MONITORING_DSN="tcp://monitoring:<PASSWORD>@clickhouse:9000/"
+
+# If collecting query logs from a clustered deployment, specify a non-empty cluster name.
+export CLICKHOUSE_CLUSTER_NAME=""
+
+# Rows from query_log table will be collected periodically based on this setting
+export QUERY_LOG_SCRAPE_INTERVAL_SECONDS=20
+
+# Must be configured to a value greater than flush_interval_milliseconds setting for query_log.
+# This setting can be found in the clickhouse server config
+# For details see https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#query-log
+# Setting a large enough value ensures all query logs for a particular time interval have been
+# flushed before an attempt to collect them is made.
+export QUERY_LOG_SCRAPE_DELAY_SECONDS=8
+
+# region specific SigNoz cloud ingestion endpoint
+export OTLP_DESTINATION_ENDPOINT="ingest.us.signoz.cloud:443"
+
+# your SigNoz ingestion key
+export SIGNOZ_INGESTION_KEY="signoz-ingestion-key"
+
+```
+
+#### Use collector config file
+
+Make the collector config file available to your otel collector and use it by adding the following flag to the command for running your collector  
+```bash
+--config clickhouse-query-logs-collection-config.yaml
+```
+Note: the collector can use multiple config files, specified by multiple occurrences of the --config flag.
+

pkg/query-service/app/integrations/builtin_integrations/clickhouse/config/prerequisites.md

@@ -0,0 +1,42 @@
+## Before You Begin  
+
+To configure metrics and logs collection for a Clickhouse server, you need the following.
+
+### Ensure Clickhouse server is prepared for monitoring
+
+- **Ensure that the Clickhouse server is running a supported version**  
+  Clickhouse versions v23 and newer are supported.  
+  You can use the following SQL statement to determine server version  
+  ```SQL
+  SELECT version();
+  ```
+
+- **If collecting metrics, ensure that Clickhouse is configured to export prometheus metrics**
+  If needed, please [configure Clickhouse to expose prometheus metrics](https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings#prometheus).
+
+- **If collecting query_log, ensure that there is a clickhouse user with required permissions**
+  To create a monitoring user for clickhouse, you can run:
+  ```SQL
+  CREATE USER monitoring IDENTIFIED BY 'monitoring_password';
+  GRANT SELECT ON system.query_log to monitoring;
+
+  -- If monitoring a clustered deployment, also grant privilege for executing remote queries
+  GRANT REMOTE ON *.* TO 'monitoring' on CLUSTER 'cluster_name';
+  ```
+
+
+### Ensure OTEL Collector is running and has access to the Clickhouse server
+
+- **Ensure that an OTEL collector is running in your deployment environment**  
+  If needed, please [install SigNoz OTEL Collector](https://signoz.io/docs/tutorial/opentelemetry-binary-usage-in-virtual-machine/)  
+  If already installed, ensure that the collector version is v0.88.0 or newer.  
+  If collecting logs from system.query_log table, ensure that the collector version is v0.88.22 or newer.
+
+  Also ensure that you can provide config files to the collector and that you can set environment variables and command line flags used for running it.  
+
+- **Ensure that the OTEL collector can access the Clickhouse server**  
+  In order to collect metrics, the collector must be able to reach clickhouse server and access the port on which prometheus metrics are being exposed.
+
+  In order to collect server logs, the collector must be able to read the Clickhouse server log file.
+
+  In order to collect logs from query_log table, the collector must be able to reach the server and connect to it as a clickhouse user with required permissions.

pkg/query-service/app/integrations/builtin_integrations/clickhouse/icon.svg

@@ -0,0 +1,33 @@
+<svg version="1.1" id="Layer_1" xmlns:x="ns_extend;" xmlns:i="ns_ai;" xmlns:graph="ns_graphs;"
+  xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+  viewBox="0 0 50.6 50.6" style="enable-background:new 0 0 50.6 50.6;" xml:space="preserve">
+ <style type="text/css">
+  .st0{fill:#FFFFFF;}
+ </style>
+ <metadata>
+  <sfw xmlns="ns_sfw;">
+   <slices>
+   </slices>
+   <sliceSourceBounds bottomLeftOrigin="true" height="24" width="24" x="0" y="0">
+   </sliceSourceBounds>
+  </sfw>
+ </metadata>
+ <g>
+  <g>
+   <path class="st0" d="M0.6,0H5c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6H0.6C0.3,50.6,0,50.4,0,50V0.6C0,0.3,0.3,0,0.6,0z">
+   </path>
+   <path class="st0" d="M11.8,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6
+			C11.3,0.3,11.5,0,11.8,0z">
+   </path>
+   <path class="st0" d="M23.1,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6
+			C22.5,0.3,22.8,0,23.1,0z">
+   </path>
+   <path class="st0" d="M34.3,0h4.4c0.3,0,0.6,0.3,0.6,0.6V50c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V0.6
+			C33.7,0.3,34,0,34.3,0z">
+   </path>
+   <path class="st0" d="M45.6,19.7H50c0.3,0,0.6,0.3,0.6,0.6v10.1c0,0.3-0.3,0.6-0.6,0.6h-4.4c-0.3,0-0.6-0.3-0.6-0.6V20.3
+			C45,20,45.3,19.7,45.6,19.7z">
+   </path>
+  </g>
+ </g>
+</svg>

pkg/query-service/app/integrations/builtin_integrations/clickhouse/integration.json

@@ -0,0 +1,59 @@
+{
+  "id": "clickhouse",
+  "title": "Clickhouse",
+  "description": "Monitor Clickhouse with metrics and logs",
+  "author": {
+    "name": "SigNoz",
+    "email": "integrations@signoz.io",
+    "homepage": "https://signoz.io"
+  },
+  "icon": "file://icon.svg",
+  "categories": [
+    "Database"
+  ],
+  "overview": "file://overview.md",
+  "configuration": [
+    {
+      "title": "Prerequisites",
+      "instructions": "file://config/prerequisites.md"
+    },
+    {
+      "title": "Collect Metrics",
+      "instructions": "file://config/collect-metrics.md"
+    },
+    {
+      "title": "Collect Server Logs",
+      "instructions": "file://config/collect-logs.md"
+    },
+    {
+      "title": "Collect Query Logs",
+      "instructions": "file://config/collect-query-logs.md"
+    }
+  ],
+  "assets": {
+    "logs": {
+      "pipelines": []
+    },
+    "dashboards": [
+      "file://assets/dashboards/overview.json"
+    ],
+    "alerts": []
+  },
+  "connection_tests": {
+    "logs": {
+      "op": "AND",
+      "items": [
+        {
+          "key": {
+            "type": "tag",
+            "key": "source",
+            "dataType": "string"
+          },
+          "op": "=",
+          "value": "clickhouse"
+        }
+      ]
+    }
+  },
+  "data_collected": "file://data-collected.json"
+}

pkg/query-service/app/integrations/builtin_integrations/clickhouse/overview.md

@@ -0,0 +1,7 @@
+### Monitor Clickhouse with SigNoz
+
+Collect key Clickhouse metrics and view them with an out of the box dashboard.  
+
+Collect and parse Clickhouse logs to populate timestamp, severity, and other log attributes for better querying and aggregation.
+
+Collect clickhouse query logs from system.query_log table and view them in SigNoz

pkg/query-service/app/logs/v3/query_builder.go

@@ -252,6 +252,8 @@ func buildLogsQuery(panelType v3.PanelType, start, end, step int64, mq *v3.Build
 	} else if panelType == v3.PanelTypeTable {
 		queryTmpl =
 			"SELECT now() as ts,"
+		// step or aggregate interval is whole time period in case of table panel
+		step = (utils.GetEpochNanoSecs(end) - utils.GetEpochNanoSecs(start)) / 1000000000
 	} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
 		// Select the aggregate value for interval
 		queryTmpl =

pkg/query-service/app/logs/v3/query_builder_test.go

@@ -906,6 +906,23 @@ var testBuildLogsQueryData = []struct {
 		TableName:     "logs",
 		ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as `name`, toFloat64(count(*)) as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(attributes_string_key, 'name') group by `name` order by value DESC",
 	},
+	{
+		Name:      "TABLE: Test rate with groupBy",
+		PanelType: v3.PanelTypeTable,
+		Start:     1680066360726210000,
+		End:       1680066458000000000,
+		BuilderQuery: &v3.BuilderQuery{
+			QueryName:         "A",
+			StepInterval:      60,
+			AggregateOperator: v3.AggregateOperatorRate,
+			Expression:        "A",
+			GroupBy: []v3.AttributeKey{
+				{Key: "name", DataType: v3.AttributeKeyDataTypeString, Type: v3.AttributeKeyTypeTag},
+			},
+		},
+		TableName:     "logs",
+		ExpectedQuery: "SELECT now() as ts, attributes_string_value[indexOf(attributes_string_key, 'name')] as `name`, count()/97.000000 as value from signoz_logs.distributed_logs where (timestamp >= 1680066360726210000 AND timestamp <= 1680066458000000000) AND has(attributes_string_key, 'name') group by `name` order by value DESC",
+	},
 	{
 		Name:      "TABLE: Test count with groupBy, orderBy",
 		PanelType: v3.PanelTypeTable,

pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml

@@ -52,10 +52,10 @@ extensions:
   zpages: {}
 exporters:
   clickhousetraces:
-    datasource: tcp://localhost:9000/?database=signoz_traces
+    datasource: tcp://localhost:9000/signoz_traces
     migrations: exporter/clickhousetracesexporter/migrations
   clickhousemetricswrite:
-    endpoint: tcp://localhost:9000/?database=signoz_metrics
+    endpoint: tcp://localhost:9000/signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
   prometheus:

pkg/query-service/app/traces/v3/query_builder.go

@@ -262,6 +262,8 @@ func buildTracesQuery(start, end, step int64, mq *v3.BuilderQuery, tableName str
 	} else if panelType == v3.PanelTypeTable {
 		queryTmpl =
 			"SELECT now() as ts,"
+		// step or aggregate interval is whole time period in case of table panel
+		step = (end*getZerosForEpochNano(end) - start*getZerosForEpochNano(start))/1000000000
 	} else if panelType == v3.PanelTypeGraph || panelType == v3.PanelTypeValue {
 		// Select the aggregate value for interval
 		queryTmpl =

pkg/query-service/app/traces/v3/query_builder_test.go

@@ -1017,7 +1017,7 @@ var testBuildTracesQueryData = []struct {
 		PanelType: v3.PanelTypeValue,
 	},
 	{
-		Name:  "Test aggregate PXX",
+		Name:  "Test aggregate PXX with groupby",
 		Start: 1680066360726210000,
 		End:   1680066458000000000,
 		BuilderQuery: &v3.BuilderQuery{
@@ -1059,6 +1059,26 @@ var testBuildTracesQueryData = []struct {
 			"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')",
 		PanelType: v3.PanelTypeTable,
 	},
+	{
+		Name:  "Test aggregate rate table panel",
+		Start: 1680066360726210000,
+		End:   1680066458000000000,
+		BuilderQuery: &v3.BuilderQuery{
+			QueryName:          "A",
+			StepInterval:       60,
+			AggregateAttribute: v3.AttributeKey{Key: "durationNano", IsColumn: true, DataType: v3.AttributeKeyDataTypeFloat64, Type: v3.AttributeKeyTypeTag},
+			AggregateOperator:  v3.AggregateOperatorRate,
+			Expression:         "A",
+			Filters:            &v3.FilterSet{Operator: "AND", Items: []v3.FilterItem{}},
+			GroupBy:            []v3.AttributeKey{},
+			OrderBy:            []v3.OrderBy{},
+		},
+		TableName: "signoz_traces.distributed_signoz_index_v2",
+		ExpectedQuery: "SELECT now() as ts, count(durationNano)/97.000000 as value " +
+			"from signoz_traces.distributed_signoz_index_v2 " +
+			"where (timestamp >= '1680066360726210000' AND timestamp <= '1680066458000000000')",
+		PanelType: v3.PanelTypeTable,
+	},
 	{
 		Name:  "Test Noop list view",
 		Start: 1680066360726210000,

pkg/query-service/config/prometheus.yml

@@ -22,4 +22,4 @@ rule_files:
 scrape_configs: []
 
 remote_read:
-  - url: tcp://localhost:9000/?database=signoz_metrics
+  - url: tcp://localhost:9000/signoz_metrics

pkg/query-service/tests/test-deploy/docker-compose.yaml

@@ -20,7 +20,7 @@ x-clickhouse-defaults: &clickhouse-defaults
         "wget",
         "--spider",
         "-q",
-        "localhost:8123/ping"
+        "0.0.0.0:8123/ping"
       ]
     interval: 30s
     timeout: 5s
@@ -192,7 +192,7 @@ services:
     <<: *db-depend
 
   otel-collector-migrator:
-    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.20}
+    image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.88.21}
     container_name: otel-migrator
     command:
       - "--dsn=tcp://clickhouse:9000"
@@ -205,7 +205,7 @@ services:
       #   condition: service_healthy
 
   otel-collector:
-    image: signoz/signoz-otel-collector:0.88.20
+    image: signoz/signoz-otel-collector:0.88.21
     container_name: signoz-otel-collector
     command:
       [

pkg/query-service/tests/test-deploy/otel-collector-config.yaml

@@ -101,21 +101,20 @@ extensions:
 
 exporters:
   clickhousetraces:
-    datasource: tcp://clickhouse:9000/?database=signoz_traces
+    datasource: tcp://clickhouse:9000/signoz_traces
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     low_cardinal_exception_grouping: ${LOW_CARDINAL_EXCEPTION_GROUPING}
   clickhousemetricswrite:
-    endpoint: tcp://clickhouse:9000/?database=signoz_metrics
+    endpoint: tcp://clickhouse:9000/signoz_metrics
     resource_to_telemetry_conversion:
       enabled: true
   prometheus:
     endpoint: 0.0.0.0:8889
-  # logging: {}
-
   clickhouselogsexporter:
-    dsn: tcp://clickhouse:9000/
+    dsn: tcp://clickhouse:9000/signoz_logs
     docker_multi_node_cluster: ${DOCKER_MULTI_NODE_CLUSTER}
     timeout: 10s
+  # logging: {}
 
 service:
   telemetry:

pkg/query-service/tests/test-deploy/prometheus.yml

@@ -22,4 +22,4 @@ rule_files:
 scrape_configs: []
 
 remote_read:
-  - url: tcp://clickhouse:9000/?database=signoz_metrics
+  - url: tcp://clickhouse:9000/signoz_metrics