From d539ca9bab863235da7320e7729f77d8016ce68a Mon Sep 17 00:00:00 2001 From: Vikrant Gupta Date: Tue, 21 Oct 2025 18:45:48 +0530 Subject: [PATCH] feat(sql): swap mattn/sqlite with modernc.org/sqlite (#9343) * feat(sql): swap mattn/sqlite with modernc.org/sqlite (#9325) * feat(sql): swap mattn/sqlite with modernc.org/sqlite * feat(sql): revert the dashboard testing changes * feat(sql): enable WAL mode for sqlite * feat(sql): revert enable WAL mode for sqlite * feat(sql): use sensible defaults for busy_timeout * feat(sql): add ldflags * feat(sql): enable WAL mode for sqlite * feat(sql): some fixes * feat(sql): some fixes * feat(sql): fix yarn lock and config defaults * feat(sql): update the defaults in example.conf * feat(sql): remove wal mode from integration tests --- .github/workflows/build-community.yaml | 7 +++-- .github/workflows/build-enterprise.yaml | 5 ++-- .github/workflows/build-staging.yaml | 5 ++-- Makefile | 12 ++++----- cmd/community/.goreleaser.yaml | 8 ------ cmd/enterprise/.goreleaser.yaml | 8 ------ conf/example.yaml | 9 ++++--- docs/contributing/development.md | 2 -- frontend/yarn.lock | 2 +- go.mod | 7 +++-- go.sum | 26 +++++++++---------- pkg/query-service/app/http_handler.go | 2 +- pkg/sqlstore/config.go | 12 ++++++++- pkg/sqlstore/sqlitesqlstore/provider.go | 17 +++++++++--- .../histogramquantile/.goreleaser.yaml | 2 -- tests/integration/fixtures/signoz.py | 6 +++-- tests/integration/fixtures/sqlite.py | 4 ++- .../src/passwordauthn/b_license.py | 16 ++++++------ .../src/passwordauthn/d_password.py | 1 + 19 files changed, 76 insertions(+), 75 deletions(-) diff --git a/.github/workflows/build-community.yaml b/.github/workflows/build-community.yaml index d8d1df3a62..22d80e4a42 100644 --- a/.github/workflows/build-community.yaml +++ b/.github/workflows/build-community.yaml @@ -3,8 +3,8 @@ name: build-community on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+" + - "v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+" defaults: run: @@ -69,14 +69,13 @@ jobs: GO_BUILD_CONTEXT: ./cmd/community GO_BUILD_FLAGS: >- -tags timetzdata - -ldflags='-linkmode external -extldflags \"-static\" -s -w + -ldflags='-s -w -X github.com/SigNoz/signoz/pkg/version.version=${{ needs.prepare.outputs.version }} -X github.com/SigNoz/signoz/pkg/version.variant=community -X github.com/SigNoz/signoz/pkg/version.hash=${{ needs.prepare.outputs.hash }} -X github.com/SigNoz/signoz/pkg/version.time=${{ needs.prepare.outputs.time }} -X github.com/SigNoz/signoz/pkg/version.branch=${{ needs.prepare.outputs.branch }} -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' - GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./cmd/community/Dockerfile.multi-arch DOCKER_MANIFEST: true diff --git a/.github/workflows/build-enterprise.yaml b/.github/workflows/build-enterprise.yaml index 96d239a387..4572dcfa21 100644 --- a/.github/workflows/build-enterprise.yaml +++ b/.github/workflows/build-enterprise.yaml @@ -84,7 +84,7 @@ jobs: JS_INPUT_ARTIFACT_CACHE_KEY: enterprise-dotenv-${{ github.sha }} JS_INPUT_ARTIFACT_PATH: frontend/.env JS_OUTPUT_ARTIFACT_CACHE_KEY: enterprise-jsbuild-${{ github.sha }} - JS_OUTPUT_ARTIFACT_PATH: frontend/build + JS_OUTPUT_ARTIFACT_PATH: frontend/build DOCKER_BUILD: false DOCKER_MANIFEST: false go-build: @@ -99,7 +99,7 @@ jobs: GO_BUILD_CONTEXT: ./cmd/enterprise GO_BUILD_FLAGS: >- -tags timetzdata - -ldflags='-linkmode external -extldflags \"-static\" -s -w + -ldflags='-s -w -X github.com/SigNoz/signoz/pkg/version.version=${{ needs.prepare.outputs.version }} -X github.com/SigNoz/signoz/pkg/version.variant=enterprise -X github.com/SigNoz/signoz/pkg/version.hash=${{ needs.prepare.outputs.hash }} @@ -110,7 +110,6 @@ jobs: -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.signoz.cloud -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.signoz.io/api/v1 -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' - GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./cmd/enterprise/Dockerfile.multi-arch DOCKER_MANIFEST: true diff --git a/.github/workflows/build-staging.yaml b/.github/workflows/build-staging.yaml index 271a270ac5..0ce065794b 100644 --- a/.github/workflows/build-staging.yaml +++ b/.github/workflows/build-staging.yaml @@ -98,7 +98,7 @@ jobs: GO_BUILD_CONTEXT: ./cmd/enterprise GO_BUILD_FLAGS: >- -tags timetzdata - -ldflags='-linkmode external -extldflags \"-static\" -s -w + -ldflags='-s -w -X github.com/SigNoz/signoz/pkg/version.version=${{ needs.prepare.outputs.version }} -X github.com/SigNoz/signoz/pkg/version.variant=enterprise -X github.com/SigNoz/signoz/pkg/version.hash=${{ needs.prepare.outputs.hash }} @@ -109,7 +109,6 @@ jobs: -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.staging.signoz.cloud -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.staging.signoz.cloud/api/v1 -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr' - GO_CGO_ENABLED: 1 DOCKER_BASE_IMAGES: '{"alpine": "alpine:3.20.3"}' DOCKER_DOCKERFILE_PATH: ./cmd/enterprise/Dockerfile.multi-arch DOCKER_MANIFEST: true @@ -125,4 +124,4 @@ jobs: GITHUB_SILENT: true GITHUB_REPOSITORY_NAME: charts-saas-v3-staging GITHUB_EVENT_NAME: releaser - GITHUB_EVENT_PAYLOAD: "{\"deployment\": \"${{ needs.prepare.outputs.deployment }}\", \"signoz_version\": \"${{ needs.prepare.outputs.version }}\"}" + GITHUB_EVENT_PAYLOAD: '{"deployment": "${{ needs.prepare.outputs.deployment }}", "signoz_version": "${{ needs.prepare.outputs.version }}"}' diff --git a/Makefile b/Makefile index 415dc82385..561078b2dd 100644 --- a/Makefile +++ b/Makefile @@ -114,9 +114,9 @@ $(GO_BUILD_ARCHS_COMMUNITY): go-build-community-%: $(TARGET_DIR) @mkdir -p $(TARGET_DIR)/$(OS)-$* @echo ">> building binary $(TARGET_DIR)/$(OS)-$*/$(NAME)-community" @if [ $* = "arm64" ]; then \ - CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_COMMUNITY) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME)-community -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_COMMUNITY)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_COMMUNITY) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME)-community -ldflags "-s -w $(GO_BUILD_LDFLAGS_COMMUNITY)"; \ else \ - CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_COMMUNITY) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME)-community -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_COMMUNITY)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_COMMUNITY) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME)-community -ldflags "-s -w $(GO_BUILD_LDFLAGS_COMMUNITY)"; \ fi @@ -127,9 +127,9 @@ $(GO_BUILD_ARCHS_ENTERPRISE): go-build-enterprise-%: $(TARGET_DIR) @mkdir -p $(TARGET_DIR)/$(OS)-$* @echo ">> building binary $(TARGET_DIR)/$(OS)-$*/$(NAME)" @if [ $* = "arm64" ]; then \ - CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ else \ - CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ fi .PHONY: go-build-enterprise-race $(GO_BUILD_ARCHS_ENTERPRISE_RACE) @@ -139,9 +139,9 @@ $(GO_BUILD_ARCHS_ENTERPRISE_RACE): go-build-enterprise-race-%: $(TARGET_DIR) @mkdir -p $(TARGET_DIR)/$(OS)-$* @echo ">> building binary $(TARGET_DIR)/$(OS)-$*/$(NAME)" @if [ $* = "arm64" ]; then \ - CC=aarch64-linux-gnu-gcc CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -race -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -race -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ else \ - CGO_ENABLED=1 GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -race -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-linkmode external -extldflags '-static' -s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ + GOARCH=$* GOOS=$(OS) go build -C $(GO_BUILD_CONTEXT_ENTERPRISE) -race -tags timetzdata -o $(TARGET_DIR)/$(OS)-$*/$(NAME) -ldflags "-s -w $(GO_BUILD_LDFLAGS_ENTERPRISE)"; \ fi ############################################################## diff --git a/cmd/community/.goreleaser.yaml b/cmd/community/.goreleaser.yaml index 74d165d7f4..dc75295f07 100644 --- a/cmd/community/.goreleaser.yaml +++ b/cmd/community/.goreleaser.yaml @@ -12,12 +12,6 @@ builds: - id: signoz binary: bin/signoz main: ./cmd/community - env: - - CGO_ENABLED=1 - - >- - {{- if eq .Os "linux" }} - {{- if eq .Arch "arm64" }}CC=aarch64-linux-gnu-gcc{{- end }} - {{- end }} goos: - linux - darwin @@ -36,8 +30,6 @@ builds: - -X github.com/SigNoz/signoz/pkg/version.time={{ .CommitTimestamp }} - -X github.com/SigNoz/signoz/pkg/version.branch={{ .Branch }} - -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr - - >- - {{- if eq .Os "linux" }}-linkmode external -extldflags '-static'{{- end }} mod_timestamp: "{{ .CommitTimestamp }}" tags: - timetzdata diff --git a/cmd/enterprise/.goreleaser.yaml b/cmd/enterprise/.goreleaser.yaml index d5647b2f49..824213c074 100644 --- a/cmd/enterprise/.goreleaser.yaml +++ b/cmd/enterprise/.goreleaser.yaml @@ -12,12 +12,6 @@ builds: - id: signoz binary: bin/signoz main: ./cmd/enterprise - env: - - CGO_ENABLED=1 - - >- - {{- if eq .Os "linux" }} - {{- if eq .Arch "arm64" }}CC=aarch64-linux-gnu-gcc{{- end }} - {{- end }} goos: - linux - darwin @@ -40,8 +34,6 @@ builds: - -X github.com/SigNoz/signoz/ee/query-service/constants.ZeusURL=https://api.signoz.cloud - -X github.com/SigNoz/signoz/ee/query-service/constants.LicenseSignozIo=https://license.signoz.io/api/v1 - -X github.com/SigNoz/signoz/pkg/analytics.key=9kRrJ7oPCGPEJLF6QjMPLt5bljFhRQBr - - >- - {{- if eq .Os "linux" }}-linkmode external -extldflags '-static'{{- end }} mod_timestamp: "{{ .CommitTimestamp }}" tags: - timetzdata diff --git a/conf/example.yaml b/conf/example.yaml index 9ace9702b6..225019eb01 100644 --- a/conf/example.yaml +++ b/conf/example.yaml @@ -1,5 +1,5 @@ ##################### SigNoz Configuration Example ##################### -# +# # Do not modify this file # @@ -58,7 +58,7 @@ cache: # The port on which the Redis server is running. Default is usually 6379. port: 6379 # The password for authenticating with the Redis server, if required. - password: + password: # The Redis database number to use db: 0 @@ -71,6 +71,10 @@ sqlstore: sqlite: # The path to the SQLite database file. path: /var/lib/signoz/signoz.db + # Mode is the mode to use for the sqlite database. + mode: delete + # BusyTimeout is the timeout for the sqlite database to wait for a lock. + busy_timeout: 10s ##################### APIServer ##################### apiserver: @@ -238,7 +242,6 @@ statsreporter: # Whether to collect identities and traits (emails). identities: true - ##################### Gateway (License only) ##################### gateway: # The URL of the gateway's api. diff --git a/docs/contributing/development.md b/docs/contributing/development.md index 41f51567b1..4ef73c31ca 100644 --- a/docs/contributing/development.md +++ b/docs/contributing/development.md @@ -13,8 +13,6 @@ Before diving in, make sure you have these tools installed: - Download from [go.dev/dl](https://go.dev/dl/) - Check [go.mod](../../go.mod#L3) for the minimum version -- **GCC** - Required for CGO dependencies - - Download from [gcc.gnu.org](https://gcc.gnu.org/) - **Node** - Powers our frontend - Download from [nodejs.org](https://nodejs.org) diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 684b9dd52e..e66970423d 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -13752,7 +13752,7 @@ on-finished@2.4.1, on-finished@^2.4.1: dependencies: ee-first "1.1.1" -on-headers@1.1.0, on-headers@~1.0.2: +on-headers@^1.1.0, on-headers@~1.0.2: version "1.1.0" resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.1.0.tgz#59da4f91c45f5f989c6e4bcedc5a3b0aed70ff65" integrity sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A== diff --git a/go.mod b/go.mod index bf72bda027..6d110d9126 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,6 @@ require ( github.com/knadh/koanf v1.5.0 github.com/knadh/koanf/v2 v2.2.0 github.com/mailru/easyjson v0.7.7 - github.com/mattn/go-sqlite3 v1.14.24 github.com/open-telemetry/opamp-go v0.19.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.128.0 github.com/openfga/api/proto v0.0.0-20250909172242-b4b2a12f5c67 @@ -84,6 +83,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.34.0 + modernc.org/sqlite v1.39.1 ) require ( @@ -93,10 +93,9 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/uptrace/opentelemetry-go-extra/otelsql v0.3.2 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect - modernc.org/libc v1.66.3 // indirect + modernc.org/libc v1.66.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.39.0 // indirect ) require ( @@ -330,7 +329,7 @@ require ( go.uber.org/mock v0.6.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.27.0 // indirect - golang.org/x/sys v0.35.0 // indirect + golang.org/x/sys v0.36.0 // indirect golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.36.0 // indirect gonum.org/v1/gonum v0.16.0 // indirect diff --git a/go.sum b/go.sum index 804ffc42a2..aa4bdf0ac6 100644 --- a/go.sum +++ b/go.sum @@ -680,8 +680,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= -github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -1461,8 +1459,8 @@ golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= @@ -1785,18 +1783,18 @@ k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOP k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -modernc.org/cc/v4 v4.26.2 h1:991HMkLjJzYBIfha6ECZdjrIYz2/1ayr+FL8GN+CNzM= -modernc.org/cc/v4 v4.26.2/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= -modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.8 h1:qtzNm7ED75pd1C7WgAGcK4edm4fvhtBsEiI/0NQ54YM= -modernc.org/fileutil v1.3.8/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= -modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= -modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -1805,8 +1803,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.39.0 h1:6bwu9Ooim0yVYA7IZn9demiQk/Ejp0BtTjBWFLymSeY= -modernc.org/sqlite v1.39.0/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/sqlite v1.39.1 h1:H+/wGFzuSCIEVCvXYVHX5RQglwhMOvtHSv+VtidL2r4= +modernc.org/sqlite v1.39.1/go.mod h1:9fjQZ0mB1LLP0GYrp39oOJXx/I2sxEnZtzCmEQIKvGE= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 781b7685c7..be3646415f 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -38,7 +38,7 @@ import ( "github.com/gorilla/mux" "github.com/gorilla/websocket" jsoniter "github.com/json-iterator/go" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/SigNoz/signoz/pkg/contextlinks" traceFunnelsModule "github.com/SigNoz/signoz/pkg/modules/tracefunnel" diff --git a/pkg/sqlstore/config.go b/pkg/sqlstore/config.go index 97e58dc897..7ca733cacc 100644 --- a/pkg/sqlstore/config.go +++ b/pkg/sqlstore/config.go @@ -1,6 +1,8 @@ package sqlstore import ( + "time" + "github.com/SigNoz/signoz/pkg/factory" ) @@ -23,6 +25,12 @@ type PostgresConfig struct { type SqliteConfig struct { // Path is the path to the sqlite database. Path string `mapstructure:"path"` + + // Mode is the mode to use for the sqlite database. + Mode string `mapstructure:"mode"` + + // BusyTimeout is the timeout for the sqlite database to wait for a lock. + BusyTimeout time.Duration `mapstructure:"busy_timeout"` } type ConnectionConfig struct { @@ -41,7 +49,9 @@ func newConfig() factory.Config { MaxOpenConns: 100, }, Sqlite: SqliteConfig{ - Path: "/var/lib/signoz/signoz.db", + Path: "/var/lib/signoz/signoz.db", + Mode: "delete", + BusyTimeout: 10000 * time.Millisecond, // increasing the defaults from https://github.com/mattn/go-sqlite3/blob/master/sqlite3.go#L1098 because of transpilation from C to GO }, } diff --git a/pkg/sqlstore/sqlitesqlstore/provider.go b/pkg/sqlstore/sqlitesqlstore/provider.go index ca61bf9f30..e778f3cf0f 100644 --- a/pkg/sqlstore/sqlitesqlstore/provider.go +++ b/pkg/sqlstore/sqlitesqlstore/provider.go @@ -3,13 +3,17 @@ package sqlitesqlstore import ( "context" "database/sql" + "fmt" + "net/url" "github.com/SigNoz/signoz/pkg/errors" "github.com/SigNoz/signoz/pkg/factory" "github.com/SigNoz/signoz/pkg/sqlstore" - sqlite3 "github.com/mattn/go-sqlite3" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect/sqlitedialect" + + "modernc.org/sqlite" + sqlite3 "modernc.org/sqlite/lib" ) type provider struct { @@ -38,7 +42,12 @@ func NewFactory(hookFactories ...factory.ProviderFactory[sqlstore.SQLStoreHook, func New(ctx context.Context, providerSettings factory.ProviderSettings, config sqlstore.Config, hooks ...sqlstore.SQLStoreHook) (sqlstore.SQLStore, error) { settings := factory.NewScopedProviderSettings(providerSettings, "github.com/SigNoz/signoz/pkg/sqlitesqlstore") - sqldb, err := sql.Open("sqlite3", "file:"+config.Sqlite.Path+"?_foreign_keys=true") + connectionParams := url.Values{} + // do not update the order of the connection params as busy_timeout doesn't work if it's not the first parameter + connectionParams.Add("_pragma", fmt.Sprintf("busy_timeout(%d)", config.Sqlite.BusyTimeout.Milliseconds())) + connectionParams.Add("_pragma", fmt.Sprintf("journal_mode(%s)", config.Sqlite.Mode)) + connectionParams.Add("_pragma", "foreign_keys(1)") + sqldb, err := sql.Open("sqlite", "file:"+config.Sqlite.Path+"?"+connectionParams.Encode()) if err != nil { return nil, err } @@ -82,8 +91,8 @@ func (provider *provider) WrapNotFoundErrf(err error, code errors.Code, format s } func (provider *provider) WrapAlreadyExistsErrf(err error, code errors.Code, format string, args ...any) error { - if sqlite3Err, ok := err.(sqlite3.Error); ok { - if sqlite3Err.ExtendedCode == sqlite3.ErrConstraintUnique { + if sqlite3Err, ok := err.(*sqlite.Error); ok { + if sqlite3Err.Code() == sqlite3.SQLITE_CONSTRAINT_UNIQUE || sqlite3Err.Code() == sqlite3.SQLITE_CONSTRAINT_PRIMARYKEY { return errors.Wrapf(err, errors.TypeAlreadyExists, code, format, args...) } } diff --git a/scripts/clickhouse/histogramquantile/.goreleaser.yaml b/scripts/clickhouse/histogramquantile/.goreleaser.yaml index 49d3f5f4d1..d4b0fec047 100644 --- a/scripts/clickhouse/histogramquantile/.goreleaser.yaml +++ b/scripts/clickhouse/histogramquantile/.goreleaser.yaml @@ -15,8 +15,6 @@ builds: - id: signoz binary: bin/histogram-quantile main: scripts/clickhouse/histogramquantile/main.go - env: - - CGO_ENABLED=0 goos: - linux - darwin diff --git a/tests/integration/fixtures/signoz.py b/tests/integration/fixtures/signoz.py index d0f350c8ff..0110515732 100644 --- a/tests/integration/fixtures/signoz.py +++ b/tests/integration/fixtures/signoz.py @@ -1,3 +1,4 @@ +from os import path import platform import time from http import HTTPStatus @@ -68,9 +69,10 @@ def signoz( # pylint: disable=too-many-arguments,too-many-positional-arguments provider = request.config.getoption("--sqlstore-provider") if provider == "sqlite": + dir_path = path.dirname(sqlstore.env["SIGNOZ_SQLSTORE_SQLITE_PATH"]) container.with_volume_mapping( - sqlstore.env["SIGNOZ_SQLSTORE_SQLITE_PATH"], - sqlstore.env["SIGNOZ_SQLSTORE_SQLITE_PATH"], + dir_path, + dir_path, "rw", ) diff --git a/tests/integration/fixtures/sqlite.py b/tests/integration/fixtures/sqlite.py index bcf1b9bbd9..1ad33e3da8 100644 --- a/tests/integration/fixtures/sqlite.py +++ b/tests/integration/fixtures/sqlite.py @@ -27,6 +27,7 @@ def sqlite( with engine.connect() as conn: result = conn.execute(sql.text("SELECT 1")) assert result.fetchone()[0] == 1 + return types.TestContainerSQL( container=types.TestContainerDocker( @@ -52,13 +53,14 @@ def sqlite( result = conn.execute(sql.text("SELECT 1")) assert result.fetchone()[0] == 1 + return types.TestContainerSQL( container=types.TestContainerDocker( id="", host_configs={}, container_configs={}, ), - conn=conn, + conn=engine, env=cache["env"], ) diff --git a/tests/integration/src/passwordauthn/b_license.py b/tests/integration/src/passwordauthn/b_license.py index 61b475c999..d5d850c619 100644 --- a/tests/integration/src/passwordauthn/b_license.py +++ b/tests/integration/src/passwordauthn/b_license.py @@ -131,14 +131,14 @@ def test_refresh_license( assert response.status_code == http.HTTPStatus.NO_CONTENT - with signoz.sqlstore.conn.connect() as conn: - result = conn.execute( - sql.text("SELECT data FROM license WHERE id=:id"), - {"id": "0196360e-90cd-7a74-8313-1aa815ce2a67"}, - ) - record = result.fetchone()[0] - assert json.loads(record)["valid_from"] == 1732146922 - + response = requests.get( + url=signoz.self.host_configs["8080"].get("/api/v3/licenses/active"), + headers={"Authorization": "Bearer " + access_token}, + timeout=5, + ) + assert response.status_code == http.HTTPStatus.OK + assert response.json()["data"]["valid_from"] == 1732146922 + response = requests.post( url=signoz.zeus.host_configs["8080"].get("/__admin/requests/count"), json={"method": "GET", "url": "/v2/licenses/me"}, diff --git a/tests/integration/src/passwordauthn/d_password.py b/tests/integration/src/passwordauthn/d_password.py index 1b87b235ad..1779c5f717 100644 --- a/tests/integration/src/passwordauthn/d_password.py +++ b/tests/integration/src/passwordauthn/d_password.py @@ -185,6 +185,7 @@ def test_reset_password( assert token is not None + def test_reset_password_with_no_password( signoz: types.SigNoz, get_token: Callable[[str, str], str] ) -> None: