Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(config): refactor config package #6782

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

refactor(config): refactor config package #6782

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ae5ba2b
refactor(config): refactor config package
grandwizard28 Jan 8, 2025
4c94386
refactor(migrations): move migrations into a single package
grandwizard28 Jan 9, 2025
5d26d2b
fix(tests): fix unit tests
grandwizard28 Jan 9, 2025
0c3af05
feat(signoz): remove references to sqlite
grandwizard28 Jan 9, 2025
e1cb905
feat(signoz): make mattn the permanent driver
grandwizard28 Jan 9, 2025
2060905
feat(signoz): refactor connection config struct
grandwizard28 Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions conf/defaults.yaml → conf/example.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

##################### Web #####################
web:
# Whether to enable the web frontend
enabled: true
# The prefix to serve web on
prefix: /
# The directory containing the static build files.
Expand Down
52 changes: 17 additions & 35 deletions ee/query-service/app/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ import (
const AppDbEngine = "sqlite"

type ServerOptions struct {
SigNoz *signoz.SigNoz
PromConfigPath string
SkipTopLvlOpsPath string
HTTPHostPort string
Expand All @@ -81,7 +80,6 @@ type ServerOptions struct {
GatewayUrl string
UseLogsNewSchema bool
UseTraceNewSchema bool
SkipWebFrontend bool
}

// Server runs HTTP api service
Expand Down Expand Up @@ -111,34 +109,23 @@ func (s Server) HealthCheckStatus() chan healthcheck.Status {
}

// NewServer creates and initializes Server
func NewServer(serverOptions *ServerOptions) (*Server, error) {

modelDao, err := dao.InitDao("sqlite", baseconst.RELATIONAL_DATASOURCE_PATH)
if err != nil {
return nil, err
}

baseexplorer.InitWithDSN(baseconst.RELATIONAL_DATASOURCE_PATH)

if err := preferences.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH); err != nil {
return nil, err
}

localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH)

func NewServer(serverOptions *ServerOptions, config signoz.Config, signoz *signoz.SigNoz) (*Server, error) {
modelDao, err := dao.InitDao(signoz.SqlStore.Provider().SqlxDB())
if err != nil {
return nil, err
}

localDB.SetMaxOpenConns(10)
baseexplorer.InitWithDB(signoz.SqlStore.Provider().SqlxDB())
preferences.InitDB(signoz.SqlStore.Provider().SqlxDB())
dashboards.InitDB(signoz.SqlStore.Provider().SqlxDB())

gatewayProxy, err := gateway.NewProxy(serverOptions.GatewayUrl, gateway.RoutePrefix)
if err != nil {
return nil, err
}

// initiate license manager
lm, err := licensepkg.StartManager("sqlite", localDB)
lm, err := licensepkg.StartManager("sqlite", signoz.SqlStore.Provider().SqlxDB())
if err != nil {
return nil, err
}
Expand All @@ -152,7 +139,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
if storage == "clickhouse" {
zap.L().Info("Using ClickHouse as datastore ...")
qb := db.NewDataConnector(
localDB,
signoz.SqlStore.Provider().SqlxDB(),
serverOptions.PromConfigPath,
lm,
serverOptions.MaxIdleConns,
Expand Down Expand Up @@ -188,7 +175,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
rm, err := makeRulesManager(serverOptions.PromConfigPath,
baseconst.GetAlertManagerApiPrefix(),
serverOptions.RuleRepoURL,
localDB,
signoz.SqlStore.Provider().SqlxDB(),
reader,
c,
serverOptions.DisableRules,
Expand All @@ -209,12 +196,9 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
}()

// initiate opamp
_, err = opAmpModel.InitDB(localDB)
if err != nil {
return nil, err
}
_ = opAmpModel.InitDB(signoz.SqlStore.Provider().SqlxDB())

integrationsController, err := integrations.NewController(localDB)
integrationsController, err := integrations.NewController(signoz.SqlStore.Provider().SqlxDB())
if err != nil {
return nil, fmt.Errorf(
"couldn't create integrations controller: %w", err,
Expand All @@ -223,15 +207,15 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {

// ingestion pipelines manager
logParsingPipelineController, err := logparsingpipeline.NewLogParsingPipelinesController(
localDB, "sqlite", integrationsController.GetPipelinesForInstalledIntegrations,
signoz.SqlStore.Provider().SqlxDB(), "sqlite", integrationsController.GetPipelinesForInstalledIntegrations,
)
if err != nil {
return nil, err
}

// initiate agent config handler
agentConfMgr, err := agentConf.Initiate(&agentConf.ManagerOptions{
DB: localDB,
DB: signoz.SqlStore.Provider().SqlxDB(),
DBEngine: AppDbEngine,
AgentFeatures: []agentConf.AgentFeature{logParsingPipelineController},
})
Expand Down Expand Up @@ -293,7 +277,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) {
usageManager: usageManager,
}

httpServer, err := s.createPublicServer(apiHandler, serverOptions.SigNoz.Web)
httpServer, err := s.createPublicServer(apiHandler, signoz.Web)

if err != nil {
return nil, err
Expand Down Expand Up @@ -342,7 +326,7 @@ func (s *Server) createPrivateServer(apiHandler *api.APIHandler) (*http.Server,
}, nil
}

func (s *Server) createPublicServer(apiHandler *api.APIHandler, web *web.Web) (*http.Server, error) {
func (s *Server) createPublicServer(apiHandler *api.APIHandler, web web.Web) (*http.Server, error) {

r := baseapp.NewRouter()

Expand Down Expand Up @@ -386,11 +370,9 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler, web *web.Web) (*

handler = handlers.CompressHandler(handler)

if !s.serverOptions.SkipWebFrontend {
err := web.AddToRouter(r)
if err != nil {
return nil, err
}
err := web.AddToRouter(r)
if err != nil {
return nil, err
}

return &http.Server{
Expand Down
13 changes: 3 additions & 10 deletions ee/query-service/dao/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,18 +1,11 @@
package dao

import (
"fmt"

"github.com/jmoiron/sqlx"
"go.signoz.io/signoz/ee/query-service/dao/sqlite"
)

func InitDao(engine, path string) (ModelDao, error) {

switch engine {
case "sqlite":
return sqlite.InitDB(path)
default:
return nil, fmt.Errorf("qsdb type: %s is not supported in query service", engine)
}
func InitDao(inputDB *sqlx.DB) (ModelDao, error) {
return sqlite.InitDB(inputDB)

}
4 changes: 2 additions & 2 deletions ee/query-service/dao/sqlite/modelDao.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,8 @@ func columnExists(db *sqlx.DB, tableName, columnName string) bool {
}

// InitDB creates and extends base model DB repository
func InitDB(dataSourceName string) (*modelDao, error) {
dao, err := basedsql.InitDB(dataSourceName)
func InitDB(inputDB *sqlx.DB) (*modelDao, error) {
dao, err := basedsql.InitDB(inputDB)
if err != nil {
return nil, err
}
Expand Down
29 changes: 8 additions & 21 deletions ee/query-service/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,14 @@ import (
"syscall"
"time"

"go.opentelemetry.io/collector/confmap"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
"go.signoz.io/signoz/ee/query-service/app"
"go.signoz.io/signoz/pkg/config"
signozconfig "go.signoz.io/signoz/pkg/config"
"go.signoz.io/signoz/pkg/confmap/provider/signozenvprovider"
"go.signoz.io/signoz/pkg/config/provider/envprovider"
"go.signoz.io/signoz/pkg/query-service/auth"
baseconst "go.signoz.io/signoz/pkg/query-service/constants"
"go.signoz.io/signoz/pkg/query-service/migrate"
"go.signoz.io/signoz/pkg/query-service/version"
"go.signoz.io/signoz/pkg/signoz"
"google.golang.org/grpc"
Expand Down Expand Up @@ -108,7 +107,6 @@ func main() {
var dialTimeout time.Duration
var gatewayUrl string
var useLicensesV3 bool
var skipWebFrontend bool

flag.BoolVar(&useLogsNewSchema, "use-logs-new-schema", false, "use logs_v2 schema for logs")
flag.BoolVar(&useTraceNewSchema, "use-trace-new-schema", false, "use new schema for traces")
Expand All @@ -126,7 +124,6 @@ func main() {
flag.StringVar(&cluster, "cluster", "cluster", "(cluster name - defaults to 'cluster')")
flag.StringVar(&gatewayUrl, "gateway-url", "", "(url to the gateway)")
flag.BoolVar(&useLicensesV3, "use-licenses-v3", false, "use licenses_v3 schema for licenses")
flag.BoolVar(&skipWebFrontend, "skip-web-frontend", false, "skip web frontend")
flag.Parse()

loggerMgr := initZapLog(enableQueryServiceLogOTLPExport)
Expand All @@ -136,25 +133,22 @@ func main() {

version.PrintVersion()

config, err := signozconfig.New(context.Background(), signozconfig.ProviderSettings{
ResolverSettings: confmap.ResolverSettings{
URIs: []string{"signozenv:"},
ProviderFactories: []confmap.ProviderFactory{
signozenvprovider.NewFactory(),
},
config, err := signoz.NewConfig(context.Background(), signozconfig.ResolverConfig{
Uris: []string{"env:"},
ProviderFactories: []config.ProviderFactory{
envprovider.NewFactory(),
},
})
if err != nil {
zap.L().Fatal("Failed to create config", zap.Error(err))
}

signoz, err := signoz.New(config, skipWebFrontend)
signoz, err := signoz.New(config)
if err != nil {
zap.L().Fatal("Failed to create signoz struct", zap.Error(err))
}

serverOptions := &app.ServerOptions{
SigNoz: signoz,
HTTPHostPort: baseconst.HTTPHostPort,
PromConfigPath: promConfigPath,
SkipTopLvlOpsPath: skipTopLvlOpsPath,
Expand All @@ -171,7 +165,6 @@ func main() {
GatewayUrl: gatewayUrl,
UseLogsNewSchema: useLogsNewSchema,
UseTraceNewSchema: useTraceNewSchema,
SkipWebFrontend: skipWebFrontend,
}

// Read the jwt secret key
Expand All @@ -183,13 +176,7 @@ func main() {
zap.L().Info("JWT secret key set successfully.")
}

if err := migrate.Migrate(baseconst.RELATIONAL_DATASOURCE_PATH); err != nil {
zap.L().Error("Failed to migrate", zap.Error(err))
} else {
zap.L().Info("Migration successful")
}

server, err := app.NewServer(serverOptions)
server, err := app.NewServer(serverOptions, config, signoz)
if err != nil {
zap.L().Fatal("Failed to create server", zap.Error(err))
}
Expand Down
28 changes: 24 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
github.com/go-kit/log v0.2.1
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redis/redismock/v8 v8.11.5
github.com/go-viper/mapstructure/v2 v2.1.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/uuid v1.6.0
github.com/gorilla/handlers v1.5.1
Expand All @@ -29,6 +30,7 @@ require (
github.com/jmoiron/sqlx v1.3.4
github.com/json-iterator/go v1.1.12
github.com/knadh/koanf v1.5.0
github.com/knadh/koanf/v2 v2.1.1
github.com/mailru/easyjson v0.7.7
github.com/mattn/go-sqlite3 v2.0.3+incompatible
github.com/oklog/oklog v0.3.2
Expand All @@ -48,6 +50,9 @@ require (
github.com/soheilhy/cmux v0.1.5
github.com/srikanthccv/ClickHouse-go-mock v0.9.0
github.com/stretchr/testify v1.9.0
github.com/uptrace/bun v1.2.8
github.com/uptrace/bun/dialect/sqlitedialect v1.2.8
github.com/uptrace/bun/driver/sqliteshim v1.2.8
go.opentelemetry.io/collector/confmap v1.17.0
go.opentelemetry.io/collector/pdata v1.17.0
go.opentelemetry.io/collector/processor v0.111.0
Expand All @@ -61,7 +66,7 @@ require (
go.uber.org/multierr v1.11.0
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.31.0
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/exp v0.0.0-20250103183323-7d7fa50e5329
golang.org/x/net v0.29.0
golang.org/x/oauth2 v0.23.0
golang.org/x/text v0.21.0
Expand Down Expand Up @@ -99,14 +104,14 @@ require (
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/go-jose/go-jose/v4 v4.0.2 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-viper/mapstructure/v2 v2.1.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
Expand All @@ -119,27 +124,30 @@ require (
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
github.com/hashicorp/go-version v1.7.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jonboulle/clockwork v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/klauspost/compress v1.17.10 // indirect
github.com/knadh/koanf/v2 v2.1.1 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/leodido/go-syslog/v4 v4.2.0 // indirect
github.com/leodido/ragel-machinery v0.0.0-20190525184631-5f46317e436b // indirect
github.com/lufia/plan9stats v0.0.0-20240408141607-282e7b5d6b74 // indirect
github.com/magefile/mage v1.15.0 // indirect
github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.1-0.20231216201459-8508981c8b6c // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.111.0 // indirect
github.com/paulmach/orb v0.11.1 // indirect
Expand All @@ -151,6 +159,8 @@ require (
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common/sigv4 v0.1.0 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/segmentio/backo-go v1.0.1 // indirect
Expand All @@ -162,8 +172,11 @@ require (
github.com/spf13/pflag v1.0.5 // indirect
github.com/tklauser/go-sysconf v0.3.13 // indirect
github.com/tklauser/numcpus v0.7.0 // indirect
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
github.com/valyala/fastjson v1.6.4 // indirect
github.com/vjeantet/grok v1.0.1 // indirect
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opencensus.io v0.24.0 // indirect
Expand Down Expand Up @@ -212,7 +225,7 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.30.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/time v0.6.0 // indirect
gonum.org/v1/gonum v0.15.1 // indirect
google.golang.org/api v0.199.0 // indirect
Expand All @@ -221,6 +234,13 @@ require (
k8s.io/client-go v0.31.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
modernc.org/gc/v3 v3.0.0-20250105121824-520be1a3aee6 // indirect
modernc.org/libc v1.61.6 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.8.1 // indirect
modernc.org/sqlite v1.34.4 // indirect
modernc.org/strutil v1.2.1 // indirect
modernc.org/token v1.1.0 // indirect
)

replace github.com/prometheus/prometheus => github.com/SigNoz/prometheus v1.12.0
Loading
Loading