diff --git a/cmd/serve.go b/cmd/serve.go index d21e6d2b4..4e7d8a906 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -11,6 +11,7 @@ import ( "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" "github.com/stellar/stellar-disbursement-platform-backend/internal/data" di "github.com/stellar/stellar-disbursement-platform-backend/internal/dependencyinjection" + "github.com/stellar/stellar-disbursement-platform-backend/internal/events" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/scheduler" "github.com/stellar/stellar-disbursement-platform-backend/internal/scheduler/jobs" @@ -24,6 +25,12 @@ import ( serveadmin "github.com/stellar/stellar-disbursement-platform-backend/stellar-multitenant/pkg/serve" ) +var ( + eventBrokerType events.EventBrokerType + brokers []string + consumerGroupID string +) + type ServeCommand struct{} type ServerServiceInterface interface { @@ -121,7 +128,7 @@ func (c *ServeCommand) Command(serverService ServerServiceInterface, monitorServ Required: true, }, { - Name: "admin-serve-port", + Name: "admin-port", Usage: "Port where the admin tenant server will be listening on", OptType: types.Int, ConfigKey: &adminServeOpts.Port, @@ -309,6 +316,30 @@ func (c *ServeCommand) Command(serverService ServerServiceInterface, monitorServ FlagDefault: true, Required: false, }, + { + Name: "event-broker-type", + Usage: `Event Broker type. Options: "KAFKA", "NONE"`, + OptType: types.String, + ConfigKey: &eventBrokerType, + CustomSetValue: cmdUtils.SetConfigOptionEventBrokerType, + FlagDefault: string(events.KafkaEventBrokerType), + Required: true, + }, + { + Name: "brokers", + Usage: "List of Message Brokers Connection string comma separated.", + OptType: types.String, + ConfigKey: &brokers, + CustomSetValue: cmdUtils.SetConfigOptionURLList, + Required: false, + }, + { + Name: "consumer-group-id", + Usage: "Message Broker Consumer Group ID.", + OptType: types.String, + ConfigKey: &consumerGroupID, + Required: false, + }, } messengerOptions := message.MessengerOptions{} @@ -424,6 +455,27 @@ func (c *ServeCommand) Command(serverService ServerServiceInterface, monitorServ } serveOpts.AnchorPlatformAPIService = apAPIService + // Kafka (background) + if eventBrokerType == events.KafkaEventBrokerType { + kafkaProducer, err := events.NewKafkaProducer(brokers) + if err != nil { + log.Ctx(ctx).Fatalf("error creating Kafka Producer: %v", err) + } + defer kafkaProducer.Close() + serveOpts.EventProducer = kafkaProducer + + // TODO: remove this example when start implementing the actual consumers + pingPongConsumer, err := events.NewKafkaConsumer(brokers, "ping-pong", consumerGroupID, &events.PingPongEventHandler{}) + if err != nil { + log.Ctx(ctx).Fatalf("error creating Kafka Consumer: %v", err) + } + defer pingPongConsumer.Close() + + go events.Consume(ctx, pingPongConsumer, crashTrackerClient) + } else { + log.Ctx(ctx).Warn("Event Broker is NONE.") + } + // Starting Scheduler Service (background job) if enabled if serveOpts.EnableScheduler { log.Ctx(ctx).Info("Starting Scheduler Service...") diff --git a/cmd/serve_test.go b/cmd/serve_test.go index a163ff8f8..b773267b6 100644 --- a/cmd/serve_test.go +++ b/cmd/serve_test.go @@ -13,6 +13,7 @@ import ( "github.com/stellar/stellar-disbursement-platform-backend/internal/anchorplatform" "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" di "github.com/stellar/stellar-disbursement-platform-backend/internal/dependencyinjection" + "github.com/stellar/stellar-disbursement-platform-backend/internal/events" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/monitor" "github.com/stellar/stellar-disbursement-platform-backend/internal/scheduler" @@ -141,6 +142,10 @@ func Test_serve(t *testing.T) { require.NoError(t, err) serveOpts.SMSMessengerClient = smsMessengerClient + kafkaEventManager, err := events.NewKafkaProducer([]string{"kafka:9092"}) + require.NoError(t, err) + serveOpts.EventProducer = kafkaEventManager + metricOptions := monitor.MetricOptions{ MetricType: monitor.MetricTypePrometheus, Environment: "test", @@ -213,6 +218,8 @@ func Test_serve(t *testing.T) { t.Setenv("INSTANCE_NAME", serveOpts.InstanceName) t.Setenv("ENABLE_SCHEDULER", "true") t.Setenv("ENABLE_MULTITENANT_DB", "false") + t.Setenv("BROKERS", "kafka:9092") + t.Setenv("CONSUMER_GROUP_ID", "group-id") // test & assert rootCmd.SetArgs([]string{"--environment", "test", "serve", "--metrics-type", "PROMETHEUS"}) diff --git a/cmd/utils/custom_set_value.go b/cmd/utils/custom_set_value.go index 2841f83fd..0ed811af5 100644 --- a/cmd/utils/custom_set_value.go +++ b/cmd/utils/custom_set_value.go @@ -12,6 +12,7 @@ import ( "github.com/stellar/go/support/config" "github.com/stellar/go/support/log" "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" + "github.com/stellar/stellar-disbursement-platform-backend/internal/events" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/monitor" "github.com/stellar/stellar-disbursement-platform-backend/internal/utils" @@ -203,3 +204,61 @@ func SetConfigOptionURLString(co *config.ConfigOption) error { return nil } + +func SetConfigOptionURLList(co *config.ConfigOption) error { + urlsStr := viper.GetString(co.Name) + + if urlsStr == "" { + return fmt.Errorf("url list cannot be empty") + } + + urls := strings.Split(urlsStr, ",") + for _, u := range urls { + _, err := url.ParseRequestURI(strings.TrimSpace(u)) + if err != nil { + return fmt.Errorf("error parsing url: %w", err) + } + } + + key, ok := co.ConfigKey.(*[]string) + if !ok { + return fmt.Errorf("the expected type for this config key is a string slice, but got a %T instead", co.ConfigKey) + } + *key = urls + + return nil +} + +func SetConfigOptionStringList(co *config.ConfigOption) error { + listStr := viper.GetString(co.Name) + + if listStr == "" { + return fmt.Errorf("cannot be empty") + } + + list := strings.Split(listStr, ",") + for i, el := range list { + list[i] = strings.TrimSpace(el) + } + + key, ok := co.ConfigKey.(*[]string) + if !ok { + return fmt.Errorf("the expected type for this config key is a string slice, but got a %T instead", co.ConfigKey) + } + + *key = list + + return nil +} + +func SetConfigOptionEventBrokerType(co *config.ConfigOption) error { + ebType := viper.GetString(co.Name) + + ebTypeParsed, err := events.ParseEventBrokerType(ebType) + if err != nil { + return fmt.Errorf("couldn't parse event broker type: %w", err) + } + + *(co.ConfigKey.(*events.EventBrokerType)) = ebTypeParsed + return nil +} diff --git a/cmd/utils/custom_set_value_test.go b/cmd/utils/custom_set_value_test.go index 9d7d62438..bc961b3af 100644 --- a/cmd/utils/custom_set_value_test.go +++ b/cmd/utils/custom_set_value_test.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/go/support/config" "github.com/stellar/go/support/log" "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" + "github.com/stellar/stellar-disbursement-platform-backend/internal/events" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/monitor" "github.com/stellar/stellar-disbursement-platform-backend/internal/utils" @@ -582,3 +583,118 @@ func Test_SetConfigOptionURLString(t *testing.T) { }) } } + +func Test_SetConfigOptionURLList(t *testing.T) { + opts := struct{ brokers []string }{} + + co := config.ConfigOption{ + Name: "brokers", + OptType: types.String, + CustomSetValue: SetConfigOptionURLList, + ConfigKey: &opts.brokers, + Required: false, + } + + testCases := []customSetterTestCase[[]string]{ + { + name: "returns an error if the list is empty", + args: []string{"--brokers", ""}, + wantErrContains: "cannot be empty", + }, + { + name: "🎉 handles string list successfully (from CLI args)", + args: []string{"--brokers", "kafka:9092,localhost:9093,kafka://broker:9092"}, + wantResult: []string{"kafka:9092", "localhost:9093", "kafka://broker:9092"}, + }, + { + name: "🎉 string list successfully (from ENV vars)", + envValue: "kafka:9092,localhost:9093", + wantResult: []string{"kafka:9092", "localhost:9093"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.brokers = []string{} + customSetterTester[[]string](t, tc, co) + }) + } +} + +func Test_SetConfigOptionStringList(t *testing.T) { + opts := struct{ topics []string }{} + + co := config.ConfigOption{ + Name: "topics", + OptType: types.String, + CustomSetValue: SetConfigOptionStringList, + ConfigKey: &opts.topics, + Required: false, + } + + testCases := []customSetterTestCase[[]string]{ + { + name: "returns an error if the list is empty", + args: []string{"--topics", ""}, + wantErrContains: "cannot be empty", + }, + { + name: "🎉 handles string list successfully (from CLI args)", + args: []string{"--topics", "topic1, topic2,topic3"}, + wantResult: []string{"topic1", "topic2", "topic3"}, + }, + { + name: "🎉 string list successfully (from ENV vars)", + envValue: "topic1, topic2", + wantResult: []string{"topic1", "topic2"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.topics = []string{} + customSetterTester[[]string](t, tc, co) + }) + } +} + +func Test_SetConfigOptionEventBrokerType(t *testing.T) { + opts := struct{ eventBrokerType events.EventBrokerType }{} + + co := config.ConfigOption{ + Name: "event-broker-type", + OptType: types.String, + CustomSetValue: SetConfigOptionEventBrokerType, + ConfigKey: &opts.eventBrokerType, + } + + testCases := []customSetterTestCase[events.EventBrokerType]{ + { + name: "returns an error if event broker type is empty", + args: []string{"--event-broker-type", ""}, + wantErrContains: "couldn't parse event broker type: invalid event broker type", + }, + { + name: "🎉 handles event broker type (through CLI args): KAFKA", + args: []string{"--event-broker-type", "kafka"}, + wantResult: events.KafkaEventBrokerType, + }, + { + name: "🎉 handles event broker type (through CLI args): NONE", + args: []string{"--event-broker-type", "NONE"}, + wantResult: events.NoneEventBrokerType, + }, + { + name: "returns an error if a invalid event broker type", + args: []string{"--event-broker-type", "invalid"}, + wantErrContains: "couldn't parse event broker type: invalid event broker type", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opts.eventBrokerType = "" + customSetterTester[events.EventBrokerType](t, tc, co) + }) + } +} diff --git a/db/migrate_test.go b/db/migrate_test.go index aa8d64eb9..4b511f29b 100644 --- a/db/migrate_test.go +++ b/db/migrate_test.go @@ -125,11 +125,11 @@ func TestMigrate_downApplyOne_Tenant_migrations(t *testing.T) { n, err := Migrate(db.DSN, migrate.Up, 2, adminmigrations.FS, StellarAdminMigrationsTableName) require.NoError(t, err) - require.Equal(t, 1, n) + require.Equal(t, 2, n) - n, err = Migrate(db.DSN, migrate.Down, 1, adminmigrations.FS, StellarAdminMigrationsTableName) + n, err = Migrate(db.DSN, migrate.Down, 2, adminmigrations.FS, StellarAdminMigrationsTableName) require.NoError(t, err) - require.Equal(t, 1, n) + require.Equal(t, 2, n) ids := []string{} err = dbConnectionPool.SelectContext(ctx, &ids, fmt.Sprintf("SELECT id FROM %s", StellarAdminMigrationsTableName)) diff --git a/db/migrations/tenant-migrations/2023-11-14.0.drop-unused-cors-column.sql b/db/migrations/tenant-migrations/2023-11-14.0.drop-unused-cors-column.sql new file mode 100644 index 000000000..3b3f5f5b5 --- /dev/null +++ b/db/migrations/tenant-migrations/2023-11-14.0.drop-unused-cors-column.sql @@ -0,0 +1,12 @@ +-- Drop unused cors allowed origins column. + +-- +migrate Up + +ALTER TABLE public.tenants + DROP COLUMN IF EXISTS cors_allowed_origins; + + +-- +migrate Down + +ALTER TABLE public.tenants + ADD COLUMN cors_allowed_origins text[] NULL; diff --git a/dev/docker-compose-sdp-anchor.yml b/dev/docker-compose-sdp-anchor.yml index 29ae95a44..ec9c989ca 100644 --- a/dev/docker-compose-sdp-anchor.yml +++ b/dev/docker-compose-sdp-anchor.yml @@ -43,6 +43,8 @@ services: CORS_ALLOWED_ORIGINS: "*" ENABLE_MFA: "false" ENABLE_RECAPTCHA: "false" + BROKERS: "kafka:9092" + CONSUMER_GROUP_ID: "group-id" # multi-tenant INSTANCE_NAME: "SDP Testnet on Docker" @@ -74,6 +76,7 @@ services: ./stellar-disbursement-platform serve depends_on: - db + - kafka db-anchor-platform: container_name: anchor-platform-postgres-db @@ -170,8 +173,69 @@ services: SECRET_SEP10_SIGNING_SEED: ${SEP10_SIGNING_PRIVATE_KEY} SECRET_SEP24_INTERACTIVE_URL_JWT_SECRET: jwt_secret_1234567890 SECRET_SEP24_MORE_INFO_URL_JWT_SECRET: jwt_secret_1234567890 + + kafka: + image: docker.io/bitnami/kafka:3.6 + ports: + - "9094:9094" + volumes: + - "kafka-data:/bitnami" + environment: + # KRaft settings + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:9093 + # Listeners + - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092,CONTROLLER://:9093,EXTERNAL://:9094 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + + db-conduktor: + image: postgres:14 + hostname: postgresql + volumes: + - db-conduktor-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: "conduktor-platform" + POSTGRES_USER: "conduktor" + POSTGRES_PASSWORD: "change_me" + POSTGRES_HOST_AUTH_METHOD: "scram-sha-256" + + conduktor-platform: + image: conduktor/conduktor-platform:1.19.2 + depends_on: + - db-conduktor + ports: + - "9090:8080" + volumes: + - conduktor-platform-data:/var/conduktor + environment: + CDK_DATABASE_URL: "postgresql://conduktor:change_me@db-conduktor:5432/conduktor-platform" + CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/ + CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/ + CDK_MONITORING_CALLBACK-URL: http://conduktor-platform:9090/monitoring/api/ + CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:9090 + healthcheck: + test: curl -f http://localhost:9090/platform/api/modules/health/live || exit 1 + interval: 10s + start_period: 10s + timeout: 5s + retries: 3 + + conduktor-monitoring: + image: conduktor/conduktor-platform-cortex:1.19.2 + environment: + CDK_CONSOLE-URL: "http://conduktor-platform:9090" + volumes: postgres-db: driver: local postgres-ap-db: driver: local + kafka-data: + driver: local + db-conduktor-data: + driver: local + conduktor-platform-data: + driver: local diff --git a/go.list b/go.list index 104adb63c..af2d8fd22 100644 --- a/go.list +++ b/go.list @@ -71,6 +71,7 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 github.com/go-gorp/gorp/v3 v3.1.0 github.com/go-kit/log v0.2.1 github.com/go-logfmt/logfmt v0.5.1 +github.com/go-logr/logr v1.2.3 github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab github.com/go-playground/locales v0.14.0 github.com/go-playground/universal-translator v0.18.0 @@ -93,7 +94,7 @@ github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.3 github.com/golang/snappy v0.0.4 github.com/google/btree v1.0.0 -github.com/google/go-cmp v0.5.9 +github.com/google/go-cmp v0.6.0 github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 github.com/google/martian v2.1.0+incompatible github.com/google/martian/v3 v3.1.0 @@ -144,7 +145,7 @@ github.com/kataras/pio v0.0.11 github.com/kataras/sitemap v0.0.6 github.com/kataras/tunnel v0.0.4 github.com/kisielk/gotool v1.0.0 -github.com/klauspost/compress v1.16.0 +github.com/klauspost/compress v1.16.7 github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/kr/fs v0.1.0 github.com/kr/pretty v0.3.1 @@ -185,11 +186,13 @@ github.com/nelsam/hel/v2 v2.3.3 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e github.com/nyaruka/phonenumbers v1.1.8 github.com/olekukonko/tablewriter v0.0.5 -github.com/onsi/ginkgo v1.7.0 -github.com/onsi/gomega v1.4.3 +github.com/onsi/ginkgo v1.12.0 +github.com/onsi/ginkgo/v2 v2.4.0 +github.com/onsi/gomega v1.23.0 github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.9.0 github.com/pelletier/go-toml/v2 v2.0.9 +github.com/pierrec/lz4/v4 v4.1.18 github.com/pingcap/errors v0.11.4 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.1 @@ -208,7 +211,8 @@ github.com/russross/blackfriday/v2 v2.1.0 github.com/sagikazarmark/crypt v0.10.0 github.com/schollz/closestmatch v2.1.0+incompatible github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 -github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca +github.com/segmentio/kafka-go v0.4.46 +github.com/sergi/go-diff v1.2.0 github.com/shopspring/decimal v1.3.1 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 github.com/sirupsen/logrus v1.9.3 @@ -235,10 +239,13 @@ github.com/valyala/fasthttp v1.40.0 github.com/valyala/fasttemplate v1.2.2 github.com/vmihailenco/msgpack/v5 v5.3.5 github.com/vmihailenco/tagparser/v2 v2.0.0 +github.com/xdg-go/pbkdf2 v1.0.0 +github.com/xdg-go/scram v1.1.2 +github.com/xdg-go/stringprep v1.0.4 github.com/xdrpp/goxdr v0.1.1 -github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 -github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c -github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 +github.com/xeipuuv/gojsonschema v1.2.0 github.com/xhit/go-str2duration/v2 v2.1.0 github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb github.com/yosssi/ace v0.0.5 diff --git a/go.mod b/go.mod index 4d7e3b6a5..a7c799416 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/rs/cors v1.9.0 github.com/rubenv/sql-migrate v1.5.2 + github.com/segmentio/kafka-go v0.4.46 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.16.0 @@ -41,22 +42,28 @@ require ( github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/manucorporat/sse v0.0.0-20160126180136-ee05b128a739 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/onsi/ginkgo v1.12.0 // indirect + github.com/onsi/gomega v1.23.0 // indirect github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 // indirect + github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -64,6 +71,8 @@ require ( github.com/stellar/go-xdr v0.0.0-20211103144802-8017fc4bdfee // indirect github.com/stretchr/objx v0.5.1 // indirect github.com/subosito/gotenv v1.4.2 // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index bca470715..d1ccf6e7f 100644 --- a/go.sum +++ b/go.sum @@ -80,6 +80,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gavv/monotime v0.0.0-20161010190848-47d58efa6955 h1:gmtGRvSexPU4B1T/yYo0sLOKzER1YT+b4kPxPpm0Ty4= @@ -149,7 +150,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v0.0.0-20160401233042-9235644dd9e5 h1:oERTZ1buOUYlpmKaqlO5fYmz8cZ1rYu5DieJzF4ZVmU= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -178,6 +180,7 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= @@ -194,7 +197,9 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -230,10 +235,17 @@ github.com/moul/http2curl v0.0.0-20161031194548-4e24498b31db h1:eZgFHVkk9uOTaOQL github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nyaruka/phonenumbers v1.1.8 h1:mjFu85FeoH2Wy18aOMUvxqi1GgAqiQSJsa/cCC5yu2s= github.com/nyaruka/phonenumbers v1.1.8/go.mod h1:DC7jZd321FqUe+qWSNcHi10tyIyGNXGcNbfkPvdp1Vs= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -259,7 +271,10 @@ github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWx github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2 h1:S4OC0+OBKz6mJnzuHioeEat74PuQ4Sgvbf8eus695sc= github.com/segmentio/go-loggly v0.5.1-0.20171222203950-eb91657e62b2/go.mod h1:8zLRYR5npGjaOXgPSKat5+oOh+UHd8OdbS18iqX9F6Y= -github.com/sergi/go-diff v0.0.0-20161205080420-83532ca1c1ca h1:oR/RycYTFTVXzND5r4FdsvbnBn0HJXSVeNAnwaTXRwk= +github.com/segmentio/kafka-go v0.4.46 h1:Sx8/kvtY+/G8nM0roTNnFezSJj3bT2sW0Xy/YY3CgBI= +github.com/segmentio/kafka-go v0.4.46/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= +github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -284,6 +299,7 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -298,10 +314,20 @@ github.com/twilio/twilio-go v1.11.0 h1:ixO2DfAV4c0Yza0Tom5F5ZZB8WUbigiFc9wD84vbY github.com/twilio/twilio-go v1.11.0/go.mod h1:tdnfQ5TjbewoAu4lf9bMsGvfuJ/QU9gYuv9yx3TSIXU= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/fasthttp v1.40.0 h1:CRq/00MfruPGFLTQKY8b+8SfdK60TxNztjRMnH0t1Yc= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xdrpp/goxdr v0.1.1 h1:E1B2c6E8eYhOVyd7yEpOyopzTPirUeF6mVOfXfGyJyc= -github.com/xeipuuv/gojsonpointer v0.0.0-20151027082146-e0fe6f683076 h1:KM4T3G70MiR+JtqplcYkNVoNz7pDwYaBxWBXQK804So= -github.com/xeipuuv/gojsonreference v0.0.0-20150808065054-e02fc20de94c h1:XZWnr3bsDQWAZg4Ne+cPoXRPILrNlPNQfxBuwLl43is= -github.com/xeipuuv/gojsonschema v0.0.0-20161231055540-f06f290571ce h1:cVSRGH8cOveJNwFEEZLXtB+XMnRqKLjUP6V/ZFYQCXI= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= +github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yalp/jsonpath v0.0.0-20150812003900-31a79c7593bb h1:06WAhQa+mYv7BiOk13B/ywyTlkoE/S7uu6TBKU6FHnE= github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d h1:yJIizrfO599ot2kQ6Af1enICnwBD3XoxgX3MrMwot2M= github.com/yudai/golcs v0.0.0-20150405163532-d1c525dea8ce h1:888GrqRxabUce7lj4OaoShPxodm3kXOMpSa85wdYzfY= @@ -325,6 +351,7 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -364,8 +391,10 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -399,6 +428,9 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -422,7 +454,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -433,6 +467,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -466,11 +501,17 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -480,7 +521,11 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -535,6 +580,7 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -633,17 +679,21 @@ google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gavv/httpexpect.v1 v1.0.0-20170111145843-40724cf1e4a0 h1:r5ptJ1tBxVAeqw4CrYWhXIMr0SybY3CDHuIbCg5CFVw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/tylerb/graceful.v1 v1.2.15 h1:1JmOyhKqAyX3BgTXMI84LwT6FOJ4tP2N9e2kwTCM0nQ= gopkg.in/tylerb/graceful.v1 v1.2.15/go.mod h1:yBhekWvR20ACXVObSSdD3u6S9DeSylanL2PAbAC/uJ8= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helmchart/sdp/Chart.yaml b/helmchart/sdp/Chart.yaml index 30eed34ec..88b8bf823 100644 --- a/helmchart/sdp/Chart.yaml +++ b/helmchart/sdp/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: stellar-disbursement-platform description: A Helm chart for the Stellar Disbursement Platform Backend (A.K.A. `sdp`) -version: 0.9.3 +version: 1.0.0 appVersion: "1.0.0" type: application maintainers: diff --git a/helmchart/sdp/templates/01.1-configmap-sdp.yaml b/helmchart/sdp/templates/01.1-configmap-sdp.yaml index 637140676..2c72184a8 100644 --- a/helmchart/sdp/templates/01.1-configmap-sdp.yaml +++ b/helmchart/sdp/templates/01.1-configmap-sdp.yaml @@ -24,6 +24,7 @@ data: BASE_URL: {{ include "sdp.schema" . }}://{{ include "sdp.domain" . }} PORT: {{ include "sdp.port" . | quote }} METRICS_PORT: {{ include "sdp.metricsPort" . | quote }} + ADMIN_PORT: {{ include "sdp.adminPort" . | quote }} ANCHOR_PLATFORM_BASE_SEP_URL: {{ include "sdp.ap.schema" . }}://{{ include "sdp.ap.domain" . }} ANCHOR_PLATFORM_BASE_PLATFORM_URL: {{ include "sdp.ap.platformServiceAddress" . }} {{- tpl (toYaml .Values.sdp.configMap.data | nindent 2) . }} diff --git a/helmchart/sdp/templates/01.3-configmap-tss.yaml b/helmchart/sdp/templates/01.3-configmap-tss.yaml index a39480fcb..5e0de4f0f 100644 --- a/helmchart/sdp/templates/01.3-configmap-tss.yaml +++ b/helmchart/sdp/templates/01.3-configmap-tss.yaml @@ -1,3 +1,4 @@ +{{- if .Values.tss.enabled -}} --- apiVersion: v1 kind: ConfigMap @@ -22,3 +23,4 @@ data: {{- end }} TSS_METRICS_PORT: {{ include "tss.metricsPort" . | quote }} {{- tpl (toYaml .Values.tss.configMap.data | nindent 2) . }} +{{- end }} \ No newline at end of file diff --git a/helmchart/sdp/templates/02.1-deployment-sdp.yaml b/helmchart/sdp/templates/02.1-deployment-sdp.yaml index 9acbf0f87..8c6470f05 100644 --- a/helmchart/sdp/templates/02.1-deployment-sdp.yaml +++ b/helmchart/sdp/templates/02.1-deployment-sdp.yaml @@ -59,11 +59,10 @@ spec: - sh - -c - | - ./stellar-disbursement-platform db migrate up --all && - ./stellar-disbursement-platform db auth migrate up --all && - ./stellar-disbursement-platform db setup-for-network && - ./stellar-disbursement-platform channel-accounts verify --delete-invalid-accounts - ./stellar-disbursement-platform channel-accounts ensure --num-channel-accounts-ensure {{ .Values.tss.configMap.data.NUM_CHANNEL_ACCOUNTS | default 1 }} + ./stellar-disbursement-platform db tenant migrate up + ./stellar-disbursement-platform db migrate up --all + ./stellar-disbursement-platform db auth migrate up --all + ./stellar-disbursement-platform db setup-for-network --all containers: # ============================= Stellar Disbursement Platform: ============================= @@ -86,6 +85,9 @@ spec: - name: metrics containerPort: {{ include "sdp.metricsPort" . }} protocol: TCP + - name: admin + containerPort: {{ include "sdp.adminPort" . }} + protocol: TCP livenessProbe: httpGet: path: /health diff --git a/helmchart/sdp/templates/02.3-deployment-tss.yaml b/helmchart/sdp/templates/02.3-deployment-tss.yaml index 99a464627..bd5eb630b 100644 --- a/helmchart/sdp/templates/02.3-deployment-tss.yaml +++ b/helmchart/sdp/templates/02.3-deployment-tss.yaml @@ -1,3 +1,5 @@ +{{- if .Values.tss.enabled -}} +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -84,3 +86,4 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} +{{- end }} \ No newline at end of file diff --git a/helmchart/sdp/templates/04.1-ingress-sdp.yaml b/helmchart/sdp/templates/04.1-ingress-sdp.yaml index 03fd80753..b326c11b4 100644 --- a/helmchart/sdp/templates/04.1-ingress-sdp.yaml +++ b/helmchart/sdp/templates/04.1-ingress-sdp.yaml @@ -32,4 +32,17 @@ spec: name: {{ include "sdp.fullname" . }} port: number: {{ include "sdp.port" . }} +{{- if .Values.sdp.route.mtnDomain }} + - host: {{ include "sdp.mtnDomain" . | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "sdp.fullname" . }} + port: + number: {{ include "sdp.port" . }} +{{- end }} + {{- end }} diff --git a/helmchart/sdp/templates/04.3-ingress-dashboard.yaml b/helmchart/sdp/templates/04.3-ingress-dashboard.yaml index 2a9610aeb..405e9881d 100644 --- a/helmchart/sdp/templates/04.3-ingress-dashboard.yaml +++ b/helmchart/sdp/templates/04.3-ingress-dashboard.yaml @@ -27,5 +27,17 @@ spec: name: {{ include "sdp.fullname" . }}-dashboard port: number: {{ include "dashboard.port" . }} + {{- if .Values.sdp.route.mtnDomain }} + - host: {{ include "dashboard.mtnDomain" . | quote }} + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: {{ include "sdp.fullname" . }}-dashboard + port: + number: {{ include "dashboard.port" . }} + {{- end }} {{- end }} diff --git a/helmchart/sdp/templates/05.3-secrets-tss.yaml b/helmchart/sdp/templates/05.3-secrets-tss.yaml index 2f6b8464a..4a8d143b8 100644 --- a/helmchart/sdp/templates/05.3-secrets-tss.yaml +++ b/helmchart/sdp/templates/05.3-secrets-tss.yaml @@ -1,4 +1,4 @@ -{{- if .Values.tss.kubeSecrets.create -}} +{{- if and .Values.tss.enabled .Values.tss.kubeSecrets.create -}} --- apiVersion: v1 kind: Secret diff --git a/helmchart/sdp/templates/_helpers.tpl b/helmchart/sdp/templates/_helpers.tpl index d60e5225e..bd2a44e92 100644 --- a/helmchart/sdp/templates/_helpers.tpl +++ b/helmchart/sdp/templates/_helpers.tpl @@ -74,6 +74,13 @@ SDP domain {{- .Values.sdp.route.domain | default "localhost" }} {{- end }} +{{/* +SDP MTN domain +*/}} +{{- define "sdp.mtnDomain" -}} +{{- .Values.sdp.route.mtnDomain | default "localhost" }} +{{- end }} + {{/* SDP domain schema */}} @@ -95,6 +102,13 @@ SDP Metrics port {{- .Values.sdp.route.metricsPort | default "8002" }} {{- end }} +{{/* +SDP Admin port +*/}} +{{- define "sdp.adminPort" -}} +{{- .Values.sdp.route.adminPort | default "8003" }} +{{- end }} + {{/* Define the full address to the SDP service. */}} @@ -175,6 +189,13 @@ Dashboard domain {{- .Values.dashboard.route.domain | default "localhost" }} {{- end }} +{{/* +Dashboard MTN domain +*/}} +{{- define "dashboard.mtnDomain" -}} +{{- .Values.dashboard.route.mtnDomain | default "localhost" }} +{{- end }} + {{/* Dashboard domain schema */}} diff --git a/helmchart/sdp/values.yaml b/helmchart/sdp/values.yaml index 570d008fe..5e4500c59 100644 --- a/helmchart/sdp/values.yaml +++ b/helmchart/sdp/values.yaml @@ -82,6 +82,7 @@ sdp: domain: sdp.localhost.com port: "8000" metricsPort: "8002" + adminPort: "8003" ## @extra sdp.image Configuration related to the Docker image used by the SDP service. ## @param sdp.image.repository Docker image repository for the SDP backend service. @@ -359,6 +360,10 @@ anchorPlatform: ## This service is designed to maximize payment throughput, handle queuing, and graceful resubmission/error handling ## @descriptionEnd tss: + + ## @param tss.enabled If true, the tss will be deployed. + enabled: true + ## @extra tss.route Configuration related to the routing of the TSS. ## @param tss.route.schema Protocol scheme used for the service. Can be "http" or "https". ## @param tss.route.port Primary port on which the TSS listens. @@ -497,4 +502,3 @@ dashboard: secretName: dashboard-tls-cert-name # You need to create this secret manually. For more instructions, please refer to helmchart/docs/README.md - diff --git a/internal/events/events.go b/internal/events/events.go new file mode 100644 index 000000000..2d8beb713 --- /dev/null +++ b/internal/events/events.go @@ -0,0 +1,96 @@ +package events + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "syscall" + + "github.com/stellar/go/support/log" + "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" +) + +var ( + ErrTopicRequired = errors.New("message topic is required") + ErrKeyRequired = errors.New("message key is required") + ErrTenantIDRequired = errors.New("message tenant ID is required") + ErrTypeRequired = errors.New("message type is required") + ErrDataRequired = errors.New("message data is required") +) + +type Message struct { + Topic string `json:"topic"` + Key string `json:"key"` + TenantID string `json:"tenant_id"` + Type string `json:"type"` + Data any `json:"data"` +} + +func (m Message) String() string { + return fmt.Sprintf("Topic: %s - Key: %s - Type: %s - Tenant ID: %s", m.Topic, m.Key, m.Type, m.TenantID) +} + +func (m Message) Validate() error { + if m.Topic == "" { + return ErrTopicRequired + } + + if m.Key == "" { + return ErrKeyRequired + } + + if m.TenantID == "" { + return ErrTenantIDRequired + } + + if m.Type == "" { + return ErrTypeRequired + } + + if m.Data == nil { + return ErrDataRequired + } + + return nil +} + +type Producer interface { + WriteMessages(ctx context.Context, messages ...Message) error + Close() error +} + +type Consumer interface { + ReadMessage(ctx context.Context) error + Close() error +} + +func Consume(ctx context.Context, consumer Consumer, crashTracker crashtracker.CrashTrackerClient) { + log.Ctx(ctx).Info("starting consuming messages...") + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGQUIT) + + for { + select { + case <-ctx.Done(): + log.Ctx(ctx).Infof("Stopping consuming messages due to context cancellation...") + return + + case sig := <-signalChan: + log.Ctx(ctx).Infof("Stopping consuming messages due to OS signal '%+v'", sig) + return + + default: + if err := consumer.ReadMessage(ctx); err != nil { + if errors.Is(err, io.EOF) { + log.Ctx(ctx).Warn("message broker returned EOF") // This is an end state + return + } + crashTracker.LogAndReportErrors(ctx, err, "consuming messages") + } + } + } +} diff --git a/internal/events/events_test.go b/internal/events/events_test.go new file mode 100644 index 000000000..b067722a5 --- /dev/null +++ b/internal/events/events_test.go @@ -0,0 +1,105 @@ +package events + +import ( + "context" + "errors" + "io" + "testing" + "time" + + "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockConsumer struct { + mock.Mock +} + +var _ Consumer = new(MockConsumer) + +func (c *MockConsumer) ReadMessage(ctx context.Context) error { + args := c.Called(ctx) + return args.Error(0) +} + +func (c *MockConsumer) RegisterEventHandler(ctx context.Context, eventHandlers ...EventHandler) error { + args := c.Called(ctx, eventHandlers) + return args.Error(0) +} + +func (c *MockConsumer) Close() error { + args := c.Called() + return args.Error(0) +} + +func TestConsume(t *testing.T) { + ctx := context.Background() + consumer := &MockConsumer{} + crashTracker := &crashtracker.MockCrashTrackerClient{} + + unexpectedErr := errors.New("unexpected error") + consumer. + On("ReadMessage", ctx). + Return(unexpectedErr). + Once(). + On("ReadMessage", ctx). + Return(io.EOF). + Once(). + On("ReadMessage", ctx). + Return(nil) + + crashTracker. + On("LogAndReportErrors", ctx, unexpectedErr, "consuming messages"). + Return(). + Once() + + Consume(ctx, consumer, crashTracker) + + tick := time.Tick(time.Second * 1) + go func() { + Consume(ctx, consumer, crashTracker) + }() + + <-tick + + consumer.AssertExpectations(t) + crashTracker.AssertExpectations(t) +} + +func Test_Message_Validate(t *testing.T) { + m := Message{} + + err := m.Validate() + assert.ErrorIs(t, err, ErrTopicRequired) + + m.Topic = "test-topic" + err = m.Validate() + assert.ErrorIs(t, err, ErrKeyRequired) + + m.Key = "test-key" + err = m.Validate() + assert.ErrorIs(t, err, ErrTenantIDRequired) + + m.TenantID = "tenant-ID" + err = m.Validate() + assert.ErrorIs(t, err, ErrTypeRequired) + + m.Type = "test-type" + err = m.Validate() + assert.ErrorIs(t, err, ErrDataRequired) + + m.Data = "test" + err = m.Validate() + assert.NoError(t, err) + + m.Data = nil + m.Data = map[string]string{"test": "test"} + err = m.Validate() + assert.NoError(t, err) + + m.Data = nil + m.Data = struct{ Name string }{Name: "test"} + err = m.Validate() + assert.NoError(t, err) +} diff --git a/internal/events/handler.go b/internal/events/handler.go new file mode 100644 index 000000000..b14fccd42 --- /dev/null +++ b/internal/events/handler.go @@ -0,0 +1,37 @@ +package events + +import ( + "context" + "fmt" +) + +type EventHandler interface { + Name() string + CanHandleMessage(ctx context.Context, message *Message) bool + Handle(ctx context.Context, message *Message) +} + +type PingPongRequest struct { + Message string `json:"message"` +} + +// PingPongEventHandler is a example of event handler +type PingPongEventHandler struct{} + +var _ EventHandler = new(PingPongEventHandler) + +func (h *PingPongEventHandler) Name() string { + return "PingPong.EventHandler" +} + +func (h *PingPongEventHandler) CanHandleMessage(ctx context.Context, message *Message) bool { + return message.Topic == "ping-pong" +} + +func (h *PingPongEventHandler) Handle(ctx context.Context, message *Message) { + if message.Type == "ping" { + fmt.Println("pong") + } else { + fmt.Println("ping") + } +} diff --git a/internal/events/kafka.go b/internal/events/kafka.go new file mode 100644 index 000000000..abb6e87cc --- /dev/null +++ b/internal/events/kafka.go @@ -0,0 +1,126 @@ +package events + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/segmentio/kafka-go" + "github.com/stellar/go/support/log" + "golang.org/x/exp/maps" +) + +type KafkaProducer struct { + writer *kafka.Writer +} + +// Implements Producer interface +var _ Producer = new(KafkaProducer) + +func NewKafkaProducer(brokers []string) (*KafkaProducer, error) { + k := KafkaProducer{} + + if len(brokers) == 0 { + return nil, fmt.Errorf("brokers cannot be empty") + } + + k.writer = &kafka.Writer{ + Addr: kafka.TCP(brokers...), + Balancer: &kafka.RoundRobin{}, + RequiredAcks: -1, + } + + return &k, nil +} + +func (k *KafkaProducer) WriteMessages(ctx context.Context, messages ...Message) error { + kafkaMessages := make([]kafka.Message, 0, len(messages)) + for _, msg := range messages { + msgJSON, err := json.Marshal(msg) + if err != nil { + return fmt.Errorf("marshalling message: %w", err) + } + + kafkaMessages = append(kafkaMessages, kafka.Message{ + Topic: msg.Topic, + Key: []byte(msg.Key), + Value: msgJSON, + }) + } + + if err := k.writer.WriteMessages(ctx, kafkaMessages...); err != nil { + log.Ctx(ctx).Errorf("writing message on kafka: %s", err.Error()) + return fmt.Errorf("writing message on kafka: %w", err) + } + + return nil +} + +func (k *KafkaProducer) Close() error { + log.Info("closing kafka producer") + return k.writer.Close() +} + +type KafkaConsumer struct { + handlers []EventHandler + reader *kafka.Reader +} + +// Implements Consumer interface +var _ Consumer = new(KafkaConsumer) + +func NewKafkaConsumer(brokers []string, topic string, consumerGroupID string, handlers ...EventHandler) (*KafkaConsumer, error) { + k := KafkaConsumer{} + + k.reader = kafka.NewReader(kafka.ReaderConfig{ + Brokers: brokers, + Topic: topic, + GroupID: consumerGroupID, + }) + + if len(handlers) == 0 { + return nil, fmt.Errorf("handlers cannot be empty") + } + + ehMap := make(map[string]EventHandler) + for _, handler := range handlers { + log.Infof("registering event handler %s for topic %s", handler.Name(), topic) + ehMap[handler.Name()] = handler + } + k.handlers = maps.Values(ehMap) + + return &k, nil +} + +func (k *KafkaConsumer) ReadMessage(ctx context.Context) error { + log.Ctx(ctx).Info("fetching messages from kafka") + kafkaMessage, err := k.reader.FetchMessage(ctx) + if err != nil { + return fmt.Errorf("fetching message from kafka: %w", err) + } + + log.Ctx(ctx).Info("unmarshalling new message") + var msg Message + if err = json.Unmarshal(kafkaMessage.Value, &msg); err != nil { + return fmt.Errorf("unmarshaling message: %w", err) + } + + log.Ctx(ctx).Infof("new message being processed: %s", msg.String()) + for _, handler := range k.handlers { + if handler.CanHandleMessage(ctx, &msg) { + handler.Handle(ctx, &msg) + } + } + + // Acknowledgement + if err = k.reader.CommitMessages(ctx, kafkaMessage); err != nil { + return fmt.Errorf("committing message: %w", err) + } + + return nil +} + +func (k *KafkaConsumer) Close() error { + log.Info("closing kafka consumer") + return k.reader.Close() +} diff --git a/internal/events/types.go b/internal/events/types.go new file mode 100644 index 000000000..49fb75f5f --- /dev/null +++ b/internal/events/types.go @@ -0,0 +1,25 @@ +package events + +import ( + "fmt" + "strings" +) + +type EventBrokerType string + +const ( + KafkaEventBrokerType EventBrokerType = "KAFKA" + // NoneEventBrokerType means that no event broker was chosen. + NoneEventBrokerType EventBrokerType = "NONE" +) + +func ParseEventBrokerType(ebType string) (EventBrokerType, error) { + switch EventBrokerType(strings.ToUpper(ebType)) { + case KafkaEventBrokerType: + return KafkaEventBrokerType, nil + case NoneEventBrokerType: + return NoneEventBrokerType, nil + default: + return "", fmt.Errorf("invalid event broker type") + } +} diff --git a/internal/serve/serve.go b/internal/serve/serve.go index a1c469f80..fa725832f 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -20,6 +20,7 @@ import ( "github.com/stellar/stellar-disbursement-platform-backend/internal/anchorplatform" "github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker" "github.com/stellar/stellar-disbursement-platform-backend/internal/data" + "github.com/stellar/stellar-disbursement-platform-backend/internal/events" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/monitor" "github.com/stellar/stellar-disbursement-platform-backend/internal/serve/httpclient" @@ -90,6 +91,7 @@ type ServeOptions struct { EnableMultiTenantDB bool tenantManager tenant.ManagerInterface tenantRouter db.DataSourceRouter + EventProducer events.Producer } // SetupDependencies uses the serve options to setup the dependencies for the server. diff --git a/stellar-multitenant/pkg/cli/config_tenant.go b/stellar-multitenant/pkg/cli/config_tenant.go index 39c29e352..dec991133 100644 --- a/stellar-multitenant/pkg/cli/config_tenant.go +++ b/stellar-multitenant/pkg/cli/config_tenant.go @@ -14,14 +14,13 @@ import ( ) type tenantOptions struct { - ID string - EmailSenderType *tenant.EmailSenderType - SMSSenderType *tenant.SMSSenderType - EnableMFA *bool - EnableReCAPTCHA *bool - CORSAllowedOrigins []string - BaseURL *string - SDPUIBaseURL *string + ID string + EmailSenderType *tenant.EmailSenderType + SMSSenderType *tenant.SMSSenderType + EnableMFA *bool + EnableReCAPTCHA *bool + BaseURL *string + SDPUIBaseURL *string } func ConfigTenantCmd() *cobra.Command { @@ -66,14 +65,6 @@ func ConfigTenantCmd() *cobra.Command { ConfigKey: &to.EnableReCAPTCHA, Required: false, }, - { - Name: "cors-allowed-origins", - Usage: `Cors URLs that are allowed to access the endpoints, separated by ","`, - OptType: types.String, - CustomSetValue: utils.SetCORSAllowedOrigins, - ConfigKey: &to.CORSAllowedOrigins, - Required: false, - }, { Name: "base-url", Usage: "The SDP backend server's base URL.", @@ -128,14 +119,13 @@ func executeConfigTenant(ctx context.Context, to *tenantOptions, dbURL string) e m := tenant.NewManager(tenant.WithDatabase(dbConnectionPool)) _, err = m.UpdateTenantConfig(ctx, &tenant.TenantUpdate{ - ID: to.ID, - EmailSenderType: to.EmailSenderType, - SMSSenderType: to.SMSSenderType, - EnableMFA: to.EnableMFA, - EnableReCAPTCHA: to.EnableReCAPTCHA, - CORSAllowedOrigins: to.CORSAllowedOrigins, - BaseURL: to.BaseURL, - SDPUIBaseURL: to.SDPUIBaseURL, + ID: to.ID, + EmailSenderType: to.EmailSenderType, + SMSSenderType: to.SMSSenderType, + EnableMFA: to.EnableMFA, + EnableReCAPTCHA: to.EnableReCAPTCHA, + BaseURL: to.BaseURL, + SDPUIBaseURL: to.SDPUIBaseURL, }) if err != nil { return fmt.Errorf("updating tenant config: %w", err) diff --git a/stellar-multitenant/pkg/cli/utils/custom_set_value.go b/stellar-multitenant/pkg/cli/utils/custom_set_value.go index 5bf9314ff..fe98b7f66 100644 --- a/stellar-multitenant/pkg/cli/utils/custom_set_value.go +++ b/stellar-multitenant/pkg/cli/utils/custom_set_value.go @@ -4,12 +4,9 @@ import ( "fmt" "net/url" "strconv" - "strings" "github.com/spf13/viper" - "github.com/stellar/go/keypair" "github.com/stellar/go/support/config" - "github.com/stellar/go/support/log" "github.com/stellar/stellar-disbursement-platform-backend/internal/message" "github.com/stellar/stellar-disbursement-platform-backend/internal/utils" "github.com/stellar/stellar-disbursement-platform-backend/stellar-multitenant/pkg/tenant" @@ -45,55 +42,6 @@ func SetConfigOptionSMSSenderType(co *config.ConfigOption) error { return nil } -func SetConfigOptionStellarPublicKey(co *config.ConfigOption) error { - publicKey := viper.GetString(co.Name) - if publicKey == "" { - return nil - } - - kp, err := keypair.ParseAddress(publicKey) - if err != nil { - return fmt.Errorf("error validating public key: %w", err) - } - - key, ok := co.ConfigKey.(**string) - if !ok { - return fmt.Errorf("the expected type for this config key is a string, but got a %T instead", co.ConfigKey) - } - addr := kp.Address() - *key = &addr - - return nil -} - -func SetCORSAllowedOrigins(co *config.ConfigOption) error { - corsAllowedOriginsOptions := viper.GetString(co.Name) - if corsAllowedOriginsOptions == "" { - return nil - } - - corsAllowedOrigins := strings.Split(corsAllowedOriginsOptions, ",") - - // validate addresses - for _, address := range corsAllowedOrigins { - _, err := url.ParseRequestURI(address) - if err != nil { - return fmt.Errorf("error parsing cors addresses: %w", err) - } - if address == "*" { - log.Warn(`The value "*" for the CORS Allowed Origins is too permissive and not recommended.`) - } - } - - key, ok := co.ConfigKey.(*[]string) - if !ok { - return fmt.Errorf("the expected type for this config key is a string slice, but got a %T instead", co.ConfigKey) - } - *key = corsAllowedOrigins - - return nil -} - func SetConfigOptionURLString(co *config.ConfigOption) error { u := viper.GetString(co.Name) if u == "" { diff --git a/stellar-multitenant/pkg/internal/httphandler/tenants_handler.go b/stellar-multitenant/pkg/internal/httphandler/tenants_handler.go index 11af81d66..a36b6affb 100644 --- a/stellar-multitenant/pkg/internal/httphandler/tenants_handler.go +++ b/stellar-multitenant/pkg/internal/httphandler/tenants_handler.go @@ -80,13 +80,12 @@ func (h TenantsHandler) Post(rw http.ResponseWriter, req *http.Request) { } tnt, err = h.Manager.UpdateTenantConfig(ctx, &tenant.TenantUpdate{ - ID: tnt.ID, - EmailSenderType: &reqBody.EmailSenderType, - SMSSenderType: &reqBody.SMSSenderType, - EnableMFA: &reqBody.EnableMFA, - EnableReCAPTCHA: &reqBody.EnableReCAPTCHA, - CORSAllowedOrigins: reqBody.CORSAllowedOrigins, - BaseURL: &reqBody.BaseURL, + ID: tnt.ID, + EmailSenderType: &reqBody.EmailSenderType, + SMSSenderType: &reqBody.SMSSenderType, + EnableMFA: &reqBody.EnableMFA, + EnableReCAPTCHA: &reqBody.EnableReCAPTCHA, + BaseURL: &reqBody.BaseURL, }) if err != nil { httperror.InternalError(ctx, "Could not update tenant config", err, nil).Render(rw) @@ -117,15 +116,14 @@ func (t TenantsHandler) Patch(w http.ResponseWriter, r *http.Request) { tenantID := chi.URLParam(r, "id") tnt, err := t.Manager.UpdateTenantConfig(ctx, &tenant.TenantUpdate{ - ID: tenantID, - EmailSenderType: reqBody.EmailSenderType, - SMSSenderType: reqBody.SMSSenderType, - EnableMFA: reqBody.EnableMFA, - EnableReCAPTCHA: reqBody.EnableReCAPTCHA, - CORSAllowedOrigins: reqBody.CORSAllowedOrigins, - BaseURL: reqBody.BaseURL, - SDPUIBaseURL: reqBody.SDPUIBaseURL, - Status: reqBody.Status, + ID: tenantID, + EmailSenderType: reqBody.EmailSenderType, + SMSSenderType: reqBody.SMSSenderType, + EnableMFA: reqBody.EnableMFA, + EnableReCAPTCHA: reqBody.EnableReCAPTCHA, + BaseURL: reqBody.BaseURL, + SDPUIBaseURL: reqBody.SDPUIBaseURL, + Status: reqBody.Status, }) if err != nil { if errors.Is(tenant.ErrEmptyUpdateTenant, err) { diff --git a/stellar-multitenant/pkg/internal/httphandler/tenants_handler_test.go b/stellar-multitenant/pkg/internal/httphandler/tenants_handler_test.go index 83030aae6..5f0d73614 100644 --- a/stellar-multitenant/pkg/internal/httphandler/tenants_handler_test.go +++ b/stellar-multitenant/pkg/internal/httphandler/tenants_handler_test.go @@ -135,7 +135,6 @@ func Test_TenantHandler_Get(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -149,7 +148,6 @@ func Test_TenantHandler_Get(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -184,7 +182,6 @@ func Test_TenantHandler_Get(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -218,7 +215,6 @@ func Test_TenantHandler_Get(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -297,7 +293,6 @@ func Test_TenantHandler_Post(t *testing.T) { "base_url": "invalid base URL value", "email_sender_type": "invalid email sender type. Expected one of these values: [AWS_EMAIL DRY_RUN]", "sms_sender_type": "invalid sms sender type. Expected one of these values: [TWILIO_SMS AWS_SMS DRY_RUN]", - "cors_allowed_origins": "provide at least one CORS allowed origins", "sdp_ui_base_url": "invalid SDP UI base URL value" } } @@ -330,7 +325,6 @@ func Test_TenantHandler_Post(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_recaptcha": true, "enable_mfa": false, - "cors_allowed_origins": ["*"], "base_url": "https://backend.sdp.org", "sdp_ui_base_url": "https://aid-org.sdp.org" } @@ -359,7 +353,6 @@ func Test_TenantHandler_Post(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": false, "enable_recaptcha": true, - "cors_allowed_origins": ["*"], "base_url": "https://backend.sdp.org", "sdp_ui_base_url": "https://aid-org.sdp.org", "status": "TENANT_PROVISIONED", @@ -484,28 +477,6 @@ func Test_TenantHandler_Patch(t *testing.T) { runBadRequestPatchTest(t, r, url, "status", "invalid status value") }) - t.Run("returns BadRequest when CORSAllowedOrigins is not valid", func(t *testing.T) { - rr := httptest.NewRecorder() - req, err := http.NewRequest(http.MethodPatch, url, strings.NewReader(`{"cors_allowed_origins": ["invalid"]}`)) - require.NoError(t, err) - r.ServeHTTP(rr, req) - - resp := rr.Result() - defer resp.Body.Close() - respBody, err := io.ReadAll(resp.Body) - require.NoError(t, err) - - assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - - expectedRespBody := `{ - "error": "invalid request body", - "extras": { - "cors_allowed_origins":"invalid URL value for cors_allowed_origins[0] = invalid" - } - }` - assert.JSONEq(t, string(expectedRespBody), string(respBody)) - }) - t.Run("successfully updates EmailSenderType of a tenant", func(t *testing.T) { reqBody := `{"email_sender_type": "AWS_EMAIL"}` expectedRespBody := ` @@ -513,7 +484,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -529,7 +499,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "TWILIO_SMS", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -545,7 +514,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": false, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -561,23 +529,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": false, - "cors_allowed_origins": null, - "base_url": null, - "sdp_ui_base_url": null, - "status": "TENANT_CREATED", - ` - - runSuccessfulRequestPatchTest(t, r, ctx, dbConnectionPool, handler, reqBody, expectedRespBody) - }) - - t.Run("successfully updates CORSAllowedOrigins of a tenant", func(t *testing.T) { - reqBody := `{"cors_allowed_origins": ["http://valid.com"]}` - expectedRespBody := ` - "email_sender_type": "DRY_RUN", - "sms_sender_type": "DRY_RUN", - "enable_mfa": true, - "enable_recaptcha": true, - "cors_allowed_origins": ["http://valid.com"], "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -593,7 +544,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": "http://valid.com", "sdp_ui_base_url": null, "status": "TENANT_CREATED", @@ -609,7 +559,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": "http://valid.com", "status": "TENANT_CREATED", @@ -625,7 +574,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "DRY_RUN", "enable_mfa": true, "enable_recaptcha": true, - "cors_allowed_origins": null, "base_url": null, "sdp_ui_base_url": null, "status": "TENANT_ACTIVATED", @@ -640,7 +588,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "AWS_SMS", "enable_mfa": false, "enable_recaptcha": false, - "cors_allowed_origins": ["http://valid.com"], "base_url": "http://valid.com", "sdp_ui_base_url": "http://valid.com", "status": "TENANT_ACTIVATED" @@ -651,7 +598,6 @@ func Test_TenantHandler_Patch(t *testing.T) { "sms_sender_type": "AWS_SMS", "enable_mfa": false, "enable_recaptcha": false, - "cors_allowed_origins": ["http://valid.com"], "base_url": "http://valid.com", "sdp_ui_base_url": "http://valid.com", "status": "TENANT_ACTIVATED", diff --git a/stellar-multitenant/pkg/internal/validators/tenant_validator.go b/stellar-multitenant/pkg/internal/validators/tenant_validator.go index aa1162a40..7b0ddc238 100644 --- a/stellar-multitenant/pkg/internal/validators/tenant_validator.go +++ b/stellar-multitenant/pkg/internal/validators/tenant_validator.go @@ -12,29 +12,27 @@ import ( var validTenantName *regexp.Regexp = regexp.MustCompile(`^[a-z-]+$`) type TenantRequest struct { - Name string `json:"name"` - OwnerEmail string `json:"owner_email"` - OwnerFirstName string `json:"owner_first_name"` - OwnerLastName string `json:"owner_last_name"` - OrganizationName string `json:"organization_name"` - EmailSenderType tenant.EmailSenderType `json:"email_sender_type"` - SMSSenderType tenant.SMSSenderType `json:"sms_sender_type"` - EnableMFA bool `json:"enable_mfa"` - EnableReCAPTCHA bool `json:"enable_recaptcha"` - CORSAllowedOrigins []string `json:"cors_allowed_origins"` - BaseURL string `json:"base_url"` - SDPUIBaseURL string `json:"sdp_ui_base_url"` + Name string `json:"name"` + OwnerEmail string `json:"owner_email"` + OwnerFirstName string `json:"owner_first_name"` + OwnerLastName string `json:"owner_last_name"` + OrganizationName string `json:"organization_name"` + EmailSenderType tenant.EmailSenderType `json:"email_sender_type"` + SMSSenderType tenant.SMSSenderType `json:"sms_sender_type"` + EnableMFA bool `json:"enable_mfa"` + EnableReCAPTCHA bool `json:"enable_recaptcha"` + BaseURL string `json:"base_url"` + SDPUIBaseURL string `json:"sdp_ui_base_url"` } type UpdateTenantRequest struct { - EmailSenderType *tenant.EmailSenderType `json:"email_sender_type"` - SMSSenderType *tenant.SMSSenderType `json:"sms_sender_type"` - EnableMFA *bool `json:"enable_mfa"` - EnableReCAPTCHA *bool `json:"enable_recaptcha"` - CORSAllowedOrigins []string `json:"cors_allowed_origins"` - BaseURL *string `json:"base_url"` - SDPUIBaseURL *string `json:"sdp_ui_base_url"` - Status *tenant.TenantStatus `json:"status"` + EmailSenderType *tenant.EmailSenderType `json:"email_sender_type"` + SMSSenderType *tenant.SMSSenderType `json:"sms_sender_type"` + EnableMFA *bool `json:"enable_mfa"` + EnableReCAPTCHA *bool `json:"enable_recaptcha"` + BaseURL *string `json:"base_url"` + SDPUIBaseURL *string `json:"sdp_ui_base_url"` + Status *tenant.TenantStatus `json:"status"` } type TenantValidator struct { @@ -72,13 +70,6 @@ func (tv *TenantValidator) ValidateCreateTenantRequest(reqBody *TenantRequest) * tv.Check(false, "sdp_ui_base_url", "invalid SDP UI base URL value") } - tv.Check(len(reqBody.CORSAllowedOrigins) != 0, "cors_allowed_origins", "provide at least one CORS allowed origins") - for i, cors := range reqBody.CORSAllowedOrigins { - if _, err = url.ParseRequestURI(cors); err != nil { - tv.Check(false, "cors_allowed_origins", fmt.Sprintf("invalid URL value for cors_allowed_origins[%d] = %s", i, cors)) - } - } - if tv.HasErrors() { return nil } @@ -105,14 +96,6 @@ func (tv *TenantValidator) ValidateUpdateTenantRequest(reqBody *UpdateTenantRequ reqBody.SMSSenderType = &SMSSenderType } - if reqBody.CORSAllowedOrigins != nil && len(reqBody.CORSAllowedOrigins) != 0 { - for i, cors := range reqBody.CORSAllowedOrigins { - if _, err = url.ParseRequestURI(cors); err != nil { - tv.Check(false, "cors_allowed_origins", fmt.Sprintf("invalid URL value for cors_allowed_origins[%d] = %s", i, cors)) - } - } - } - if reqBody.BaseURL != nil { if _, err = url.ParseRequestURI(*reqBody.BaseURL); err != nil { tv.Check(false, "base_url", "invalid base URL value") diff --git a/stellar-multitenant/pkg/internal/validators/tenant_validator_test.go b/stellar-multitenant/pkg/internal/validators/tenant_validator_test.go index 317a505be..18620f1af 100644 --- a/stellar-multitenant/pkg/internal/validators/tenant_validator_test.go +++ b/stellar-multitenant/pkg/internal/validators/tenant_validator_test.go @@ -22,16 +22,15 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { tv.ValidateCreateTenantRequest(reqBody) assert.True(t, tv.HasErrors()) assert.Equal(t, map[string]interface{}{ - "name": "invalid tenant name. It should only contains lower case letters and dash (-)", - "owner_email": "invalid email", - "owner_first_name": "owner_first_name is required", - "owner_last_name": "owner_last_name is required", - "organization_name": "organization_name is required", - "base_url": "invalid base URL value", - "email_sender_type": "invalid email sender type. Expected one of these values: [AWS_EMAIL DRY_RUN]", - "sms_sender_type": "invalid sms sender type. Expected one of these values: [TWILIO_SMS AWS_SMS DRY_RUN]", - "cors_allowed_origins": "provide at least one CORS allowed origins", - "sdp_ui_base_url": "invalid SDP UI base URL value", + "name": "invalid tenant name. It should only contains lower case letters and dash (-)", + "owner_email": "invalid email", + "owner_first_name": "owner_first_name is required", + "owner_last_name": "owner_last_name is required", + "organization_name": "organization_name is required", + "base_url": "invalid base URL value", + "email_sender_type": "invalid email sender type. Expected one of these values: [AWS_EMAIL DRY_RUN]", + "sms_sender_type": "invalid sms sender type. Expected one of these values: [TWILIO_SMS AWS_SMS DRY_RUN]", + "sdp_ui_base_url": "invalid SDP UI base URL value", }, tv.Errors) reqBody.Name = "aid-org" @@ -39,33 +38,31 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { tv.ValidateCreateTenantRequest(reqBody) assert.True(t, tv.HasErrors()) assert.Equal(t, map[string]interface{}{ - "owner_email": "invalid email", - "owner_first_name": "owner_first_name is required", - "owner_last_name": "owner_last_name is required", - "organization_name": "organization_name is required", - "base_url": "invalid base URL value", - "email_sender_type": "invalid email sender type. Expected one of these values: [AWS_EMAIL DRY_RUN]", - "sms_sender_type": "invalid sms sender type. Expected one of these values: [TWILIO_SMS AWS_SMS DRY_RUN]", - "cors_allowed_origins": "provide at least one CORS allowed origins", - "sdp_ui_base_url": "invalid SDP UI base URL value", + "owner_email": "invalid email", + "owner_first_name": "owner_first_name is required", + "owner_last_name": "owner_last_name is required", + "organization_name": "organization_name is required", + "base_url": "invalid base URL value", + "email_sender_type": "invalid email sender type. Expected one of these values: [AWS_EMAIL DRY_RUN]", + "sms_sender_type": "invalid sms sender type. Expected one of these values: [TWILIO_SMS AWS_SMS DRY_RUN]", + "sdp_ui_base_url": "invalid SDP UI base URL value", }, tv.Errors) }) t.Run("returns error when name is invalid", func(t *testing.T) { tv := NewTenantValidator() reqBody := &TenantRequest{ - Name: "aid org", - OwnerEmail: "owner@email.org", - OwnerFirstName: "Owner", - OwnerLastName: "Owner", - OrganizationName: "Aid Org", - EmailSenderType: tenant.AWSEmailSenderType, - SMSSenderType: tenant.TwilioSMSSenderType, - EnableMFA: true, - EnableReCAPTCHA: true, - CORSAllowedOrigins: []string{"*"}, - SDPUIBaseURL: "http://localhost:3000", - BaseURL: "http://localhost:8000", + Name: "aid org", + OwnerEmail: "owner@email.org", + OwnerFirstName: "Owner", + OwnerLastName: "Owner", + OrganizationName: "Aid Org", + EmailSenderType: tenant.AWSEmailSenderType, + SMSSenderType: tenant.TwilioSMSSenderType, + EnableMFA: true, + EnableReCAPTCHA: true, + SDPUIBaseURL: "http://localhost:3000", + BaseURL: "http://localhost:8000", } tv.ValidateCreateTenantRequest(reqBody) @@ -78,18 +75,17 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { t.Run("returns error when owner info is invalid", func(t *testing.T) { tv := NewTenantValidator() reqBody := &TenantRequest{ - Name: "aid-org", - OwnerEmail: "invalid", - OwnerFirstName: "", - OwnerLastName: "", - OrganizationName: "", - EmailSenderType: tenant.AWSEmailSenderType, - SMSSenderType: tenant.TwilioSMSSenderType, - EnableMFA: true, - EnableReCAPTCHA: true, - CORSAllowedOrigins: []string{"*"}, - BaseURL: "http://localhost:8000", - SDPUIBaseURL: "http://localhost:3000", + Name: "aid-org", + OwnerEmail: "invalid", + OwnerFirstName: "", + OwnerLastName: "", + OrganizationName: "", + EmailSenderType: tenant.AWSEmailSenderType, + SMSSenderType: tenant.TwilioSMSSenderType, + EnableMFA: true, + EnableReCAPTCHA: true, + BaseURL: "http://localhost:8000", + SDPUIBaseURL: "http://localhost:3000", } tv.ValidateCreateTenantRequest(reqBody) @@ -114,18 +110,17 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { t.Run("validates the email sender type successfully", func(t *testing.T) { tv := NewTenantValidator() reqBody := &TenantRequest{ - Name: "aid-org", - OwnerEmail: "owner@email.org", - OwnerFirstName: "Owner", - OwnerLastName: "Owner", - OrganizationName: "Aid Org", - EmailSenderType: "invalid", - SMSSenderType: tenant.TwilioSMSSenderType, - EnableMFA: true, - EnableReCAPTCHA: true, - CORSAllowedOrigins: []string{"*"}, - SDPUIBaseURL: "http://localhost:3000", - BaseURL: "http://localhost:8000", + Name: "aid-org", + OwnerEmail: "owner@email.org", + OwnerFirstName: "Owner", + OwnerLastName: "Owner", + OrganizationName: "Aid Org", + EmailSenderType: "invalid", + SMSSenderType: tenant.TwilioSMSSenderType, + EnableMFA: true, + EnableReCAPTCHA: true, + SDPUIBaseURL: "http://localhost:3000", + BaseURL: "http://localhost:8000", } tv.ValidateCreateTenantRequest(reqBody) @@ -141,18 +136,17 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { t.Run("validates the sms sender type successfully", func(t *testing.T) { tv := NewTenantValidator() reqBody := &TenantRequest{ - Name: "aid-org", - OwnerEmail: "owner@email.org", - OwnerFirstName: "Owner", - OwnerLastName: "Owner", - OrganizationName: "Aid Org", - EmailSenderType: tenant.AWSEmailSenderType, - SMSSenderType: "invalid", - EnableMFA: true, - EnableReCAPTCHA: true, - CORSAllowedOrigins: []string{"*"}, - SDPUIBaseURL: "http://localhost:3000", - BaseURL: "http://localhost:8000", + Name: "aid-org", + OwnerEmail: "owner@email.org", + OwnerFirstName: "Owner", + OwnerLastName: "Owner", + OrganizationName: "Aid Org", + EmailSenderType: tenant.AWSEmailSenderType, + SMSSenderType: "invalid", + EnableMFA: true, + EnableReCAPTCHA: true, + SDPUIBaseURL: "http://localhost:3000", + BaseURL: "http://localhost:8000", } tv.ValidateCreateTenantRequest(reqBody) @@ -168,26 +162,24 @@ func TestTenantValidator_ValidateCreateTenantRequest(t *testing.T) { t.Run("validates the URLs successfully", func(t *testing.T) { tv := NewTenantValidator() reqBody := &TenantRequest{ - Name: "aid-org", - OwnerEmail: "owner@email.org", - OwnerFirstName: "Owner", - OwnerLastName: "Owner", - OrganizationName: "Aid Org", - EmailSenderType: tenant.AWSEmailSenderType, - SMSSenderType: tenant.TwilioSMSSenderType, - EnableMFA: true, - EnableReCAPTCHA: true, - CORSAllowedOrigins: []string{"http://valid.com", "%invalid%"}, - SDPUIBaseURL: "%invalid%", - BaseURL: "%invalid%", + Name: "aid-org", + OwnerEmail: "owner@email.org", + OwnerFirstName: "Owner", + OwnerLastName: "Owner", + OrganizationName: "Aid Org", + EmailSenderType: tenant.AWSEmailSenderType, + SMSSenderType: tenant.TwilioSMSSenderType, + EnableMFA: true, + EnableReCAPTCHA: true, + SDPUIBaseURL: "%invalid%", + BaseURL: "%invalid%", } tv.ValidateCreateTenantRequest(reqBody) assert.True(t, tv.HasErrors()) assert.Equal(t, map[string]interface{}{ - "base_url": "invalid base URL value", - "sdp_ui_base_url": "invalid SDP UI base URL value", - "cors_allowed_origins": "invalid URL value for cors_allowed_origins[1] = %invalid%", + "base_url": "invalid base URL value", + "sdp_ui_base_url": "invalid SDP UI base URL value", }, tv.Errors) }) } @@ -209,16 +201,14 @@ func TestTenantValidator_ValidateUpdateTenantRequest(t *testing.T) { tv := NewTenantValidator() invalidValue := "invalid" reqBody := &UpdateTenantRequest{ - CORSAllowedOrigins: []string{invalidValue}, - BaseURL: &invalidValue, - SDPUIBaseURL: &invalidValue, + BaseURL: &invalidValue, + SDPUIBaseURL: &invalidValue, } tv.ValidateUpdateTenantRequest(reqBody) assert.True(t, tv.HasErrors()) assert.Equal(t, map[string]interface{}{ - "base_url": "invalid base URL value", - "cors_allowed_origins": "invalid URL value for cors_allowed_origins[0] = invalid", - "sdp_ui_base_url": "invalid SDP UI base URL value", + "base_url": "invalid base URL value", + "sdp_ui_base_url": "invalid SDP UI base URL value", }, tv.Errors) }) @@ -227,13 +217,12 @@ func TestTenantValidator_ValidateUpdateTenantRequest(t *testing.T) { enable := false url := "http://valid.com" reqBody := &UpdateTenantRequest{ - EmailSenderType: &tenant.AWSEmailSenderType, - SMSSenderType: &tenant.AWSSMSSenderType, - EnableMFA: &enable, - EnableReCAPTCHA: &enable, - CORSAllowedOrigins: []string{url}, - BaseURL: &url, - SDPUIBaseURL: &url, + EmailSenderType: &tenant.AWSEmailSenderType, + SMSSenderType: &tenant.AWSSMSSenderType, + EnableMFA: &enable, + EnableReCAPTCHA: &enable, + BaseURL: &url, + SDPUIBaseURL: &url, } tv.ValidateUpdateTenantRequest(reqBody) assert.False(t, tv.HasErrors()) diff --git a/stellar-multitenant/pkg/tenant/fixtures.go b/stellar-multitenant/pkg/tenant/fixtures.go index cd082a871..528ead873 100644 --- a/stellar-multitenant/pkg/tenant/fixtures.go +++ b/stellar-multitenant/pkg/tenant/fixtures.go @@ -40,7 +40,7 @@ func ResetTenantConfigFixture(t *testing.T, ctx context.Context, dbConnectionPoo SET email_sender_type = DEFAULT, sms_sender_type = DEFAULT, enable_mfa = DEFAULT, enable_recaptcha = DEFAULT, - cors_allowed_origins = NULL, base_url = NULL, sdp_ui_base_url = NULL + base_url = NULL, sdp_ui_base_url = NULL WHERE id = $1 RETURNING * diff --git a/stellar-multitenant/pkg/tenant/manager.go b/stellar-multitenant/pkg/tenant/manager.go index cf92f3ef7..0fc38a785 100644 --- a/stellar-multitenant/pkg/tenant/manager.go +++ b/stellar-multitenant/pkg/tenant/manager.go @@ -180,11 +180,6 @@ func (m *Manager) UpdateTenantConfig(ctx context.Context, tu *TenantUpdate) (*Te args = append(args, *tu.SDPUIBaseURL) } - if tu.CORSAllowedOrigins != nil && len(tu.CORSAllowedOrigins) > 0 { - fields = append(fields, "cors_allowed_origins = ?") - args = append(args, pq.Array(tu.CORSAllowedOrigins)) - } - if tu.Status != nil { fields = append(fields, "status = ?") args = append(args, *tu.Status) diff --git a/stellar-multitenant/pkg/tenant/manager_test.go b/stellar-multitenant/pkg/tenant/manager_test.go index 265208c42..8adacbc19 100644 --- a/stellar-multitenant/pkg/tenant/manager_test.go +++ b/stellar-multitenant/pkg/tenant/manager_test.go @@ -90,15 +90,13 @@ func Test_Manager_UpdateTenantConfig(t *testing.T) { assert.True(t, tntDB.EnableReCAPTCHA) assert.Nil(t, tntDB.BaseURL) assert.Nil(t, tntDB.SDPUIBaseURL) - assert.Empty(t, tntDB.CORSAllowedOrigins) // Partial Update tnt, err := m.UpdateTenantConfig(ctx, &TenantUpdate{ - ID: tntDB.ID, - EmailSenderType: &AWSEmailSenderType, - EnableMFA: &[]bool{false}[0], - CORSAllowedOrigins: []string{"https://myorg.sdp.io", "https://myorg-dev.sdp.io"}, - SDPUIBaseURL: &[]string{"https://myorg.frontend.io"}[0], + ID: tntDB.ID, + EmailSenderType: &AWSEmailSenderType, + EnableMFA: &[]bool{false}[0], + SDPUIBaseURL: &[]string{"https://myorg.frontend.io"}[0], }) require.NoError(t, err) @@ -108,7 +106,6 @@ func Test_Manager_UpdateTenantConfig(t *testing.T) { assert.True(t, tnt.EnableReCAPTCHA) assert.Nil(t, tnt.BaseURL) assert.Equal(t, "https://myorg.frontend.io", *tnt.SDPUIBaseURL) - assert.ElementsMatch(t, []string{"https://myorg.sdp.io", "https://myorg-dev.sdp.io"}, tnt.CORSAllowedOrigins) tnt, err = m.UpdateTenantConfig(ctx, &TenantUpdate{ ID: tntDB.ID, @@ -124,7 +121,6 @@ func Test_Manager_UpdateTenantConfig(t *testing.T) { assert.False(t, tnt.EnableReCAPTCHA) assert.Equal(t, "https://myorg.backend.io", *tnt.BaseURL) assert.Equal(t, "https://myorg.frontend.io", *tnt.SDPUIBaseURL) - assert.ElementsMatch(t, []string{"https://myorg.sdp.io", "https://myorg-dev.sdp.io"}, tnt.CORSAllowedOrigins) }) } diff --git a/stellar-multitenant/pkg/tenant/tenant.go b/stellar-multitenant/pkg/tenant/tenant.go index 76f8c56b2..e3bf1057f 100644 --- a/stellar-multitenant/pkg/tenant/tenant.go +++ b/stellar-multitenant/pkg/tenant/tenant.go @@ -5,7 +5,6 @@ import ( "net/url" "time" - "github.com/lib/pq" "golang.org/x/exp/slices" ) @@ -43,30 +42,28 @@ func ParseSMSSenderType(smsSenderTypeStr string) (SMSSenderType, error) { } type Tenant struct { - ID string `json:"id" db:"id"` - Name string `json:"name" db:"name"` - EmailSenderType EmailSenderType `json:"email_sender_type" db:"email_sender_type"` - SMSSenderType SMSSenderType `json:"sms_sender_type" db:"sms_sender_type"` - EnableMFA bool `json:"enable_mfa" db:"enable_mfa"` - EnableReCAPTCHA bool `json:"enable_recaptcha" db:"enable_recaptcha"` - CORSAllowedOrigins pq.StringArray `json:"cors_allowed_origins" db:"cors_allowed_origins"` - BaseURL *string `json:"base_url" db:"base_url"` - SDPUIBaseURL *string `json:"sdp_ui_base_url" db:"sdp_ui_base_url"` - Status TenantStatus `json:"status" db:"status"` - CreatedAt time.Time `json:"created_at" db:"created_at"` - UpdatedAt time.Time `json:"updated_at" db:"updated_at"` + ID string `json:"id" db:"id"` + Name string `json:"name" db:"name"` + EmailSenderType EmailSenderType `json:"email_sender_type" db:"email_sender_type"` + SMSSenderType SMSSenderType `json:"sms_sender_type" db:"sms_sender_type"` + EnableMFA bool `json:"enable_mfa" db:"enable_mfa"` + EnableReCAPTCHA bool `json:"enable_recaptcha" db:"enable_recaptcha"` + BaseURL *string `json:"base_url" db:"base_url"` + SDPUIBaseURL *string `json:"sdp_ui_base_url" db:"sdp_ui_base_url"` + Status TenantStatus `json:"status" db:"status"` + CreatedAt time.Time `json:"created_at" db:"created_at"` + UpdatedAt time.Time `json:"updated_at" db:"updated_at"` } type TenantUpdate struct { - ID string `db:"id"` - EmailSenderType *EmailSenderType `db:"email_sender_type"` - SMSSenderType *SMSSenderType `db:"sms_sender_type"` - EnableMFA *bool `db:"enable_mfa"` - EnableReCAPTCHA *bool `db:"enable_recaptcha"` - CORSAllowedOrigins []string `db:"cors_allowed_origins"` - BaseURL *string `db:"base_url"` - SDPUIBaseURL *string `db:"sdp_ui_base_url"` - Status *TenantStatus `db:"status"` + ID string `db:"id"` + EmailSenderType *EmailSenderType `db:"email_sender_type"` + SMSSenderType *SMSSenderType `db:"sms_sender_type"` + EnableMFA *bool `db:"enable_mfa"` + EnableReCAPTCHA *bool `db:"enable_recaptcha"` + BaseURL *string `db:"base_url"` + SDPUIBaseURL *string `db:"sdp_ui_base_url"` + Status *TenantStatus `db:"status"` } type TenantStatus string @@ -112,12 +109,6 @@ func (tu *TenantUpdate) Validate() error { return fmt.Errorf("invalid SDP UI base URL") } - for _, u := range tu.CORSAllowedOrigins { - if !isValidURL(u) { - return fmt.Errorf("invalid CORS allowed origin url: %q", u) - } - } - if tu.Status != nil && !tu.Status.IsValid() { return fmt.Errorf("invalid tenant status: %q", *tu.Status) } @@ -130,7 +121,6 @@ func (tu *TenantUpdate) areAllFieldsEmpty() bool { tu.SMSSenderType == nil && tu.EnableMFA == nil && tu.EnableReCAPTCHA == nil && - tu.CORSAllowedOrigins == nil && tu.BaseURL == nil && tu.SDPUIBaseURL == nil && tu.Status == nil) diff --git a/stellar-multitenant/pkg/tenant/tenant_test.go b/stellar-multitenant/pkg/tenant/tenant_test.go index 169dbc2c4..b6217a03e 100644 --- a/stellar-multitenant/pkg/tenant/tenant_test.go +++ b/stellar-multitenant/pkg/tenant/tenant_test.go @@ -40,11 +40,6 @@ func Test_TenantUpdate_Validate(t *testing.T) { assert.EqualError(t, err, "invalid SDP UI base URL") tu.SDPUIBaseURL = nil - tu.CORSAllowedOrigins = []string{"inv@lid$"} - err = tu.Validate() - assert.EqualError(t, err, `invalid CORS allowed origin url: "inv@lid$"`) - - tu.CORSAllowedOrigins = nil tenantStatus := TenantStatus("invalid") tu.Status = &tenantStatus err = tu.Validate() @@ -53,15 +48,14 @@ func Test_TenantUpdate_Validate(t *testing.T) { t.Run("valid values", func(t *testing.T) { tu := TenantUpdate{ - ID: "abc", - EmailSenderType: &AWSEmailSenderType, - SMSSenderType: &TwilioSMSSenderType, - EnableMFA: &[]bool{true}[0], - EnableReCAPTCHA: &[]bool{true}[0], - CORSAllowedOrigins: []string{"https://myorg.sdp.io", "https://myorg-dev.sdp.io"}, - BaseURL: &[]string{"https://myorg.backend.io"}[0], - SDPUIBaseURL: &[]string{"https://myorg.frontend.io"}[0], - Status: &[]TenantStatus{ProvisionedTenantStatus}[0], + ID: "abc", + EmailSenderType: &AWSEmailSenderType, + SMSSenderType: &TwilioSMSSenderType, + EnableMFA: &[]bool{true}[0], + EnableReCAPTCHA: &[]bool{true}[0], + BaseURL: &[]string{"https://myorg.backend.io"}[0], + SDPUIBaseURL: &[]string{"https://myorg.frontend.io"}[0], + Status: &[]TenantStatus{ProvisionedTenantStatus}[0], } err := tu.Validate() assert.NoError(t, err)