Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SDP-1001] refactor migration commands to keep a consistent naming approach, in preparation for the TSS migrations (pt 1/2) #123

Merged
merged 22 commits into from
Jan 2, 2024
Merged

[SDP-1001] refactor migration commands to keep a consistent naming approach, in preparation for the TSS migrations (pt 1/2) #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
2cad4c2
Small reorganization on the migration CLI commands.
marcelosalloum Dec 12, 2023
70f445b
Update the `migrate up|down` CMD to `sdp migrate up|down`.
marcelosalloum Dec 12, 2023
e5c57e0
Update description of the --all and --tenant-id flags.
marcelosalloum Dec 12, 2023
302e4e9
Rename the `tenant migrate ...` command to `admin migrate ...`
marcelosalloum Dec 12, 2023
aa5cac8
Update docker migrations.
marcelosalloum Dec 12, 2023
9944d63
Fix flag default value.
marcelosalloum Dec 13, 2023
b24bd94
Rename variables and folders for consistency.
marcelosalloum Dec 13, 2023
4aebef7
Update wording.
marcelosalloum Dec 13, 2023
671fc97
Fix command that wasn't being inserted.
marcelosalloum Dec 13, 2023
fd00a3c
Add TODO.
marcelosalloum Dec 13, 2023
1c0535c
[WIP] Polish wording.
marcelosalloum Dec 13, 2023
1a4bc7d
Fix imports.
marcelosalloum Dec 13, 2023
5ecc8a1
Fix broken tests.
marcelosalloum Dec 14, 2023
94bb4a7
Rename gorp_migrations to sdp_migrations, for consistency.
marcelosalloum Dec 14, 2023
f7547df
Rename table `migrations` to `admin_migrations`.
marcelosalloum Dec 14, 2023
04261ae
Fix tests related to the new `sdp` hierarchy in `db sdp migrate up | …
marcelosalloum Dec 14, 2023
4836d63
Separate child commands in different methods.
marcelosalloum Dec 14, 2023
44140a5
Polishes.
marcelosalloum Dec 14, 2023
52c2391
Rename constants.
marcelosalloum Dec 14, 2023
dc903ea
Update wording.
marcelosalloum Dec 14, 2023
4c6696c
Add reusable method.
marcelosalloum Dec 14, 2023
88995fc
Fix migrations on helmchart.
marcelosalloum Jan 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
226 changes: 103 additions & 123 deletions cmd/db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@ import (
"github.com/spf13/cobra"
"github.com/stellar/go/support/config"
"github.com/stellar/go/support/log"
"github.com/stellar/stellar-disbursement-platform-backend/cmd/utils"
"github.com/stellar/stellar-disbursement-platform-backend/db"
authmigrations "github.com/stellar/stellar-disbursement-platform-backend/db/migrations/auth-migrations"
sdpmigrations "github.com/stellar/stellar-disbursement-platform-backend/db/migrations/sdp-migrations"
"github.com/stellar/stellar-disbursement-platform-backend/internal/services"
"github.com/stellar/stellar-disbursement-platform-backend/internal/utils"
sdpUtils "github.com/stellar/stellar-disbursement-platform-backend/internal/utils"
tenantcli "github.com/stellar/stellar-disbursement-platform-backend/stellar-multitenant/pkg/cli"
"github.com/stellar/stellar-disbursement-platform-backend/stellar-multitenant/pkg/tenant"
)
Expand All @@ -24,24 +25,23 @@ type databaseCommandConfigOptions struct {
All bool
TenantID string
}

type DatabaseCommand struct{}

func (c *DatabaseCommand) Command() *cobra.Command {
opts := databaseCommandConfigOptions{}

// TODO: tie these configs only where needed
configOptions := config.ConfigOptions{
{
Name: "all",
Usage: "Apply the migrations to all tenants.",
Usage: "Apply the migrations to all tenants. Either --tenant-id or --all must be set, but the --all option will be ignored if --tenant-id is set.",
OptType: types.Bool,
FlagDefault: false,
ConfigKey: &opts.All,
Required: false,
},
{
Name: "tenant-id",
Usage: "The tenant ID where the migrations will be applied.",
Usage: "The tenant ID where the migrations will be applied. Either --tenant-id or --all must be set, but the --all option will be ignored if --tenant-id is set.",
OptType: types.String,
ConfigKey: &opts.TenantID,
Required: false,
Expand All @@ -52,43 +52,39 @@ func (c *DatabaseCommand) Command() *cobra.Command {
Use: "db",
Short: "Database related commands",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
utils.PropagatePersistentPreRun(cmd, args)
configOptions.Require()
if err := configOptions.SetValues(); err != nil {
log.Fatalf("Error setting values of config options: %s", err.Error())
}
},
Run: func(cmd *cobra.Command, _ []string) {
err := cmd.Help()
if err != nil {
log.Fatalf("Error calling help command: %s", err.Error())
log.Ctx(cmd.Context()).Fatalf("Error setting values of config options: %s", err.Error())
}
},
RunE: utils.CallHelpCommand,
}

// migrate CMD
sdpMigrateCmd := c.migrateCmd()
authMigrateCmd := c.migrateCmd()
cmd.AddCommand(sdpMigrateCmd)
// ADD COMMANDs:
cmd.AddCommand(c.setupForNetworkCmd(cmd.Context(), &opts)) // 'setup-for-network'
cmd.AddCommand(c.sdpPerTenantMigrationsCmd(cmd.Context(), &opts)) // 'sdp migrate up|down'
cmd.AddCommand(c.authPerTenantMigrationsCmd(cmd.Context(), &opts)) // 'auth migrate up|down'
cmd.AddCommand(c.adminMigrationsCmd(cmd.Context(), &opts)) // 'admin migrate up|down'

// migrate Up CMD
sdpMigrateCmd.AddCommand(c.migrateUpCmd(&opts))
authMigrateCmd.AddCommand(c.migrateUpCmd(&opts))
if err := configOptions.Init(cmd); err != nil {
log.Ctx(cmd.Context()).Fatalf("initializing config options: %v", err)
}

// migrate Down CMD
sdpMigrateCmd.AddCommand(c.migrateDownCmd(&opts))
authMigrateCmd.AddCommand(c.migrateDownCmd(&opts))
return cmd
}

setupForNetwork := &cobra.Command{
// setupForNetworkCmd returns a cobra.Command responsible for setting up the assets and wallets registered in the
// database based on the network passphrase.
func (c *DatabaseCommand) setupForNetworkCmd(ctx context.Context, opts *databaseCommandConfigOptions) *cobra.Command {
return &cobra.Command{
Use: "setup-for-network",
Short: "Set up the assets and wallets registered in the database based on the network passphrase.",
Long: "Set up the assets and wallets registered in the database based on the network passphrase. It inserts or updates the entries of these tables according with the configured Network Passphrase.",
Run: func(cmd *cobra.Command, args []string) {
ctx := cmd.Context()

if err := c.validateFlags(&opts); err != nil {
if err := c.validateFlags(opts); err != nil {
log.Ctx(ctx).Fatal(err.Error())
}

Expand All @@ -101,19 +97,19 @@ func (c *DatabaseCommand) Command() *cobra.Command {
if dsn, ok := tenantsDSNMap[opts.TenantID]; ok {
tenantsDSNMap = map[string]string{opts.TenantID: dsn}
} else {
log.Fatalf("tenant ID %s does not exist", opts.TenantID)
log.Ctx(ctx).Fatalf("tenant ID %s does not exist", opts.TenantID)
}
}

for tenantID, dsn := range tenantsDSNMap {
log.Infof("running for tenant ID %s", tenantID)
log.Ctx(ctx).Infof("running for tenant ID %s", tenantID)
dbConnectionPool, err := db.OpenDBConnectionPool(dsn)
if err != nil {
log.Ctx(ctx).Fatalf("error connection to the database: %s", err.Error())
}
defer dbConnectionPool.Close()

networkType, err := utils.GetNetworkTypeFromNetworkPassphrase(globalOptions.networkPassphrase)
networkType, err := sdpUtils.GetNetworkTypeFromNetworkPassphrase(globalOptions.networkPassphrase)
if err != nil {
log.Ctx(ctx).Fatalf("error getting network type: %s", err.Error())
}
Expand All @@ -128,142 +124,118 @@ func (c *DatabaseCommand) Command() *cobra.Command {
}
},
}
cmd.AddCommand(setupForNetwork)
}

stellarAuthMigrateCmd := &cobra.Command{
Use: "auth",
Short: "Stellar Auth schema migration helpers",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
},
Run: func(cmd *cobra.Command, args []string) {
if err := cmd.Help(); err != nil {
log.Fatalf("Error calling help command: %s", err.Error())
}
},
// sdpPerTenantMigrationsCmd returns a cobra.Command responsible for running the migrations of the `sdp-migrations`
// folder on the desired tenant(s).
func (c *DatabaseCommand) sdpPerTenantMigrationsCmd(ctx context.Context, opts *databaseCommandConfigOptions) *cobra.Command {
sdpCmd := &cobra.Command{
Use: "sdp",
Short: "Stellar Disbursement Platform's per-tenant schema migration helpers. Will execute the migrations of the `sdp-migrations` folder on the desired tenant, according with the --all or --tenant-id configs. The migrations are tracked in the table `sdp_migrations`.",
PersistentPreRun: utils.PropagatePersistentPreRun,
RunE: utils.CallHelpCommand,
}
stellarAuthMigrateCmd.AddCommand(authMigrateCmd)
sdpCmd.AddCommand(c.migrateCmd(ctx, opts))
return sdpCmd
}

tenantMigrateCmd := &cobra.Command{
Use: "tenant",
Short: "Stellar Multi-Tenant migration helpers",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
},
Run: func(cmd *cobra.Command, args []string) {
if err := cmd.Help(); err != nil {
log.Fatalf("Error calling help command: %s", err.Error())
}
},
// authPerTenantMigrationsCmd returns a cobra.Command responsible for running the migrations of the `auth-migrations`
// folder on the desired tenant(s).
func (c *DatabaseCommand) authPerTenantMigrationsCmd(ctx context.Context, opts *databaseCommandConfigOptions) *cobra.Command {
authCmd := &cobra.Command{
Use: "auth",
Short: "Authentication's per-tenant schema migration helpers. Will execute the migrations of the `auth-migrations` folder on the desired tenant, according with the --all or --tenant-id configs. The migrations are tracked in the table `auth_migrations`.",
PersistentPreRun: utils.PropagatePersistentPreRun,
RunE: utils.CallHelpCommand,
}
tenantMigrateCmd.AddCommand(tenantcli.MigrateCmd(dbConfigOptionFlagName))

// Add `auth` as a sub-command to `db`. Usage: db auth migrate up
cmd.AddCommand(stellarAuthMigrateCmd)

// Add `tenant` as a sub-command to `db`. Usage: db tenant migrate up
cmd.AddCommand(tenantMigrateCmd)
authCmd.AddCommand(c.migrateCmd(ctx, opts))
return authCmd
}

if err := configOptions.Init(cmd); err != nil {
log.Fatalf("initializing config options: %v", err)
// adminMigrationsCmd returns a cobra.Command responsible for running the migrations of the `admin-migrations`
// folder, that are used to configure the multi-tenant module that manages the tenants.
func (c *DatabaseCommand) adminMigrationsCmd(ctx context.Context, opts *databaseCommandConfigOptions) *cobra.Command {
adminCmd := &cobra.Command{
Use: "admin",
Short: "Admin migrations used to configure the multi-tenant module that manages the tenants. Will execute the migrations of the `admin-migrations` and the migrations are tracked in the table `admin_migrations`.",
PersistentPreRun: utils.PropagatePersistentPreRun,
RunE: utils.CallHelpCommand,
}

return cmd
adminCmd.AddCommand(tenantcli.MigrateCmd(dbConfigOptionFlagName))
return adminCmd
}

func (c *DatabaseCommand) migrateCmd() *cobra.Command {
return &cobra.Command{
Use: "migrate",
Short: "Schema migration helpers",
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
},
Run: func(cmd *cobra.Command, _ []string) {
err := cmd.Help()
if err != nil {
log.Fatalf("Error calling help command: %s", err.Error())
}
},
// migrateCmd returns a cobra.Command responsible for running the database migrations.
func (c *DatabaseCommand) migrateCmd(ctx context.Context, opts *databaseCommandConfigOptions) *cobra.Command {
migrateCmd := &cobra.Command{
Use: "migrate",
Short: "Schema migration helpers",
PersistentPreRun: utils.PropagatePersistentPreRun,
RunE: utils.CallHelpCommand,
}
}

func (c *DatabaseCommand) migrateUpCmd(opts *databaseCommandConfigOptions) *cobra.Command {
return &cobra.Command{
Use: "up",
Short: "Migrates database up [count]",
Args: cobra.MaximumNArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
},
migrateUpCmd := cobra.Command{
Use: "up",
Short: "Migrates database up [count] migrations",
Args: cobra.MaximumNArgs(1),
PersistentPreRun: utils.PropagatePersistentPreRun,
Run: func(cmd *cobra.Command, args []string) {
var count int
if len(args) > 0 {
var err error
count, err = strconv.Atoi(args[0])
if err != nil {
log.Fatalf("Invalid [count] argument: %s", args[0])
log.Ctx(ctx).Fatalf("Invalid [count] argument: %s", args[0])
}
}

migrationFiles := sdpmigrations.FS
migrationTableName := db.StellarSDPMigrationsTableName
migrationTableName := db.StellarPerTenantSDPMigrationsTableName

migrateCmd := cmd.Parent()
if migrateCmd.Parent().Name() == "auth" {
if cmd.Parent().Parent().Name() == "auth" {
migrationFiles = authmigrations.FS
migrationTableName = db.StellarAuthMigrationsTableName
migrationTableName = db.StellarPerTenantAuthMigrationsTableName
}

if err := c.executeMigrate(cmd.Context(), opts, migrate.Up, count, migrationFiles, migrationTableName); err != nil {
log.Fatalf("Error executing migrate up: %v", err)
log.Ctx(ctx).Fatalf("Error executing migrate up: %v", err)
}
},
}
}

func (c *DatabaseCommand) migrateDownCmd(opts *databaseCommandConfigOptions) *cobra.Command {
return &cobra.Command{
Use: "down [count]",
Short: "Migrates database down [count] migrations",
Args: cobra.ExactArgs(1),
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if cmd.Parent().PersistentPreRun != nil {
cmd.Parent().PersistentPreRun(cmd.Parent(), args)
}
},
migrateDownCmd := &cobra.Command{
Use: "down [count]",
Short: "Migrates database down [count] migrations",
Args: cobra.ExactArgs(1),
PersistentPreRun: utils.PropagatePersistentPreRun,
Run: func(cmd *cobra.Command, args []string) {
count, err := strconv.Atoi(args[0])
if err != nil {
log.Fatalf("Invalid [count] argument: %s", args[0])
log.Ctx(ctx).Fatalf("Invalid [count] argument: %s", args[0])
}

migrationFiles := sdpmigrations.FS
migrationTableName := db.StellarSDPMigrationsTableName
migrationTableName := db.StellarPerTenantSDPMigrationsTableName

migrateCmd := cmd.Parent()
if migrateCmd.Parent().Name() == "auth" {
if cmd.Parent().Parent().Name() == "auth" {
migrationFiles = authmigrations.FS
migrationTableName = db.StellarAuthMigrationsTableName
migrationTableName = db.StellarPerTenantAuthMigrationsTableName
}

if err := c.executeMigrate(cmd.Context(), opts, migrate.Down, count, migrationFiles, migrationTableName); err != nil {
log.Fatalf("Error executing migrate down: %v", err)
log.Ctx(ctx).Fatalf("Error executing migrate down: %v", err)
}
},
}

migrateCmd.AddCommand(&migrateUpCmd)
migrateCmd.AddCommand(migrateDownCmd)
return migrateCmd
}

func (c *DatabaseCommand) executeMigrate(ctx context.Context, opts *databaseCommandConfigOptions, dir migrate.MigrationDirection, count int, migrationFiles embed.FS, tableName db.MigrationTableName) error {
if err := c.validateFlags(opts); err != nil {
log.Fatal(err.Error())
log.Ctx(ctx).Fatal(err.Error())
}

tenantsDSNMap, err := c.getTenantsDSN(ctx, globalOptions.databaseURL)
Expand All @@ -275,15 +247,15 @@ func (c *DatabaseCommand) executeMigrate(ctx context.Context, opts *databaseComm
if dsn, ok := tenantsDSNMap[opts.TenantID]; ok {
tenantsDSNMap = map[string]string{opts.TenantID: dsn}
} else {
log.Fatalf("tenant ID %s does not exist", opts.TenantID)
log.Ctx(ctx).Fatalf("tenant ID %s does not exist", opts.TenantID)
}
}

for tenantID, dsn := range tenantsDSNMap {
log.Infof("applying migrations on tenant ID %s", tenantID)
log.Ctx(ctx).Infof("Applying migrations on tenant ID %s", tenantID)
err = c.applyMigrations(dsn, dir, count, migrationFiles, tableName)
if err != nil {
log.Fatalf("Error migrating database Up: %s", err.Error())
log.Ctx(ctx).Fatalf("Error migrating database Up: %s", err.Error())
}
}

Expand All @@ -299,11 +271,19 @@ func (c *DatabaseCommand) applyMigrations(dbURL string, dir migrate.MigrationDir
if numMigrationsRun == 0 {
log.Info("No migrations applied.")
} else {
log.Infof("Successfully applied %d migrations.", numMigrationsRun)
log.Infof("Successfully applied %d migrations %s.", numMigrationsRun, migrationDirectionStr(dir))
}
return nil
}

// migrationDirectionStr returns a string representation of the migration direction (up or down).
func migrationDirectionStr(dir migrate.MigrationDirection) string {
if dir == migrate.Up {
return "up"
}
return "down"
}

func (c *DatabaseCommand) validateFlags(opts *databaseCommandConfigOptions) error {
if !opts.All && opts.TenantID == "" {
return fmt.Errorf(
Expand Down
Loading
Loading