Skip to content

Commit

Permalink
Merge pull request #2363 from GoogleCloudPlatform/release-candidate
Browse files Browse the repository at this point in the history 
Release v1.30.0
  • Loading branch information
@harshthakkar01
harshthakkar01 authored Mar 18, 2024
2 parents c024e72 + ae04459 commit 08ae77e
Show file tree
Hide file tree
Showing 224 changed files with 4,751 additions and 1,853 deletions.
11 changes: 10 additions & 1 deletion .pre-commit-config.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ repos:
entry: python3 tools/duplicate-diff.py
language: python
language_version: python3
files: '.*(\.sh|\.tf|\.tftpl)$'
files: '.*(\.sh|\.tf|\.tftpl|\.tpl)$'
pass_filenames: true
require_serial: true
- id: module-label-check
Expand All @@ -65,6 +65,15 @@ repos:
args: ['-c', '"Google LLC"', '-l', 'apache']
exclude: docs/videos/healthcare-and-life-sciences/lysozyme-example/submit.sh
pass_filenames: true
- id: tests-metadata
name: tests-metadata
entry: python3 tools/cloud-build/daily-tests/validate_tests_metadata.py
language: python
language_version: python3
additional_dependencies: ['pyyaml']
files: 'tools/cloud-build/daily-tests/builds/.*\.yaml'
pass_filenames: false
require_serial: true

- repo: https://github.com/dnephin/pre-commit-golang
rev: v0.5.1
Expand Down
106 changes: 45 additions & 61 deletions cmd/create.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,105 +33,91 @@ import (
"gopkg.in/yaml.v3"
)

const msgCLIVars = "Comma-separated list of name=value variables to override YAML configuration. Can be used multiple times."
const msgCLIBackendConfig = "Comma-separated list of name=value variables to set Terraform backend configuration. Can be used multiple times."

func init() {
createCmd.Flags().StringVarP(&bpFilenameDeprecated, "config", "c", "", "")
cobra.CheckErr(createCmd.Flags().MarkDeprecated("config",
"please see the command usage for more details."))

deploymentFileFlag := "deployment-file"
createCmd.Flags().StringVarP(&deploymentFile, deploymentFileFlag, "d", "",
"Toolkit Deployment File.")
createCmd.Flags().MarkHidden(deploymentFileFlag)
createCmd.MarkFlagFilename(deploymentFileFlag, "yaml", "yml")
createCmd.Flags().StringVarP(&outputDir, "out", "o", "",
func addCreateFlags(c *cobra.Command) *cobra.Command {
c.Flags().StringVarP(&createFlags.outputDir, "out", "o", "",
"Sets the output directory where the HPC deployment directory will be created.")
createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars)
createCmd.Flags().StringSliceVar(&cliBEConfigVars, "backend-config", nil, msgCLIBackendConfig)
createCmd.Flags().StringVarP(&validationLevel, "validation-level", "l", "WARNING", validationLevelDesc)
createCmd.Flags().StringSliceVar(&validatorsToSkip, "skip-validators", nil, skipValidatorsDesc)
createCmd.Flags().BoolVarP(&overwriteDeployment, "overwrite-deployment", "w", false,
c.Flags().BoolVarP(&createFlags.overwriteDeployment, "overwrite-deployment", "w", false,
"If specified, an existing deployment directory is overwritten by the new deployment. \n"+
"Note: Terraform state IS preserved. \n"+
"Note: Terraform workspaces are NOT supported (behavior undefined). \n"+
"Note: Packer is NOT supported.")
createCmd.Flags().BoolVar(&forceOverwrite, "force", false,
c.Flags().BoolVar(&createFlags.forceOverwrite, "force", false,
"Forces overwrite of existing deployment directory. \n"+
"If set, --overwrite-deployment is implied. \n"+
"No validation is performed on the existing deployment directory.")
return addExpandFlags(c, false /*addOutFlag to avoid clash with "create" `out` flag*/)
}

func init() {
rootCmd.AddCommand(createCmd)
}

var (
bpFilenameDeprecated string
deploymentFile string
outputDir string
cliVariables []string

cliBEConfigVars []string
overwriteDeployment bool
forceOverwrite bool
validationLevel string
validationLevelDesc = "Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")"
validatorsToSkip []string
skipValidatorsDesc = "Validators to skip"

createCmd = &cobra.Command{
Use: "create BLUEPRINT_NAME",
createFlags = struct {
outputDir string
overwriteDeployment bool
forceOverwrite bool
}{}

createCmd = addCreateFlags(&cobra.Command{
Use: "create <BLUEPRINT_FILE>",
Short: "Create a new deployment.",
Long: "Create a new deployment based on a provided blueprint.",
Run: runCreateCmd,
Args: cobra.ExactArgs(1),
Args: cobra.MatchAll(cobra.ExactArgs(1), checkExists),
ValidArgsFunction: filterYaml,
}
})
)

func runCreateCmd(cmd *cobra.Command, args []string) {
bp := expandOrDie(args[0], deploymentFile)
deplDir := filepath.Join(outputDir, bp.DeploymentName())
checkErr(checkOverwriteAllowed(deplDir, bp, overwriteDeployment, forceOverwrite))
checkErr(modulewriter.WriteDeployment(bp, deplDir))

deplDir := doCreate(args[0])
logging.Info("To deploy your infrastructure please run:")
logging.Info("")
logging.Info(boldGreen("%s deploy %s"), execPath(), deplDir)
logging.Info("")
printAdvancedInstructionsMessage(deplDir)
}

func doCreate(path string) string {
bp := expandOrDie(path)
deplDir := filepath.Join(createFlags.outputDir, bp.DeploymentName())
checkErr(checkOverwriteAllowed(deplDir, bp, createFlags.overwriteDeployment, createFlags.forceOverwrite))
checkErr(modulewriter.WriteDeployment(bp, deplDir))
return deplDir
}

func printAdvancedInstructionsMessage(deplDir string) {
logging.Info("Find instructions for cleanly destroying infrastructure and advanced manual")
logging.Info("deployment instructions at:")
logging.Info("")
logging.Info(modulewriter.InstructionsPath(deplDir))
}

func expandOrDie(path string, dPath string) config.Blueprint {
// TODO: move to expand.go
func expandOrDie(path string) config.Blueprint {
bp, ctx, err := config.NewBlueprint(path)
if err != nil {
logging.Fatal(renderError(err, ctx))
}

var ds config.DeploymentSettings
var dCtx config.YamlCtx
if dPath != "" {
ds, dCtx, err = config.NewDeploymentSettings(dPath)
if expandFlags.deploymentFile != "" {
ds, dCtx, err = config.NewDeploymentSettings(expandFlags.deploymentFile)
if err != nil {
logging.Fatal(renderError(err, dCtx))
}
}
if err := setCLIVariables(&ds, cliVariables); err != nil {
if err := setCLIVariables(&ds, expandFlags.cliVariables); err != nil {
logging.Fatal("Failed to set the variables at CLI: %v", err)
}
if err := setBackendConfig(&ds, cliBEConfigVars); err != nil {
if err := setBackendConfig(&ds, expandFlags.cliBEConfigVars); err != nil {
logging.Fatal("Failed to set the backend config at CLI: %v", err)
}

mergeDeploymentSettings(&bp, ds)

checkErr(setValidationLevel(&bp, validationLevel))
checkErr(setValidationLevel(&bp, expandFlags.validationLevel))
skipValidators(&bp)

if bp.GhpcVersion != "" {
Expand All @@ -148,6 +134,7 @@ func expandOrDie(path string, dPath string) config.Blueprint {
return bp
}

// TODO: move to expand.go
func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {
err := validators.Execute(bp)
if err == nil {
Expand Down Expand Up @@ -181,6 +168,7 @@ func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {

}

// TODO: move to expand.go
func setCLIVariables(ds *config.DeploymentSettings, s []string) error {
for _, cliVar := range s {
arr := strings.SplitN(cliVar, "=", 2)
Expand All @@ -194,11 +182,12 @@ func setCLIVariables(ds *config.DeploymentSettings, s []string) error {
if err := yaml.Unmarshal([]byte(arr[1]), &v); err != nil {
return fmt.Errorf("invalid input: unable to convert '%s' value '%s' to known type", key, arr[1])
}
ds.Vars.Set(key, v.Unwrap())
ds.Vars = ds.Vars.With(key, v.Unwrap())
}
return nil
}

// TODO: move to expand.go
func setBackendConfig(ds *config.DeploymentSettings, s []string) error {
if len(s) == 0 {
return nil // no op
Expand All @@ -216,7 +205,7 @@ func setBackendConfig(ds *config.DeploymentSettings, s []string) error {
case "type":
be.Type = value
default:
be.Configuration.Set(key, cty.StringVal(value))
be.Configuration = be.Configuration.With(key, cty.StringVal(value))
}
}
ds.TerraformBackendDefaults = be
Expand All @@ -225,7 +214,7 @@ func setBackendConfig(ds *config.DeploymentSettings, s []string) error {

func mergeDeploymentSettings(bp *config.Blueprint, ds config.DeploymentSettings) error {
for k, v := range ds.Vars.Items() {
bp.Vars.Set(k, v)
bp.Vars = bp.Vars.With(k, v)
}
if ds.TerraformBackendDefaults.Type != "" {
bp.TerraformBackendDefaults = ds.TerraformBackendDefaults
Expand All @@ -234,6 +223,7 @@ func mergeDeploymentSettings(bp *config.Blueprint, ds config.DeploymentSettings)
}

// SetValidationLevel allows command-line tools to set the validation level
// TODO: move to expand.go
func setValidationLevel(bp *config.Blueprint, s string) error {
switch s {
case "ERROR":
Expand All @@ -248,19 +238,13 @@ func setValidationLevel(bp *config.Blueprint, s string) error {
return nil
}

// TODO: move to expand.go
func skipValidators(bp *config.Blueprint) {
for _, v := range validatorsToSkip {
for _, v := range expandFlags.validatorsToSkip {
bp.SkipValidator(v)
}
}

func filterYaml(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if len(args) != 0 {
return nil, cobra.ShellCompDirectiveNoFileComp
}
return []string{"yaml", "yml"}, cobra.ShellCompDirectiveFilterFileExt
}

func forceErr(err error) error {
return config.HintError{
Err: err,
Expand Down Expand Up @@ -299,11 +283,11 @@ func checkOverwriteAllowed(depDir string, bp config.Blueprint, overwriteFlag boo
}

newGroups := map[config.GroupName]bool{}
for _, g := range bp.DeploymentGroups {
for _, g := range bp.Groups {
newGroups[g.Name] = true
}

for _, g := range prev.DeploymentGroups {
for _, g := range prev.Groups {
if !newGroups[g.Name] {
return forceErr(fmt.Errorf("you are attempting to remove a deployment group %q, which is not supported", g.Name))
}
Expand Down
29 changes: 13 additions & 16 deletions cmd/create_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,9 @@ import (
)

func (s *MySuite) TestSetCLIVariables(c *C) {
ds := config.DeploymentSettings{}
ds.Vars.Set("deployment_name", cty.StringVal("bush"))
ds := config.DeploymentSettings{
Vars: config.Dict{}.
With("deployment_name", cty.StringVal("bush"))}

vars := []string{
"project_id=cli_test_project_id",
Expand Down Expand Up @@ -103,18 +104,14 @@ func (s *MySuite) TestSetBackendConfig(c *C) {

func (s *MySuite) TestMergeDeploymentSettings(c *C) {
ds1 := config.DeploymentSettings{
Vars: config.NewDict(map[string]cty.Value{
"project_id": cty.StringVal("ds_test_project_id"),
"deployment_name": cty.StringVal("ds_deployment_name"),
}),
}
Vars: config.Dict{}.
With("project_id", cty.StringVal("ds_test_project_id")).
With("deployment_name", cty.StringVal("ds_deployment_name"))}

bp1 := config.Blueprint{
Vars: config.NewDict(map[string]cty.Value{
"project_id": cty.StringVal("bp_test_project_id"),
"example_var": cty.StringVal("bp_example_value"),
}),
}
Vars: config.Dict{}.
With("project_id", cty.StringVal("bp_test_project_id")).
With("example_var", cty.StringVal("bp_example_value"))}

// test priority-based merging of deployment variables
mergeDeploymentSettings(&bp1, ds1)
Expand Down Expand Up @@ -275,7 +272,7 @@ func (s *MySuite) TestIsOverwriteAllowed_Present(c *C) {

prev := config.Blueprint{
GhpcVersion: "TaleOfBygoneYears",
DeploymentGroups: []config.DeploymentGroup{
Groups: []config.Group{
{Name: "isildur"}}}
if err := prev.Export(filepath.Join(artDir, "expanded_blueprint.yaml")); err != nil {
c.Fatal(err)
Expand All @@ -285,7 +282,7 @@ func (s *MySuite) TestIsOverwriteAllowed_Present(c *C) {
{ // Superset
bp := config.Blueprint{
GhpcVersion: "TaleOfBygoneYears",
DeploymentGroups: []config.DeploymentGroup{
Groups: []config.Group{
{Name: "isildur"},
{Name: "elendil"}}}
c.Check(checkOverwriteAllowed(p, bp, noW, noForce), ErrorMatches, ".* already exists, use -w to overwrite")
Expand All @@ -295,7 +292,7 @@ func (s *MySuite) TestIsOverwriteAllowed_Present(c *C) {
{ // Version mismatch
bp := config.Blueprint{
GhpcVersion: "TheAlloyOfLaw",
DeploymentGroups: []config.DeploymentGroup{
Groups: []config.Group{
{Name: "isildur"}}}
c.Check(checkOverwriteAllowed(p, bp, noW, noForce), ErrorMatches, ".*ghpc_version has changed.*")
c.Check(checkOverwriteAllowed(p, bp, yesW, noForce), ErrorMatches, ".*ghpc_version has changed.*")
Expand All @@ -305,7 +302,7 @@ func (s *MySuite) TestIsOverwriteAllowed_Present(c *C) {
{ // Subset
bp := config.Blueprint{
GhpcVersion: "TaleOfBygoneYears",
DeploymentGroups: []config.DeploymentGroup{
Groups: []config.Group{
{Name: "aragorn"}}}
c.Check(checkOverwriteAllowed(p, bp, noW, noForce), ErrorMatches, `.* already exists, use -w to overwrite`)
c.Check(checkOverwriteAllowed(p, bp, yesW, noForce), ErrorMatches, `.*remove a deployment group "isildur".*`)
Expand Down
Loading

0 comments on commit 08ae77e

Please sign in to comment.