Merge pull request #2307 from GoogleCloudPlatform/release-candidate

Release v1.29.0

.github/workflows/pr-precommit.yml

@@ -44,6 +44,10 @@ jobs:
     - uses: terraform-linters/setup-tflint@v4
       with:
         tflint_version: v0.49.0
+    - run: tflint --init
+      env:
+        # https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/plugins.md#avoiding-rate-limiting
+        GITHUB_TOKEN: ${{ github.token }}
     - uses: pre-commit/[email protected]
     - uses: pre-commit-ci/[email protected]
       # this if statement looks funny but it ensures that this step runs

cmd/create.go

@@ -41,6 +41,11 @@ func init() {
 	cobra.CheckErr(createCmd.Flags().MarkDeprecated("config",
 		"please see the command usage for more details."))
 
+	deploymentFileFlag := "deployment-file"
+	createCmd.Flags().StringVarP(&deploymentFile, deploymentFileFlag, "d", "",
+		"Toolkit Deployment File.")
+	createCmd.Flags().MarkHidden(deploymentFileFlag)
+	createCmd.MarkFlagFilename(deploymentFileFlag, "yaml", "yml")
 	createCmd.Flags().StringVarP(&outputDir, "out", "o", "",
 		"Sets the output directory where the HPC deployment directory will be created.")
 	createCmd.Flags().StringSliceVar(&cliVariables, "vars", nil, msgCLIVars)
@@ -61,6 +66,7 @@ func init() {
 
 var (
 	bpFilenameDeprecated string
+	deploymentFile       string
 	outputDir            string
 	cliVariables         []string
 
@@ -83,10 +89,10 @@ var (
 )
 
 func runCreateCmd(cmd *cobra.Command, args []string) {
-	dc := expandOrDie(args[0])
-	deplDir := filepath.Join(outputDir, dc.Config.DeploymentName())
-	checkErr(checkOverwriteAllowed(deplDir, dc.Config, overwriteDeployment, forceOverwrite))
-	checkErr(modulewriter.WriteDeployment(dc, deplDir))
+	bp := expandOrDie(args[0], deploymentFile)
+	deplDir := filepath.Join(outputDir, bp.DeploymentName())
+	checkErr(checkOverwriteAllowed(deplDir, bp, overwriteDeployment, forceOverwrite))
+	checkErr(modulewriter.WriteDeployment(bp, deplDir))
 
 	logging.Info("To deploy your infrastructure please run:")
 	logging.Info("")
@@ -102,33 +108,44 @@ func printAdvancedInstructionsMessage(deplDir string) {
 	logging.Info(modulewriter.InstructionsPath(deplDir))
 }
 
-func expandOrDie(path string) config.DeploymentConfig {
-	dc, ctx, err := config.NewDeploymentConfig(path)
+func expandOrDie(path string, dPath string) config.Blueprint {
+	bp, ctx, err := config.NewBlueprint(path)
 	if err != nil {
 		logging.Fatal(renderError(err, ctx))
 	}
-	// Set properties from CLI
-	if err := setCLIVariables(&dc.Config, cliVariables); err != nil {
+
+	var ds config.DeploymentSettings
+	var dCtx config.YamlCtx
+	if dPath != "" {
+		ds, dCtx, err = config.NewDeploymentSettings(dPath)
+		if err != nil {
+			logging.Fatal(renderError(err, dCtx))
+		}
+	}
+	if err := setCLIVariables(&ds, cliVariables); err != nil {
 		logging.Fatal("Failed to set the variables at CLI: %v", err)
 	}
-	if err := setBackendConfig(&dc.Config, cliBEConfigVars); err != nil {
+	if err := setBackendConfig(&ds, cliBEConfigVars); err != nil {
 		logging.Fatal("Failed to set the backend config at CLI: %v", err)
 	}
-	checkErr(setValidationLevel(&dc.Config, validationLevel))
-	skipValidators(&dc)
 
-	if dc.Config.GhpcVersion != "" {
+	mergeDeploymentSettings(&bp, ds)
+
+	checkErr(setValidationLevel(&bp, validationLevel))
+	skipValidators(&bp)
+
+	if bp.GhpcVersion != "" {
 		logging.Info("ghpc_version setting is ignored.")
 	}
-	dc.Config.GhpcVersion = GitCommitInfo
+	bp.GhpcVersion = GitCommitInfo
 
 	// Expand the blueprint
-	if err := dc.ExpandConfig(); err != nil {
+	if err := bp.Expand(); err != nil {
 		logging.Fatal(renderError(err, ctx))
 	}
 
-	validateMaybeDie(dc.Config, ctx)
-	return dc
+	validateMaybeDie(bp, ctx)
+	return bp
 }
 
 func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {
@@ -164,7 +181,7 @@ func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {
 
 }
 
-func setCLIVariables(bp *config.Blueprint, s []string) error {
+func setCLIVariables(ds *config.DeploymentSettings, s []string) error {
 	for _, cliVar := range s {
 		arr := strings.SplitN(cliVar, "=", 2)
 
@@ -177,12 +194,12 @@ func setCLIVariables(bp *config.Blueprint, s []string) error {
 		if err := yaml.Unmarshal([]byte(arr[1]), &v); err != nil {
 			return fmt.Errorf("invalid input: unable to convert '%s' value '%s' to known type", key, arr[1])
 		}
-		bp.Vars.Set(key, v.Unwrap())
+		ds.Vars.Set(key, v.Unwrap())
 	}
 	return nil
 }
 
-func setBackendConfig(bp *config.Blueprint, s []string) error {
+func setBackendConfig(ds *config.DeploymentSettings, s []string) error {
 	if len(s) == 0 {
 		return nil // no op
 	}
@@ -202,7 +219,17 @@ func setBackendConfig(bp *config.Blueprint, s []string) error {
 			be.Configuration.Set(key, cty.StringVal(value))
 		}
 	}
-	bp.TerraformBackendDefaults = be
+	ds.TerraformBackendDefaults = be
+	return nil
+}
+
+func mergeDeploymentSettings(bp *config.Blueprint, ds config.DeploymentSettings) error {
+	for k, v := range ds.Vars.Items() {
+		bp.Vars.Set(k, v)
+	}
+	if ds.TerraformBackendDefaults.Type != "" {
+		bp.TerraformBackendDefaults = ds.TerraformBackendDefaults
+	}
 	return nil
 }
 
@@ -221,9 +248,9 @@ func setValidationLevel(bp *config.Blueprint, s string) error {
 	return nil
 }
 
-func skipValidators(dc *config.DeploymentConfig) {
+func skipValidators(bp *config.Blueprint) {
 	for _, v := range validatorsToSkip {
-		dc.SkipValidator(v)
+		bp.SkipValidator(v)
 	}
 }
 
@@ -256,15 +283,15 @@ func checkOverwriteAllowed(depDir string, bp config.Blueprint, overwriteFlag boo
 	if _, err := os.Stat(expPath); os.IsNotExist(err) {
 		return forceErr(fmt.Errorf("expanded blueprint file %q is missing, this could be a result of changing GHPC version between consecutive deployments", expPath))
 	}
-	prev, _, err := config.NewDeploymentConfig(expPath)
+	prev, _, err := config.NewBlueprint(expPath)
 	if err != nil {
 		return forceErr(err)
 	}
 
-	if prev.Config.GhpcVersion != bp.GhpcVersion {
+	if prev.GhpcVersion != bp.GhpcVersion {
 		return forceErr(fmt.Errorf(
 			"ghpc_version has changed from %q to %q, using different versions of GHPC to update a live deployment is not officially supported",
-			prev.Config.GhpcVersion, bp.GhpcVersion))
+			prev.GhpcVersion, bp.GhpcVersion))
 	}
 
 	if !overwriteFlag {
@@ -276,7 +303,7 @@ func checkOverwriteAllowed(depDir string, bp config.Blueprint, overwriteFlag boo
 		newGroups[g.Name] = true
 	}
 
-	for _, g := range prev.Config.DeploymentGroups {
+	for _, g := range prev.DeploymentGroups {
 		if !newGroups[g.Name] {
 			return forceErr(fmt.Errorf("you are attempting to remove a deployment group %q, which is not supported", g.Name))
 		}

cmd/create_test.go

@@ -25,8 +25,8 @@ import (
 )
 
 func (s *MySuite) TestSetCLIVariables(c *C) {
-	bp := config.Blueprint{}
-	bp.Vars.Set("deployment_name", cty.StringVal("bush"))
+	ds := config.DeploymentSettings{}
+	ds.Vars.Set("deployment_name", cty.StringVal("bush"))
 
 	vars := []string{
 		"project_id=cli_test_project_id",
@@ -42,9 +42,9 @@ func (s *MySuite) TestSetCLIVariables(c *C) {
 		"keyArrayOfMaps=[foo, {bar: baz, qux: 1}]",
 		"keyMapOfArrays={foo: [1, 2, 3], bar: [a, b, c]}",
 	}
-	c.Assert(setCLIVariables(&bp, vars), IsNil)
+	c.Assert(setCLIVariables(&ds, vars), IsNil)
 	c.Check(
-		bp.Vars.Items(), DeepEquals, map[string]cty.Value{
+		ds.Vars.Items(), DeepEquals, map[string]cty.Value{
 			"project_id":      cty.StringVal("cli_test_project_id"),
 			"deployment_name": cty.StringVal("cli_deployment_name"),
 			"region":          cty.StringVal("cli_region"),
@@ -72,14 +72,14 @@ func (s *MySuite) TestSetCLIVariables(c *C) {
 		})
 
 	// Failure: Variable without '='
-	bp = config.Blueprint{}
+	ds = config.DeploymentSettings{}
 	inv := []string{"project_idcli_test_project_id"}
-	c.Check(setCLIVariables(&bp, inv), ErrorMatches, "invalid format: .*")
+	c.Check(setCLIVariables(&ds, inv), ErrorMatches, "invalid format: .*")
 
 	// Failure: Unmarshalable value
-	bp = config.Blueprint{}
+	ds = config.DeploymentSettings{}
 	inv = []string{"pyrite={gold"}
-	c.Check(setCLIVariables(&bp, inv), ErrorMatches, ".*unable to convert.*pyrite.*gold.*")
+	c.Check(setCLIVariables(&ds, inv), ErrorMatches, ".*unable to convert.*pyrite.*gold.*")
 }
 
 func (s *MySuite) TestSetBackendConfig(c *C) {
@@ -90,33 +90,101 @@ func (s *MySuite) TestSetBackendConfig(c *C) {
 		"odor=strong",
 	}
 
-	bp := config.Blueprint{}
-	c.Assert(setBackendConfig(&bp, vars), IsNil)
+	ds := config.DeploymentSettings{}
+	c.Assert(setBackendConfig(&ds, vars), IsNil)
 
-	be := bp.TerraformBackendDefaults
+	be := ds.TerraformBackendDefaults
 	c.Check(be.Type, Equals, "green")
 	c.Check(be.Configuration.Items(), DeepEquals, map[string]cty.Value{
 		"taste": cty.StringVal("sweet"),
 		"odor":  cty.StringVal("strong"),
 	})
 }
 
+func (s *MySuite) TestMergeDeploymentSettings(c *C) {
+	ds1 := config.DeploymentSettings{
+		Vars: config.NewDict(map[string]cty.Value{
+			"project_id":      cty.StringVal("ds_test_project_id"),
+			"deployment_name": cty.StringVal("ds_deployment_name"),
+		}),
+	}
+
+	bp1 := config.Blueprint{
+		Vars: config.NewDict(map[string]cty.Value{
+			"project_id":  cty.StringVal("bp_test_project_id"),
+			"example_var": cty.StringVal("bp_example_value"),
+		}),
+	}
+
+	// test priority-based merging of deployment variables
+	mergeDeploymentSettings(&bp1, ds1)
+	c.Check(bp1.Vars.Items(), DeepEquals, map[string]cty.Value{
+		"project_id":      cty.StringVal("ds_test_project_id"),
+		"deployment_name": cty.StringVal("ds_deployment_name"),
+		"example_var":     cty.StringVal("bp_example_value"),
+	})
+
+	// check merging zero-value backends
+	ds2 := config.DeploymentSettings{
+		TerraformBackendDefaults: config.TerraformBackend{},
+	}
+	bp2 := config.Blueprint{
+		TerraformBackendDefaults: config.TerraformBackend{},
+	}
+	mergeDeploymentSettings(&bp2, ds2)
+	c.Check(bp2.TerraformBackendDefaults, DeepEquals, config.TerraformBackend{})
+
+	// check keeping blueprint defined backend with no backend in deployment file
+	bp3 := config.Blueprint{
+		TerraformBackendDefaults: config.TerraformBackend{
+			Type: "gsc",
+			Configuration: config.NewDict(map[string]cty.Value{
+				"bucket": cty.StringVal("bp_bucket"),
+			}),
+		},
+	}
+	mergeDeploymentSettings(&bp3, ds2)
+	c.Check(bp3.TerraformBackendDefaults, DeepEquals, config.TerraformBackend{
+		Type: "gsc",
+		Configuration: config.NewDict(map[string]cty.Value{
+			"bucket": cty.StringVal("bp_bucket"),
+		}),
+	})
+
+	// check overriding blueprint defined backend with deployment file
+	ds3 := config.DeploymentSettings{
+		TerraformBackendDefaults: config.TerraformBackend{
+			Type: "gsc",
+			Configuration: config.NewDict(map[string]cty.Value{
+				"bucket": cty.StringVal("ds_bucket"),
+			}),
+		},
+	}
+	mergeDeploymentSettings(&bp3, ds3)
+	c.Check(bp3.TerraformBackendDefaults, DeepEquals, config.TerraformBackend{
+		Type: "gsc",
+		Configuration: config.NewDict(map[string]cty.Value{
+			"bucket": cty.StringVal("ds_bucket"),
+		}),
+	})
+}
+
 func (s *MySuite) TestSetBackendConfig_Invalid(c *C) {
 	// Failure: Variable without '='
 	vars := []string{
 		"typegreen",
 	}
-	bp := config.Blueprint{}
-	c.Assert(setBackendConfig(&bp, vars), ErrorMatches, "invalid format: .*")
+	ds := config.DeploymentSettings{}
+	c.Assert(setBackendConfig(&ds, vars), ErrorMatches, "invalid format: .*")
 }
 
 func (s *MySuite) TestSetBackendConfig_NoOp(c *C) {
-	bp := config.Blueprint{
+	ds := config.DeploymentSettings{
 		TerraformBackendDefaults: config.TerraformBackend{
 			Type: "green"}}
 
-	c.Assert(setBackendConfig(&bp, []string{}), IsNil)
-	c.Check(bp.TerraformBackendDefaults, DeepEquals, config.TerraformBackend{
+	c.Assert(setBackendConfig(&ds, []string{}), IsNil)
+	c.Check(ds.TerraformBackendDefaults, DeepEquals, config.TerraformBackend{
 		Type: "green"})
 }
 
@@ -205,12 +273,11 @@ func (s *MySuite) TestIsOverwriteAllowed_Present(c *C) {
 		c.Fatal(err)
 	}
 
-	prev := config.DeploymentConfig{
-		Config: config.Blueprint{
-			GhpcVersion: "TaleOfBygoneYears",
-			DeploymentGroups: []config.DeploymentGroup{
-				{Name: "isildur"}}}}
-	if err := prev.ExportBlueprint(filepath.Join(artDir, "expanded_blueprint.yaml")); err != nil {
+	prev := config.Blueprint{
+		GhpcVersion: "TaleOfBygoneYears",
+		DeploymentGroups: []config.DeploymentGroup{
+			{Name: "isildur"}}}
+	if err := prev.Export(filepath.Join(artDir, "expanded_blueprint.yaml")); err != nil {
 		c.Fatal(err)
 	}
 	noW, yesW, noForce, yesForce := false, true, false, true

cmd/deploy.go

@@ -75,9 +75,9 @@ func getApplyBehavior(autoApprove bool) shell.ApplyBehavior {
 
 func runDeployCmd(cmd *cobra.Command, args []string) {
 	expandedBlueprintFile := filepath.Join(artifactsDir, modulewriter.ExpandedBlueprintName)
-	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
 	checkErr(err)
-	groups := dc.Config.DeploymentGroups
+	groups := bp.DeploymentGroups
 	checkErr(validateRuntimeDependencies(groups))
 	checkErr(shell.ValidateDeploymentDirectory(groups, deploymentRoot))
 

cmd/destroy.go

@@ -64,19 +64,19 @@ func parseDestroyArgs(cmd *cobra.Command, args []string) error {
 
 func runDestroyCmd(cmd *cobra.Command, args []string) error {
 	expandedBlueprintFile := filepath.Join(artifactsDir, modulewriter.ExpandedBlueprintName)
-	dc, _, err := config.NewDeploymentConfig(expandedBlueprintFile)
+	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
 	if err != nil {
 		return err
 	}
 
-	if err := shell.ValidateDeploymentDirectory(dc.Config.DeploymentGroups, deploymentRoot); err != nil {
+	if err := shell.ValidateDeploymentDirectory(bp.DeploymentGroups, deploymentRoot); err != nil {
 		return err
 	}
 
 	// destroy in reverse order of creation!
 	packerManifests := []string{}
-	for i := len(dc.Config.DeploymentGroups) - 1; i >= 0; i-- {
-		group := dc.Config.DeploymentGroups[i]
+	for i := len(bp.DeploymentGroups) - 1; i >= 0; i-- {
+		group := bp.DeploymentGroups[i]
 		groupDir := filepath.Join(deploymentRoot, string(group.Name))
 
 		var err error