Merge pull request #2483 from GoogleCloudPlatform/release-candidate

Release v1.32.0

Makefile

@@ -159,12 +159,11 @@ validate_golden_copy: ghpc
 terraform-format:
 	$(info *********** cleaning terraform files syntax and generating terraform documentation ***********)
 	@for folder in ${TERRAFORM_FOLDERS}; do \
-	  echo "cleaning syntax for $${folder}";\
+	  echo "checking syntax for $${folder}";\
 		terraform fmt -list=true $${folder};\
 	done
 	@for folder in ${TERRAFORM_FOLDERS}; do \
 		terraform-docs markdown $${folder} --config .tfdocs-markdown.yaml;\
-		terraform-docs json $${folder} --config .tfdocs-json.yaml;\
 	done
 
 endif

README.md

@@ -169,6 +169,10 @@ In the right side, expand the Filters view and then filter by label, specifying
 
 ## Troubleshooting
 
+### Authentication
+
+Confirm that you have [properly setup Google Cloud credentials](#gcp-credentials)
+
 ### Slurm Clusters
 
 Please see the dedicated [troubleshooting guide for Slurm](docs/slurm-troubleshooting.md).

cmd/create.go

@@ -79,11 +79,11 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 }
 
 func doCreate(path string) string {
-	bp := expandOrDie(path)
+	bp, ctx := expandOrDie(path)
 	deplDir := filepath.Join(createFlags.outputDir, bp.DeploymentName())
 	logging.Info("Creating deployment folder %q ...", deplDir)
-	checkErr(checkOverwriteAllowed(deplDir, bp, createFlags.overwriteDeployment, createFlags.forceOverwrite))
-	checkErr(modulewriter.WriteDeployment(bp, deplDir))
+	checkErr(checkOverwriteAllowed(deplDir, bp, createFlags.overwriteDeployment, createFlags.forceOverwrite), ctx)
+	checkErr(modulewriter.WriteDeployment(bp, deplDir), ctx)
 	return deplDir
 }
 
@@ -95,19 +95,15 @@ func printAdvancedInstructionsMessage(deplDir string) {
 }
 
 // TODO: move to expand.go
-func expandOrDie(path string) config.Blueprint {
+func expandOrDie(path string) (config.Blueprint, *config.YamlCtx) {
 	bp, ctx, err := config.NewBlueprint(path)
-	if err != nil {
-		logging.Fatal(renderError(err, ctx))
-	}
+	checkErr(err, ctx)
 
 	var ds config.DeploymentSettings
 	var dCtx config.YamlCtx
 	if expandFlags.deploymentFile != "" {
 		ds, dCtx, err = config.NewDeploymentSettings(expandFlags.deploymentFile)
-		if err != nil {
-			logging.Fatal(renderError(err, dCtx))
-		}
+		checkErr(err, &dCtx)
 	}
 	if err := setCLIVariables(&ds, expandFlags.cliVariables); err != nil {
 		logging.Fatal("Failed to set the variables at CLI: %v", err)
@@ -118,7 +114,7 @@ func expandOrDie(path string) config.Blueprint {
 
 	mergeDeploymentSettings(&bp, ds)
 
-	checkErr(setValidationLevel(&bp, expandFlags.validationLevel))
+	checkErr(setValidationLevel(&bp, expandFlags.validationLevel), ctx)
 	skipValidators(&bp)
 
 	if bp.GhpcVersion != "" {
@@ -127,12 +123,9 @@ func expandOrDie(path string) config.Blueprint {
 	bp.GhpcVersion = GitCommitInfo
 
 	// Expand the blueprint
-	if err := bp.Expand(); err != nil {
-		logging.Fatal(renderError(err, ctx))
-	}
-
-	validateMaybeDie(bp, ctx)
-	return bp
+	checkErr(bp.Expand(), ctx)
+	validateMaybeDie(bp, *ctx)
+	return bp, ctx
 }
 
 // TODO: move to expand.go
@@ -296,3 +289,13 @@ func checkOverwriteAllowed(depDir string, bp config.Blueprint, overwriteFlag boo
 
 	return nil
 }
+
+// Reads an expanded blueprint from the artifacts directory
+// IMPORTANT: returned blueprint is "materialized", see config.Blueprint.Materialize
+func artifactBlueprintOrDie(artDir string) (config.Blueprint, *config.YamlCtx) {
+	path := filepath.Join(artDir, modulewriter.ExpandedBlueprintName)
+	bp, ctx, err := config.NewBlueprint(path)
+	checkErr(err, ctx)
+	checkErr(bp.Materialize(), ctx)
+	return bp, ctx
+}

cmd/deploy.go

@@ -43,7 +43,7 @@ var (
 		Short:             "deploy all resources in a Toolkit deployment directory.",
 		Long:              "deploy all resources in a Toolkit deployment directory.",
 		Args:              cobra.MatchAll(cobra.ExactArgs(1), checkExists),
-		ValidArgsFunction: matchDirs,
+		ValidArgsFunction: filterYaml,
 		Run:               runDeployCmd,
 		SilenceUsage:      true,
 	})
@@ -59,7 +59,7 @@ func runDeployCmd(cmd *cobra.Command, args []string) {
 		// check that no "create" flags were specified
 		cmd.Flags().VisitAll(func(f *pflag.Flag) {
 			if f.Changed && createCmd.Flag(f.Name) != nil {
-				checkErr(fmt.Errorf("cannot specify flag %q with DEPLOYMENT_DIRECTORY provided", f.Name))
+				checkErr(fmt.Errorf("cannot specify flag %q with DEPLOYMENT_DIRECTORY provided", f.Name), nil)
 			}
 		})
 	}
@@ -68,38 +68,38 @@ func runDeployCmd(cmd *cobra.Command, args []string) {
 
 func doDeploy(deplRoot string) {
 	artDir := getArtifactsDir(deplRoot)
-	checkErr(shell.CheckWritableDir(artDir))
-
-	expandedBlueprintFile := filepath.Join(artDir, modulewriter.ExpandedBlueprintName)
-	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
-	checkErr(err)
+	checkErr(shell.CheckWritableDir(artDir), nil)
+	bp, ctx := artifactBlueprintOrDie(artDir)
 	groups := bp.Groups
-	checkErr(validateRuntimeDependencies(deplRoot, groups))
-	checkErr(shell.ValidateDeploymentDirectory(groups, deplRoot))
+	checkErr(validateRuntimeDependencies(deplRoot, groups), ctx)
+	checkErr(shell.ValidateDeploymentDirectory(groups, deplRoot), ctx)
 
-	for _, group := range groups {
+	for ig, group := range groups {
 		groupDir := filepath.Join(deplRoot, string(group.Name))
-		checkErr(shell.ImportInputs(groupDir, artDir, bp))
+		checkErr(shell.ImportInputs(groupDir, artDir, bp), ctx)
 
 		switch group.Kind() {
 		case config.PackerKind:
 			// Packer groups are enforced to have length 1
 			subPath, e := modulewriter.DeploymentSource(group.Modules[0])
-			checkErr(e)
+			checkErr(e, ctx)
 			moduleDir := filepath.Join(groupDir, subPath)
-			checkErr(deployPackerGroup(moduleDir, getApplyBehavior()))
+			checkErr(deployPackerGroup(moduleDir, getApplyBehavior()), ctx)
 		case config.TerraformKind:
-			checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior()))
+			checkErr(deployTerraformGroup(groupDir, artDir, getApplyBehavior()), ctx)
 		default:
-			checkErr(fmt.Errorf("group %s is an unsupported kind %s", groupDir, group.Kind().String()))
+			checkErr(
+				config.BpError{
+					Err:  fmt.Errorf("group %q is an unsupported kind %q", groupDir, group.Kind()),
+					Path: config.Root.Groups.At(ig).Name}, ctx)
 		}
 	}
 	logging.Info("\n###############################")
 	printAdvancedInstructionsMessage(deplRoot)
 }
 
 func validateRuntimeDependencies(deplDir string, groups []config.Group) error {
-	for _, group := range groups {
+	for ig, group := range groups {
 		var err error
 		switch group.Kind() {
 		case config.PackerKind:
@@ -108,7 +108,9 @@ func validateRuntimeDependencies(deplDir string, groups []config.Group) error {
 			groupDir := filepath.Join(deplDir, string(group.Name))
 			_, err = shell.ConfigureTerraform(groupDir)
 		default:
-			err = fmt.Errorf("group %s is an unsupported kind %q", group.Name, group.Kind().String())
+			err = config.BpError{
+				Path: config.Root.Groups.At(ig).Name,
+				Err:  fmt.Errorf("group %s is an unsupported kind %q", group.Name, group.Kind().String())}
 		}
 		if err != nil {
 			return err

cmd/destroy.go

@@ -49,14 +49,12 @@ func runDestroyCmd(cmd *cobra.Command, args []string) {
 	artifactsDir := getArtifactsDir(deplRoot)
 
 	if isDir, _ := shell.DirInfo(artifactsDir); !isDir {
-		checkErr(fmt.Errorf("artifacts path %s is not a directory", artifactsDir))
+		checkErr(fmt.Errorf("artifacts path %s is not a directory", artifactsDir), nil)
 	}
 
-	expandedBlueprintFile := filepath.Join(artifactsDir, modulewriter.ExpandedBlueprintName)
-	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
-	checkErr(err)
+	bp, ctx := artifactBlueprintOrDie(artifactsDir)
 
-	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot))
+	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot), ctx)
 
 	// destroy in reverse order of creation!
 	packerManifests := []string{}
@@ -76,7 +74,7 @@ func runDestroyCmd(cmd *cobra.Command, args []string) {
 		default:
 			err = fmt.Errorf("group %s is an unsupported kind %s", groupDir, group.Kind().String())
 		}
-		checkErr(err)
+		checkErr(err, ctx)
 	}
 
 	modulewriter.WritePackerDestroyInstructions(os.Stdout, packerManifests)

cmd/expand.go

@@ -28,14 +28,13 @@ func addExpandFlags(c *cobra.Command, addOutFlag bool) *cobra.Command {
 	}
 
 	c.Flags().StringVarP(&expandFlags.deploymentFile, "deployment-file", "d", "",
-		"Toolkit Deployment File.")
-	c.Flags().MarkHidden("deployment-file")
+		"Deployment file to override blueprint variables and backend configuration")
 
 	c.Flags().StringSliceVar(&expandFlags.cliVariables, "vars", nil,
 		"Comma-separated list of name=value variables to override YAML configuration. Can be used multiple times.")
 	c.Flags().StringSliceVar(&expandFlags.cliBEConfigVars, "backend-config", nil,
 		"Comma-separated list of name=value variables to set Terraform backend configuration. Can be used multiple times.")
-	c.Flags().StringVarP(&expandFlags.validationLevel, "validation-level", "l", "WARNING",
+	c.Flags().StringVarP(&expandFlags.validationLevel, "validation-level", "l", "ERROR",
 		"Set validation level to one of (\"ERROR\", \"WARNING\", \"IGNORE\")")
 	c.Flags().StringSliceVar(&expandFlags.validatorsToSkip, "skip-validators", nil, "Validators to skip")
 	return c
@@ -66,7 +65,7 @@ var (
 )
 
 func runExpandCmd(cmd *cobra.Command, args []string) {
-	bp := expandOrDie(args[0])
-	checkErr(bp.Export(expandFlags.outputPath))
+	bp, ctx := expandOrDie(args[0])
+	checkErr(bp.Export(expandFlags.outputPath), ctx)
 	logging.Info(boldGreen("Expanded Environment Definition created successfully, saved as %s."), expandFlags.outputPath)
 }

cmd/export.go

@@ -18,7 +18,6 @@ package cmd
 import (
 	"errors"
 	"hpc-toolkit/pkg/config"
-	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/shell"
 	"path/filepath"
 
@@ -43,7 +42,7 @@ var (
 
 func parseExportImportArgs(args []string) (string, string) {
 	gd, err := filepath.Abs(args[0])
-	checkErr(err)
+	checkErr(err, nil)
 	return filepath.Join(gd, ".."), gd
 }
 
@@ -53,26 +52,24 @@ func runExportCmd(cmd *cobra.Command, args []string) {
 	artifactsDir := getArtifactsDir(deplRoot)
 	groupName := config.GroupName(filepath.Base(groupDir))
 
-	checkErr(shell.CheckWritableDir(artifactsDir))
+	checkErr(shell.CheckWritableDir(artifactsDir), nil)
 
-	expandedBlueprintFile := filepath.Join(artifactsDir, modulewriter.ExpandedBlueprintName)
-	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
-	checkErr(err)
+	bp, ctx := artifactBlueprintOrDie(artifactsDir)
 
-	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot))
+	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot), ctx)
 
 	group, err := bp.Group(groupName)
-	checkErr(err)
+	checkErr(err, ctx)
 
 	if group.Kind() == config.PackerKind {
-		checkErr(errors.New("export command is unsupported on Packer modules because they do not have outputs"))
+		checkErr(errors.New("export command is unsupported on Packer modules because they do not have outputs"), ctx)
 	}
 	if group.Kind() != config.TerraformKind {
-		checkErr(errors.New("export command is supported for Terraform modules only"))
+		checkErr(errors.New("export command is supported for Terraform modules only"), ctx)
 	}
 
 	tf, err := shell.ConfigureTerraform(groupDir)
-	checkErr(err)
+	checkErr(err, ctx)
 
-	checkErr(shell.ExportOutputs(tf, artifactsDir, shell.NeverApply))
+	checkErr(shell.ExportOutputs(tf, artifactsDir, shell.NeverApply), ctx)
 }

cmd/import.go

@@ -16,10 +16,7 @@
 package cmd
 
 import (
-	"hpc-toolkit/pkg/config"
-	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/shell"
-	"path/filepath"
 
 	"github.com/spf13/cobra"
 )
@@ -43,13 +40,10 @@ var (
 func runImportCmd(cmd *cobra.Command, args []string) {
 	deplRoot, groupDir := parseExportImportArgs(args)
 	artifactsDir := getArtifactsDir(deplRoot)
+	checkErr(shell.CheckWritableDir(groupDir), nil)
 
-	checkErr(shell.CheckWritableDir(groupDir))
+	bp, ctx := artifactBlueprintOrDie(artifactsDir)
 
-	expandedBlueprintFile := filepath.Join(artifactsDir, modulewriter.ExpandedBlueprintName)
-	bp, _, err := config.NewBlueprint(expandedBlueprintFile)
-	checkErr(err)
-
-	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot))
-	checkErr(shell.ImportInputs(groupDir, artifactsDir, bp))
+	checkErr(shell.ValidateDeploymentDirectory(bp.Groups, deplRoot), ctx)
+	checkErr(shell.ImportInputs(groupDir, artifactsDir, bp), ctx)
 }

cmd/root.go

@@ -52,7 +52,7 @@ HPC deployments on the Google Cloud Platform.`,
 				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.31.1",
+		Version:     "vv1.32.0",
 		Annotations: annotation,
 	}
 )
@@ -248,8 +248,11 @@ func execPath() string {
 
 // checkErr is similar to cobra.CheckErr, but with renderError and logging.Fatal
 // NOTE: this function uses empty YamlCtx, so if you have one, use renderError directly.
-func checkErr(err error) {
+func checkErr(err error, ctx *config.YamlCtx) {
+	if ctx == nil {
+		ctx = &config.YamlCtx{}
+	}
 	if err != nil {
-		logging.Fatal(renderError(err, config.YamlCtx{}))
+		logging.Fatal(renderError(err, *ctx))
 	}
 }

community/examples/hpc-slurm-gromacs.yaml

@@ -78,12 +78,12 @@ deployment_groups:
         spack config --scope defaults add config:build_stage:/sw/spack/spack-stage
         spack config --scope defaults add -f /tmp/projections-config.yaml
 
-        spack install gcc@10.3.0 target=x86_64
-        spack load gcc@10.3.0 target=x86_64
+        spack install gcc@13.1.0 target=x86_64
+        spack load gcc@13.1.0 target=x86_64
         spack compiler find --scope site
 
-        spack install intel-mpi@2018.4.274%gcc@10.3.0
-        spack install gromacs@2023.1 %gcc@10.3.0 ^intel-mpi@2018.4.274 ^[email protected] %[email protected].0
+        spack install intel-oneapi-mpi@2021.9.0%gcc@13.1.0
+        spack install gromacs@2020.6 %gcc@13.1.0 ^intel-oneapi-mpi@2021.9.0
 
   - id: script
     source: modules/scripts/startup-script
@@ -110,7 +110,6 @@ deployment_groups:
     use: [network1]
     settings:
       name_prefix: login
-      machine_type: n2-standard-4
       disable_login_public_ips: false
 
   - id: slurm_controller