Merge pull request #1594 from Azure/dev

10.13 Release
Azure · Oct 20, 2021 · 058bd5b · 058bd5b
2 parents 0a14fbd + 1d1988e
commit 058bd5b
Show file tree

Hide file tree

Showing 33 changed files with 597 additions and 187 deletions.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,6 +1,26 @@
 
 # Change Log
 
+## Version 10.13.0
+
+### New features
+1. Added Arc VM support for authorization via managed identity.
+2. Widen managed disk scenario to all md- accounts instead of just md-impexp- accounts.
+3. The concurrency is now set to AUTO for Azure Files by default to avoid throttling.
+4. Decrease the number of create directory calls for Azure Files to avoid throttling.
+5. Added the from-to flag for sync.
+
+## Bug fixes
+1. Fixed the memory usage issue with generating the list of skipped/failed transfers in JSON output.
+2. Fixed ADLS Gen2 ACL copying where intermediate folders were missed. 
+3. Fixed the S3 to Blob scenario using the login command.
+4. Fixed dry-run for dfs endpoints.
+5. Fixed incorrect percentage-done shown while resuming job.
+6. Fixed login issues on the ARM platforms.
+7. Fixed incorrect progress status for the sync command. 
+8. Fixed concurrency map access problem for folder creation tracker.
+9. Fixed resuming with a public source.
+
 ## Version 10.12.2
 
 ## Bug fixes

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -30,8 +30,12 @@ jobs:
       - script: |
           GOARCH=amd64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_amd64"
           GOARCH=amd64 GOOS=linux go build -tags "se_integration" -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_se_amd64"
+          GOARCH=arm64 GOOS=linux go build -o "$(Build.ArtifactStagingDirectory)/azcopy_linux_arm64"
+
           GOARCH=amd64 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_amd64.exe"
           GOARCH=386 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_386.exe"
+          GOARCH=arm GOARM=7 GOOS=windows go build -o "$(Build.ArtifactStagingDirectory)/azcopy_windows_v7_arm.exe"
+
           cp NOTICE.txt $(Build.ArtifactStagingDirectory)
         displayName: 'Generate Linux And Windows Build'
         condition: eq(variables.type, 'linux')
@@ -67,7 +71,7 @@ jobs:
         inputs:
           version: '1.16'
 
-      # Running E2E Tests on Linux
+      # Running E2E Tests on Linux - AMD64
       - script: |
           set -e
           GOARCH=amd64 GOOS=linux go build -o azcopy_linux_amd64
@@ -80,10 +84,10 @@ jobs:
           AZCOPY_E2E_ACCOUNT_NAME_HNS: $(AZCOPY_E2E_ACCOUNT_NAME_HNS)
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
           CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
-        displayName: 'E2E Test Linux'
+        displayName: 'E2E Test Linux - AMD64'
         condition: eq(variables.type, 'linux')
 
-      # Running E2E Tests on Windows
+      # Running E2E Tests on Windows - AMD64
       - script: |
           go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
           go build -o $(System.DefaultWorkingDirectory)/azcopy_windows_386.exe
@@ -97,7 +101,7 @@ jobs:
           CPK_ENCRYPTION_KEY: $(CPK_ENCRYPTION_KEY)
           CPK_ENCRYPTION_KEY_SHA256: $(CPK_ENCRYPTION_KEY_SHA256)
           AZCOPY_E2E_EXECUTABLE_PATH: $(System.DefaultWorkingDirectory)/azcopy_windows_amd64.exe
-        displayName: 'E2E Test Windows'
+        displayName: 'E2E Test Windows - AMD64'
         condition: eq(variables.type, 'windows')
 
       # Running E2E Tests on Mac

diff --git a/cmd/copy.go b/cmd/copy.go
@@ -633,7 +633,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
 
 	isUserPersistingPermissions := raw.preservePermissions || raw.preserveSMBPermissions
 	if cooked.preserveSMBInfo && !isUserPersistingPermissions {
-		glcm.Info("Please note: the preserve-permissions flag is set to false, thus AzCopy will not copy SMB ACLs between the source and destination.")
+		glcm.Info("Please note: the preserve-permissions flag is set to false, thus AzCopy will not copy SMB ACLs between the source and destination. To learn more: https://aka.ms/AzCopyandAzureFiles.")
 	}
 
 	if err = validatePreserveSMBPropertyOption(isUserPersistingPermissions, cooked.FromTo, &cooked.ForceWrite, PreservePermissionsFlag); err != nil {
@@ -1287,6 +1287,10 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
 // dispatches the job order (in parts) to the storage engine
 func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 	ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
+	// Make AUTO default for Azure Files since Azure Files throttles too easily.
+	if ste.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) {
+		ste.JobsAdmin.SetConcurrencySettingsToAuto()
+	}
 
 	// Note: credential info here is only used by remove at the moment.
 	// TODO: Get the entirety of remove into the new copyEnumeratorInit script so we can remove this

diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go
@@ -562,8 +562,6 @@ func GetCredentialInfoForLocation(ctx context.Context, location common.Location,
 func getCredentialType(ctx context.Context, raw rawFromToInfo, cpkOptions common.CpkOptions) (credType common.CredentialType, err error) {
 
 	switch {
-	case raw.fromTo == common.EFromTo.S3Blob(): // S3 Bucket
-		credType, _, err = getCredentialTypeForLocation(ctx, raw.fromTo.From(), raw.source, raw.sourceSAS, true, common.CpkOptions{})
 	case raw.fromTo.To().IsRemote():
 		// we authenticate to the destination. Source is assumed to be SAS, or public, or a local resource
 		credType, _, err = getCredentialTypeForLocation(ctx, raw.fromTo.To(), raw.destination, raw.destinationSAS, false, common.CpkOptions{})

diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go
@@ -140,28 +140,30 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) {
 	urlParts := azbfs.NewBfsURLParts(*sourceURL)
 
 	if cca.ListOfFilesChannel == nil {
-		successMsg, err := removeSingleBfsResource(urlParts, p, ctx, cca.Recursive)
+		successMsg, err := removeSingleBfsResource(ctx, urlParts, p, cca.Recursive, cca.dryrunMode)
 		if err != nil {
 			return err
 		}
-
-		glcm.Exit(func(format common.OutputFormat) string {
-			if format == common.EOutputFormat.Json() {
-				summary := common.ListJobSummaryResponse{
-					JobStatus:      common.EJobStatus.Completed(),
-					TotalTransfers: 1,
-					// It's not meaningful to set FileTransfers or FolderPropertyTransfers because even if its a folder, its not really folder _properties_ which is what the name is
-					TransfersCompleted: 1,
-					PercentComplete:    100,
+		if !cca.dryrunMode {
+			glcm.Exit(func(format common.OutputFormat) string {
+				if format == common.EOutputFormat.Json() {
+					summary := common.ListJobSummaryResponse{
+						JobStatus:      common.EJobStatus.Completed(),
+						TotalTransfers: 1,
+						// It's not meaningful to set FileTransfers or FolderPropertyTransfers because even if its a folder, its not really folder _properties_ which is what the name is
+						TransfersCompleted: 1,
+						PercentComplete:    100,
+					}
+					jsonOutput, err := json.Marshal(summary)
+					common.PanicIfErr(err)
+					return string(jsonOutput)
 				}
-				jsonOutput, err := json.Marshal(summary)
-				common.PanicIfErr(err)
-				return string(jsonOutput)
-			}
-
-			return successMsg
-		}, common.EExitCode.Success())
+				return successMsg
+			}, common.EExitCode.Success())
 
+		} else {
+			glcm.Exit(nil, common.EExitCode.Success())
+		}
 		// explicitly exit, since in our tests Exit might be mocked away
 		return nil
 	}
@@ -176,7 +178,7 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) {
 	for ; ok; childPath, ok = <-cca.ListOfFilesChannel {
 		// remove the child path
 		urlParts.DirectoryOrFilePath = common.GenerateFullPath(parentPath, childPath)
-		successMessage, err := removeSingleBfsResource(urlParts, p, ctx, cca.Recursive)
+		successMessage, err := removeSingleBfsResource(ctx, urlParts, p, cca.Recursive, cca.dryrunMode)
 		if err != nil {
 			// the specific error is not included in the details, since it doesn't have a field for full error message
 			failedTransfers = append(failedTransfers, common.TransferDetail{Src: childPath, TransferStatus: common.ETransferStatus.Failed()})
@@ -187,45 +189,60 @@ func removeBfsResources(cca *CookedCopyCmdArgs) (err error) {
 		}
 	}
 
-	glcm.Exit(func(format common.OutputFormat) string {
-		if format == common.EOutputFormat.Json() {
-			status := common.EJobStatus.Completed()
-			if len(failedTransfers) > 0 {
-				status = common.EJobStatus.CompletedWithErrors()
-
-				// if nothing got deleted
-				if successCount == 0 {
-					status = common.EJobStatus.Failed()
+	if cca.dryrunMode {
+		glcm.Exit(nil, common.EExitCode.Success())
+	} else {
+		glcm.Exit(func(format common.OutputFormat) string {
+			if format == common.EOutputFormat.Json() {
+				status := common.EJobStatus.Completed()
+				if len(failedTransfers) > 0 {
+					status = common.EJobStatus.CompletedWithErrors()
+
+					// if nothing got deleted
+					if successCount == 0 {
+						status = common.EJobStatus.Failed()
+					}
 				}
-			}
 
-			summary := common.ListJobSummaryResponse{
-				JobStatus:          status,
-				TotalTransfers:     successCount + uint32(len(failedTransfers)),
-				TransfersCompleted: successCount,
-				TransfersFailed:    uint32(len(failedTransfers)),
-				PercentComplete:    100,
-				FailedTransfers:    failedTransfers,
+				summary := common.ListJobSummaryResponse{
+					JobStatus:          status,
+					TotalTransfers:     successCount + uint32(len(failedTransfers)),
+					TransfersCompleted: successCount,
+					TransfersFailed:    uint32(len(failedTransfers)),
+					PercentComplete:    100,
+					FailedTransfers:    failedTransfers,
+				}
+				jsonOutput, err := json.Marshal(summary)
+				common.PanicIfErr(err)
+				return string(jsonOutput)
 			}
-			jsonOutput, err := json.Marshal(summary)
-			common.PanicIfErr(err)
-			return string(jsonOutput)
-		}
 
-		return fmt.Sprintf("Successfully removed %v entities.", successCount)
-	}, common.EExitCode.Success())
+			return fmt.Sprintf("Successfully removed %v entities.", successCount)
+		}, common.EExitCode.Success())
+	}
 
 	return nil
 }
 
 // TODO move after ADLS/Blob interop goes public
 // TODO this simple remove command is only here to support the scenario temporarily
-func removeSingleBfsResource(urlParts azbfs.BfsURLParts, p pipeline.Pipeline, ctx context.Context, recursive bool) (successMessage string, err error) {
+func removeSingleBfsResource(ctx context.Context, urlParts azbfs.BfsURLParts, p pipeline.Pipeline, recursive bool, dryrunMode bool) (successMessage string, err error) {
 	// deleting a filesystem
 	if urlParts.DirectoryOrFilePath == "" {
 		fsURL := azbfs.NewFileSystemURL(urlParts.URL(), p)
-		_, err := fsURL.Delete(ctx)
-		return "Successfully removed the filesystem " + urlParts.FileSystemName, err
+		if !dryrunMode {
+			_, err = fsURL.Delete(ctx)
+			if err == nil {
+				return "Successfully removed the filesystem " + urlParts.FileSystemName, nil
+			}
+			return "", err
+		} else {
+			glcm.Dryrun(func(_ common.OutputFormat) string {
+				return fmt.Sprintf("DRYRUN: remove filesystem %s", urlParts.FileSystemName)
+			})
+			return "", nil
+		}
+
 	}
 
 	// we do not know if the source is a file or a directory
@@ -240,35 +257,69 @@ func removeSingleBfsResource(urlParts azbfs.BfsURLParts, p pipeline.Pipeline, ct
 	// then we should short-circuit and simply remove that file
 	if strings.EqualFold(props.XMsResourceType(), "file") {
 		fileURL := directoryURL.NewFileUrl()
-		_, err := fileURL.Delete(ctx)
 
-		if err == nil {
-			return "Successfully removed file: " + urlParts.DirectoryOrFilePath, nil
-		}
+		if !dryrunMode {
+			_, err := fileURL.Delete(ctx)
+			if err == nil {
+				return "Successfully removed file: " + urlParts.DirectoryOrFilePath, nil
+			}
 
-		return "", err
+			return "", err
+		} else {
+			glcm.Dryrun(func(_ common.OutputFormat) string {
+				return fmt.Sprintf("DRYRUN: remove file %s", urlParts.DirectoryOrFilePath)
+			})
+			return "", nil
+		}
 	}
 
 	// otherwise, remove the directory and follow the continuation token if necessary
 	// initialize an empty continuation marker
 	marker := ""
 
-	// remove the directory
-	// loop will continue until the marker received in the response is empty
-	for {
-		removeResp, err := directoryURL.Delete(ctx, &marker, recursive)
-		if err != nil {
-			return "", fmt.Errorf("cannot remove the given resource due to error: %s", err)
+	if !dryrunMode {
+		// remove the directory
+		// loop will continue until the marker received in the response is empty
+		for {
+			removeResp, err := directoryURL.Delete(ctx, &marker, recursive)
+			if err != nil {
+				return "", fmt.Errorf("cannot remove the given resource due to error: %s", err)
+			}
+
+			// update the continuation token for the next call
+			marker = removeResp.XMsContinuation()
+
+			// determine whether listing should be done
+			if marker == "" {
+				break
+			}
 		}
 
-		// update the continuation token for the next call
-		marker = removeResp.XMsContinuation()
+		return "Successfully removed directory: " + urlParts.DirectoryOrFilePath, nil
+	} else {
+		for {
+			listResp, err := directoryURL.ListDirectorySegment(ctx, &marker, recursive)
 
-		// determine whether listing should be done
-		if marker == "" {
-			break
+			if err != nil {
+				return "Could not list files", err
+			}
+
+			for _, v := range listResp.Paths {
+				entityType := "directory"
+				if v.IsDirectory == nil || *v.IsDirectory == false {
+					entityType = "file"
+				}
+
+				glcm.Dryrun(func(_ common.OutputFormat) string {
+					return fmt.Sprintf("DRYRUN: remove %s %s", entityType, *v.Name)
+				})
+			}
+
+			marker = listResp.XMsContinuation()
+			if marker == "" { // do-while pattern
+				break
+			}
 		}
+		return "", nil
 	}
-
-	return "Successfully removed directory: " + urlParts.DirectoryOrFilePath, nil
 }