Merge pull request #1111 from Azure/dev

Release 10.5.1

ChangeLog.md

@@ -1,7 +1,19 @@
 
 # Change Log
 
-## Version 10.5
+## Version 10.5.1
+
+### New features
+
+- Allow more accurate values for job status in `jobs` commands, e.g. completed with failed or skipped transfers.
+
+### Bug fixes
+
+- Fixed issue with removing blobs with hdi_isfolder=true metadata when the list-of-files flag is used.
+- Manually unfurl symbolic links to fix long file path issue on UNC locations.
+
+
+## Version 10.5.0
 
 ### New features
 

cmd/jobsClean.go

@@ -78,7 +78,8 @@ func init() {
 
 	// NOTE: we have way more job status than we normally need, only show the most common ones
 	jobsCleanCmd.PersistentFlags().StringVar(&commandLineInput.withStatus, "with-status", "All",
-		"only remove the jobs with this status, available values: Cancelled, Completed, Failed, InProgress, All")
+		"only remove the jobs with this status, available values: All, Cancelled, Failed, Completed"+
+			" CompletedWithErrors, CompletedWithSkipped, CompletedWithErrorsAndSkipped")
 }
 
 func handleCleanJobsCommand(givenStatus common.JobStatus) error {
@@ -90,7 +91,7 @@ func handleCleanJobsCommand(givenStatus common.JobStatus) error {
 
 	// we must query the jobs and find out which one to remove
 	resp := common.ListJobsResponse{}
-	Rpc(common.ERpcCmd.ListJobs(), nil, &resp)
+	Rpc(common.ERpcCmd.ListJobs(), common.EJobStatus.All(), &resp)
 
 	if resp.ErrorMessage != "" {
 		return errors.New("failed to query the list of jobs")

cmd/jobsList.go

@@ -36,6 +36,12 @@ type ListResponse struct {
 }
 
 func init() {
+	type JobsListReq struct {
+		withStatus string
+	}
+
+	commandLineInput := JobsListReq{}
+
 	// lsCmd represents the listJob command
 	lsCmd := &cobra.Command{
 		Use:     "list",
@@ -52,7 +58,13 @@ func init() {
 			return nil
 		},
 		Run: func(cmd *cobra.Command, args []string) {
-			err := HandleListJobsCommand()
+			withStatus := common.EJobStatus
+			err := withStatus.Parse(commandLineInput.withStatus)
+			if err != nil {
+				glcm.Error(fmt.Sprintf("Failed to parse --with-status due to error: %s.", err))
+			}
+
+			err = HandleListJobsCommand(withStatus)
 			if err == nil {
 				glcm.Exit(nil, common.EExitCode.Success())
 			} else {
@@ -62,13 +74,17 @@ func init() {
 	}
 
 	jobsCmd.AddCommand(lsCmd)
+
+	jobsCmd.PersistentFlags().StringVar(&commandLineInput.withStatus, "with-status", "All",
+		"List the jobs with given status, available values: All, Cancelled, Failed, InProgress, Completed,"+
+			" CompletedWithErrors, CompletedWithFailures, CompletedWithErrorsAndSkipped")
 }
 
 // HandleListJobsCommand sends the ListJobs request to transfer engine
 // Print the Jobs in the history of Azcopy
-func HandleListJobsCommand() error {
+func HandleListJobsCommand(jobStatus common.JobStatus) error {
 	resp := common.ListJobsResponse{}
-	Rpc(common.ERpcCmd.ListJobs(), nil, &resp)
+	Rpc(common.ERpcCmd.ListJobs(), jobStatus, &resp)
 	return PrintExistingJobIds(resp)
 }
 

cmd/rpc.go

@@ -47,7 +47,7 @@ func inprocSend(rpcCmd common.RpcCmd, requestData interface{}, responseData inte
 		*(responseData.(*common.LifecycleMgr)) = ste.GetJobLCMWrapper(*requestData.(*common.JobID))
 
 	case common.ERpcCmd.ListJobs():
-		*(responseData.(*common.ListJobsResponse)) = ste.ListJobs()
+		*(responseData.(*common.ListJobsResponse)) = ste.ListJobs(requestData.(common.JobStatus))
 
 	case common.ERpcCmd.ListJobSummary():
 		*(responseData.(*common.ListJobSummaryResponse)) = ste.GetJobSummary(*requestData.(*common.JobID))

cmd/syncIndexer.go

@@ -51,6 +51,7 @@ func (i *objectIndexer) store(storedObject storedObject) (err error) {
 func (i *objectIndexer) traverse(processor objectProcessor, filters []objectFilter) (err error) {
 	for _, value := range i.indexMap {
 		err = processIfPassedFilters(filters, value, processor)
+		_, err = getProcessingError(err)
 		if err != nil {
 			return
 		}

cmd/zc_enumerator.go

@@ -329,7 +329,7 @@ func initResourceTraverser(resource common.ResourceString, location common.Locat
 			}
 		}
 
-		output = newListTraverser(resource, location, credential, ctx, recursive, toFollow, getProperties, listOfFilesChannel, incrementEnumerationCounter)
+		output = newListTraverser(resource, location, credential, ctx, recursive, toFollow, getProperties, listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter)
 		return output, nil
 	}
 
@@ -356,7 +356,7 @@ func initResourceTraverser(resource common.ResourceString, location common.Locat
 			}()
 
 			baseResource := resource.CloneWithValue(cleanLocalPath(basePath))
-			output = newListTraverser(baseResource, location, nil, nil, recursive, toFollow, getProperties, globChan, incrementEnumerationCounter)
+			output = newListTraverser(baseResource, location, nil, nil, recursive, toFollow, getProperties, globChan, includeDirectoryStubs, incrementEnumerationCounter)
 		} else {
 			output = newLocalTraverser(resource.ValueLocal(), recursive, toFollow, incrementEnumerationCounter)
 		}
@@ -659,9 +659,24 @@ func passedFilters(filters []objectFilter, storedObject storedObject) bool {
 	return true
 }
 
+// This error should be treated as a flag, that we didn't fail processing, but instead, we just didn't process it.
+// Currently, this is only really used for symlink processing, but it _is_ an error, so it must be handled in all new traversers.
+// Basically, anywhere processIfPassedFilters is called, additionally call getProcessingError.
+var ignoredError = errors.New("FileIgnored")
+
+func getProcessingError(errin error) (ignored bool, err error) {
+	if errin == ignoredError {
+		return true, nil
+	}
+
+	return false, err
+}
+
 func processIfPassedFilters(filters []objectFilter, storedObject storedObject, processor objectProcessor) (err error) {
 	if passedFilters(filters, storedObject) {
 		err = processor(storedObject)
+	} else {
+		err = ignoredError
 	}
 
 	return

cmd/zc_traverser_benchmark.go

@@ -91,6 +91,7 @@ func (t *benchmarkTraverser) traverse(preprocessor objectMorpher, processor obje
 			noBlobProps,
 			noMetdata,
 			""), processor)
+		_, err = getProcessingError(err)
 		if err != nil {
 			return err
 		}

cmd/zc_traverser_blob.go

@@ -136,6 +136,7 @@ func (t *blobTraverser) traverse(preprocessor objectMorpher, processor objectPro
 		}
 
 		err := processIfPassedFilters(filters, storedObject, processor)
+		_, err = getProcessingError(err)
 
 		// short-circuit if we don't have anything else to scan
 		if isBlob || err != nil {
@@ -219,12 +220,15 @@ func (t *blobTraverser) traverse(preprocessor objectMorpher, processor objectPro
 			return workerError
 		}
 
+
+
 		if t.incrementEnumerationCounter != nil {
 			t.incrementEnumerationCounter(common.EEntityType.File())
 		}
 
 		object := item.(storedObject)
 		processErr := processIfPassedFilters(filters, object, processor)
+		_, processErr = getProcessingError(processErr)
 		if processErr != nil {
 			cancelWorkers()
 			return processErr

cmd/zc_traverser_blobfs.go

@@ -109,7 +109,9 @@ func (t *blobFSTraverser) traverse(preprocessor objectMorpher, processor objectP
 			t.incrementEnumerationCounter(common.EEntityType.File())
 		}
 
-		return processIfPassedFilters(filters, storedObject, processor)
+		err := processIfPassedFilters(filters, storedObject, processor)
+		_, err = getProcessingError(err)
+		return err
 	}
 
 	// else, its not just one file
@@ -139,6 +141,7 @@ func (t *blobFSTraverser) traverse(preprocessor objectMorpher, processor objectP
 			t.incrementEnumerationCounter(common.EEntityType.Folder())
 		}
 		err = processIfPassedFilters(filters, storedObject, processor)
+		_, err = getProcessingError(err)
 		if err != nil {
 			return err
 		}
@@ -194,6 +197,7 @@ func (t *blobFSTraverser) traverse(preprocessor objectMorpher, processor objectP
 			}
 
 			err := processIfPassedFilters(filters, storedObject, processor)
+			_, err = getProcessingError(err)
 			if err != nil {
 				return err
 			}

cmd/zc_traverser_file.go

@@ -87,7 +87,9 @@ func (t *fileTraverser) traverse(preprocessor objectMorpher, processor objectPro
 				t.incrementEnumerationCounter(common.EEntityType.File())
 			}
 
-			return processIfPassedFilters(filters, storedObject, processor)
+			err := processIfPassedFilters(filters, storedObject, processor)
+			_, err = getProcessingError(err)
+			return err
 		}
 	}
 
@@ -145,7 +147,9 @@ func (t *fileTraverser) traverse(preprocessor objectMorpher, processor objectPro
 		if t.incrementEnumerationCounter != nil {
 			t.incrementEnumerationCounter(s.entityType)
 		}
-		return processIfPassedFilters(filters, s, processor)
+		err := processIfPassedFilters(filters, s, processor)
+		_, err = getProcessingError(err)
+		return err
 	}
 
 	// get the directory URL so that we can list the files

cmd/zc_traverser_list.go

@@ -88,7 +88,7 @@ func (l *listTraverser) traverse(preprocessor objectMorpher, processor objectPro
 }
 
 func newListTraverser(parent common.ResourceString, parentType common.Location, credential *common.CredentialInfo, ctx *context.Context,
-	recursive, followSymlinks, getProperties bool, listChan chan string, incrementEnumerationCounter enumerationCounterFunc) resourceTraverser {
+	recursive, followSymlinks, getProperties bool, listChan chan string, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc) resourceTraverser {
 	var traverserGenerator childTraverserGenerator
 
 	traverserGenerator = func(relativeChildPath string) (resourceTraverser, error) {
@@ -104,7 +104,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location,
 		}
 
 		// Construct a traverser that goes through the child
-		traverser, err := initResourceTraverser(source, parentType, ctx, credential, &followSymlinks, nil, recursive, getProperties, false, incrementEnumerationCounter)
+		traverser, err := initResourceTraverser(source, parentType, ctx, credential, &followSymlinks, nil, recursive, getProperties, includeDirectoryStubs, incrementEnumerationCounter)
 		if err != nil {
 			return nil, err
 		}