From 81ba9027669a12a055d942dc98c161cdc98bc7e9 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Fri, 11 May 2018 12:13:42 -0700 Subject: [PATCH 01/23] support added to remove files; force flag feature added for file share --- cmd/copy.go | 6 +- cmd/remove.go | 13 +- ...eEnumerator.go => removeBlobEnumerator.go} | 10 +- cmd/removeFileEnumerator.go | 193 ++++++++++++++++++ cmd/validators.go | 3 +- common/fe-ste-models.go | 3 + ste/mgr-JobPartMgr.go | 2 + ste/xfer-deleteBlob.go | 1 + ste/xfer-deletefile.go | 59 ++++++ ste/xfer-fileToLocal.go | 18 ++ ste/xfer-localToFile.go | 25 +++ ste/xfer.go | 2 + 12 files changed, 323 insertions(+), 12 deletions(-) rename cmd/{removeEnumerator.go => removeBlobEnumerator.go} (94%) create mode 100644 cmd/removeFileEnumerator.go create mode 100644 ste/xfer-deletefile.go diff --git a/cmd/copy.go b/cmd/copy.go index 81d3eee26..999a8626d 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -400,7 +400,11 @@ func (cca cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { err = e.enumerate(cca.src, cca.recursive, cca.dst, &wg, cca.waitUntilJobCompletion) lastPartNumber = e.PartNum case common.EFromTo.BlobTrash(): - e := removeEnumerator(jobPartOrder) + e := removeBlobEnumerator(jobPartOrder) + err = e.enumerate(cca.src, cca.recursive, cca.dst, &wg, cca.waitUntilJobCompletion) + lastPartNumber = e.PartNum + case common.EFromTo.FileTrash(): + e := removeFileEnumerator(jobPartOrder) err = e.enumerate(cca.src, cca.recursive, cca.dst, &wg, cca.waitUntilJobCompletion) lastPartNumber = e.PartNum } diff --git a/cmd/remove.go b/cmd/remove.go index d6883f1d8..da934c7f5 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -43,12 +43,15 @@ func init() { if len(args) != 1 { return fmt.Errorf("remove command only takes 1 arguments. Passed %d arguments", len(args)) } - argsLocation := inferArgumentLocation(args[0]) - if argsLocation != argsLocation.Blob() { - return fmt.Errorf("remove command supports delete of blob only. Passed %v as an argument", argsLocation) - } raw.src = args[0] - raw.fromTo = common.EFromTo.BlobTrash().String() + srcLocationType := inferArgumentLocation(raw.src) + if srcLocationType == ELocation.Blob(){ + raw.fromTo = common.EFromTo.BlobTrash().String() + }else if srcLocationType == ELocation.File(){ + raw.fromTo = common.EFromTo.FileTrash().String() + }else { + return fmt.Errorf("invalid source type %s pased to delete. azcopy support removing blobs and files only", srcLocationType.String()) + } return nil }, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/cmd/removeEnumerator.go b/cmd/removeBlobEnumerator.go similarity index 94% rename from cmd/removeEnumerator.go rename to cmd/removeBlobEnumerator.go index 941d1f0bb..b20f81528 100644 --- a/cmd/removeEnumerator.go +++ b/cmd/removeBlobEnumerator.go @@ -13,10 +13,10 @@ import ( "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" ) -type removeEnumerator common.CopyJobPartOrderRequest +type removeBlobEnumerator common.CopyJobPartOrderRequest // this function accepts a url (with or without *) to blobs for download and processes them -func (e *removeEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, +func (e *removeBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { util := copyHandlerUtil{} @@ -154,13 +154,13 @@ func (e *removeEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, } // accept a new transfer, simply add to the list of transfers and wait for the dispatch call to send the order -func (e *removeEnumerator) addTransfer(transfer common.CopyTransfer, wg *sync.WaitGroup, +func (e *removeBlobEnumerator) addTransfer(transfer common.CopyTransfer, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { return addTransfer((*common.CopyJobPartOrderRequest)(e), transfer, wg, waitUntilJobCompletion) } // send the current list of transfer to the STE -func (e *removeEnumerator) dispatchFinalPart() error { +func (e *removeBlobEnumerator) dispatchFinalPart() error { // if the job is empty, throw an error if len(e.Transfers) == 0 { return errors.New("cannot initiate empty job, please make sure source is not empty or is a valid source") @@ -176,6 +176,6 @@ func (e *removeEnumerator) dispatchFinalPart() error { return nil } -func (e *removeEnumerator) partNum() common.PartNumber { +func (e *removeBlobEnumerator) partNum() common.PartNumber { return e.PartNum } diff --git a/cmd/removeFileEnumerator.go b/cmd/removeFileEnumerator.go new file mode 100644 index 000000000..9886f30f3 --- /dev/null +++ b/cmd/removeFileEnumerator.go @@ -0,0 +1,193 @@ +package cmd + +import ( + "context" + "fmt" + "net/url" + "strings" + "sync" + + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-azcopy/ste" + "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" +) + +type removeFileEnumerator common.CopyJobPartOrderRequest + +// enumerate accepts an URL (with or without *) pointing to file/directory for enumerate, processe and remove. +// The method supports two general cases: +// Case 1: End with star, means remove files with specified prefix. +// directory/fprefix* +// directory/* (this expression is transferred to remove from directory, means remove all files in a directory.) +// Case 2: Not end with star, means remove a single file or a directory. +// directory/dir +// directory/file +func (e *removeFileEnumerator) enumerate(sourceURLString string, isRecursiveOn bool, destinationPath string, + wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + + // Init params. + util := copyHandlerUtil{} + p := azfile.NewPipeline( + azfile.NewAnonymousCredential(), + azfile.PipelineOptions{ + Retry: azfile.RetryOptions{ + Policy: azfile.RetryPolicyExponential, + MaxTries: ste.UploadMaxTries, + TryTimeout: ste.UploadTryTimeout, + RetryDelay: ste.UploadRetryDelay, + MaxRetryDelay: ste.UploadMaxRetryDelay, + }, + }) + ctx := context.TODO() // Ensure correct context is used + cookedSourceURLString := util.replaceBackSlashWithSlash(sourceURLString) // Replace back slash with slash, otherwise url.Parse would encode the back slash. + + // Attempt to parse the source url. + sourceURL, err := url.Parse(cookedSourceURLString) + if err != nil { + return fmt.Errorf("cannot parse source URL") + } + + // Validate the source url. + numOfStartInURLPath := util.numOfStarInUrl(sourceURL.Path) + if numOfStartInURLPath > 1 || (numOfStartInURLPath == 1 && !strings.HasSuffix(sourceURL.Path, "*")) { + return fmt.Errorf("only support prefix matching (e.g: fileprefix*), or exact matching") + } + doPrefixSearch := numOfStartInURLPath == 1 + + // For prefix search, only support file name matching in file prefix's parent dir level. + if isRecursiveOn && doPrefixSearch { + return fmt.Errorf("only support file name matching in file prefix's parent dir level, prefix matching with recursive mode is not supported currently for Azure file remove") + } + + // Get the DirectoryURL or FileURL to be later used for listing. + dirURL, fileURL, fileProperties, ok := util.getDeepestDirOrFileURLFromString(ctx, *sourceURL, p) + + if !ok { + return fmt.Errorf("cannot find accessible file or base directory with specified sourceURLString") + } + + + + // Check if source URL is in directory/* expression, and transfer it to remove from directory if the express is directory/*. + if hasEquivalentDirectoryURL, equivalentURL := util.hasEquivalentDirectoryURL(*sourceURL); hasEquivalentDirectoryURL { + *sourceURL = equivalentURL + doPrefixSearch = false + } + + if doPrefixSearch { // Case 1: Do prefix search, the file pattern would be [AnyLetter]+\* + // The destination must be a directory, otherwise we don't know where to put the files. + if !util.isPathDirectory(destinationPath) { + return fmt.Errorf("the destination must be an existing directory in this remove scenario") + } + + // If there is * it's matching a file (like pattern matching) + // get the search prefix to query the service + searchPrefix := util.getPossibleFileNameFromURL(sourceURL.Path) + if searchPrefix == "" { + panic("invalid state, searchPrefix should not be emtpy in do prefix search.") + } + searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end + + // Perform list files and directories, note only files would be matched and transferred in prefix search. + for marker := (azfile.Marker{}); marker.NotDone(); { + // Look for all files that start with the prefix. + lResp, err := dirURL.ListFilesAndDirectoriesSegment(ctx, marker, azfile.ListFilesAndDirectoriesOptions{Prefix: searchPrefix}) + if err != nil { + return err + } + + // Process the files returned in this result segment. + for _, fileInfo := range lResp.Files { + f := dirURL.NewFileURL(fileInfo.Name) + _, err := f.GetProperties(ctx) // TODO: the cost is high while otherwise we cannot get the last modified time. As Azure file's PM description, list might get more valuable file properties later, optimize the logic after the change... + if err != nil { + return err + } + + e.addTransfer(common.CopyTransfer{ + Source: f.String(), + SourceSize: fileInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) + } + + marker = lResp.NextMarker + } + + err = e.dispatchFinalPart() + if err != nil { + return err + } + + } else { // Case 2: remove a single file or a directory. + + if fileURL != nil { // Single file. + e.addTransfer( + common.CopyTransfer{ + Source: sourceURL.String(), + SourceSize: fileProperties.ContentLength(), + }, + wg, + waitUntilJobCompletion) + + } else { // Directory. + dirStack := &directoryStack{} + dirStack.Push(*dirURL) + for currentDirURL, ok := dirStack.Pop(); ok; currentDirURL, ok = dirStack.Pop() { + // Perform list files and directories. + for marker := (azfile.Marker{}); marker.NotDone(); { + lResp, err := currentDirURL.ListFilesAndDirectoriesSegment(ctx, marker, azfile.ListFilesAndDirectoriesOptions{}) + if err != nil { + return fmt.Errorf("cannot list files for remove") + } + + // Process the files returned in this segment. + for _, fileInfo := range lResp.Files { + f := currentDirURL.NewFileURL(fileInfo.Name) + _, err := f.GetProperties(ctx) // TODO: the cost is high while otherwise we cannot get the last modified time. As Azure file's PM description, list might get more valuable file properties later, optimize the logic after the change... + if err != nil { + return err + } + + e.addTransfer( + common.CopyTransfer{ + Source: f.String(), + SourceSize: fileInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) + } + + // If recursive is turned on, add sub directories. + if isRecursiveOn { + for _, dirInfo := range lResp.Directories { + d := currentDirURL.NewDirectoryURL(dirInfo.Name) + dirStack.Push(d) + } + } + marker = lResp.NextMarker + } + } + } + + err = e.dispatchFinalPart() + if err != nil { + return err + } + } + + return nil +} + +func (e *removeFileEnumerator) addTransfer(transfer common.CopyTransfer, wg *sync.WaitGroup, + waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + return addTransfer((*common.CopyJobPartOrderRequest)(e), transfer, wg, waitUntilJobCompletion) +} + +func (e *removeFileEnumerator) dispatchFinalPart() error { + return dispatchFinalPart((*common.CopyJobPartOrderRequest)(e)) +} + +func (e *removeFileEnumerator) partNum() common.PartNumber { + return e.PartNum +} + diff --git a/cmd/validators.go b/cmd/validators.go index 5602687d3..f25fd5b5a 100644 --- a/cmd/validators.go +++ b/cmd/validators.go @@ -46,7 +46,8 @@ func validateFromTo(src, dst string, userSpecifiedFromTo string) (common.FromTo, if err != nil { return common.EFromTo.Unknown(), fmt.Errorf("Invalid --FromTo value specified: %q", userSpecifiedFromTo) } - if inferredFromTo == common.EFromTo.Unknown() || inferredFromTo == userFromTo || userFromTo == common.EFromTo.BlobTrash() { + if inferredFromTo == common.EFromTo.Unknown() || inferredFromTo == userFromTo || + userFromTo == common.EFromTo.BlobTrash() || userFromTo == common.EFromTo.FileTrash(){ // We couldn't infer the FromTo or what we inferred matches what the user specified // We'll accept what the user specified return userFromTo, nil diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index b32ae3938..1982d88e1 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -161,6 +161,7 @@ func (FromTo) PipeBlob() FromTo { return FromTo(6) } func (FromTo) FilePipe() FromTo { return FromTo(7) } func (FromTo) PipeFile() FromTo { return FromTo(8) } func (FromTo) BlobTrash() FromTo { return FromTo(9) } +func (FromTo) FileTrash() FromTo { return FromTo(10) } func (ft FromTo) String() string { return EnumHelper{}.StringInteger(ft, reflect.TypeOf(ft)) @@ -197,6 +198,8 @@ func (TransferStatus) BlobTierFailure() TransferStatus { return TransferStatus(- func (TransferStatus) BlobAlreadyExistsFailure() TransferStatus { return TransferStatus(-3) } +func (TransferStatus) FileAlreadyExistsFailure() TransferStatus { return TransferStatus(-4) } + func (ts TransferStatus) ShouldTransfer() bool { return ts == ETransferStatus.NotStarted() || ts == ETransferStatus.Started() } diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 6fe3bd706..520035e39 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -254,6 +254,8 @@ func (jpm *jobPartMgr) createPipeline() { RetryDelay: UploadRetryDelay, MaxRetryDelay: UploadMaxRetryDelay}, jpm.pacer) + case common.EFromTo.FileTrash(): + fallthrough case common.EFromTo.FileLocal(): // download from Azure File to local file system fallthrough case common.EFromTo.LocalFile(): // upload from local file system to Azure File diff --git a/ste/xfer-deleteBlob.go b/ste/xfer-deleteBlob.go index 1a2c2f9ae..68567bc78 100644 --- a/ste/xfer-deleteBlob.go +++ b/ste/xfer-deleteBlob.go @@ -43,6 +43,7 @@ func DeleteBlobPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pa _, err := srcBlobURL.Delete(jptm.Context(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) if err != nil { + // If the delete failed with err 404, i.e resource not found, then mark the transfer as success. if err.(azblob.StorageError) != nil { if err.(azblob.StorageError).Response().StatusCode == http.StatusNotFound { transferDone(common.ETransferStatus.Success(), nil) diff --git a/ste/xfer-deletefile.go b/ste/xfer-deletefile.go new file mode 100644 index 000000000..74d7c5321 --- /dev/null +++ b/ste/xfer-deletefile.go @@ -0,0 +1,59 @@ +package ste + +import ( + "fmt" + "github.com/Azure/azure-pipeline-go/pipeline" + "github.com/Azure/azure-storage-azcopy/common" + "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" + "net/http" + "net/url" + "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" +) + +func DeleteFilePrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { + + info := jptm.Info() + // Get the source file url of file to delete + u, _ := url.Parse(info.Source) + srcFileUrl := azfile.NewFileURL(*u, p) + + // If the transfer was cancelled, then reporting transfer as done and increasing the bytestransferred by the size of the source. + if jptm.WasCanceled() { + jptm.AddToBytesDone(info.SourceSize) + jptm.ReportTransferDone() + return + } + + // Internal function which checks the transfer status and logs the msg respectively. + // Sets the transfer status and Report Transfer as Done. + // Internal function is created to avoid redundancy of the above steps from several places in the api. + transferDone := func(status common.TransferStatus, err error) { + if jptm.ShouldLog(pipeline.LogInfo) { + var msg = "" + if status == common.ETransferStatus.Failed() { + msg = fmt.Sprintf("delete blob failed. Failed with error %s", err.Error()) + } else { + msg = "blob delete successful" + } + jptm.Log(pipeline.LogInfo, msg) + } + jptm.SetStatus(status) + jptm.AddToBytesDone(info.SourceSize) + jptm.ReportTransferDone() + } + + // Delete the source file + _, err := srcFileUrl.Delete(jptm.Context()) + if err != nil { + // If the delete failed with err 404, i.e resource not found, then mark the transfer as success. + if err.(azblob.StorageError) != nil { + if err.(azblob.StorageError).Response().StatusCode == http.StatusNotFound { + transferDone(common.ETransferStatus.Success(), nil) + } + } else { + transferDone(common.ETransferStatus.Failed(), err) + } + } else { + transferDone(common.ETransferStatus.Success(), nil) + } +} diff --git a/ste/xfer-fileToLocal.go b/ste/xfer-fileToLocal.go index ee446bfc5..6ea9ff6db 100644 --- a/ste/xfer-fileToLocal.go +++ b/ste/xfer-fileToLocal.go @@ -51,6 +51,24 @@ func FileToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { return } + // If the force Write flags is set to false + // then check the file exists locally or not. + // If it does, mark transfer as failed. + if !jptm.IsForceWriteTrue() { + _, err := os.Stat(info.Destination) + if err == nil{ + // If the error is nil, then blob exists locally and it doesn't needs to be downloaded. + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, fmt.Sprintf("skipping the transfer since blob already exists")) + } + // Mark the transfer as failed with FileAlreadyExistsFailure + jptm.SetStatus(common.ETransferStatus.FileAlreadyExistsFailure()) + jptm.AddToBytesDone(info.SourceSize) + jptm.ReportTransferDone() + return + } + } + // step 3: prep local file before download starts if fileSize == 0 { err := createEmptyFile(info.Destination) diff --git a/ste/xfer-localToFile.go b/ste/xfer-localToFile.go index 5f88e3bb0..3fc1f58b8 100644 --- a/ste/xfer-localToFile.go +++ b/ste/xfer-localToFile.go @@ -51,6 +51,31 @@ func LocalToFile(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { // TODO: remove the hard coded chunk size in case of pageblob or Azure file chunkSize = common.DefaultAzureFileChunkSize + // If the transfer was cancelled, then reporting transfer as done and increasing the bytestransferred by the size of the source. + if jptm.WasCanceled() { + jptm.AddToBytesDone(info.SourceSize) + jptm.ReportTransferDone() + return + } + + // If the force Write flags is set to false + // then check the file exists or not. + // If it does, mark transfer as failed. + if !jptm.IsForceWriteTrue() { + _, err := fileURL.GetProperties(jptm.Context()) + if err == nil{ + // If the error is nil, then blob exists and it doesn't needs to be uploaded. + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, fmt.Sprintf("skipping the transfer since blob already exists")) + } + // Mark the transfer as failed with FileAlreadyExistsFailure + jptm.SetStatus(common.ETransferStatus.FileAlreadyExistsFailure()) + jptm.AddToBytesDone(info.SourceSize) + jptm.ReportTransferDone() + return + } + } + // step 2: Map file upload before transferring chunks and get info from map file. srcFile, err := os.Open(info.Source) if err != nil { diff --git a/ste/xfer.go b/ste/xfer.go index e0652269c..a1d468b21 100644 --- a/ste/xfer.go +++ b/ste/xfer.go @@ -62,6 +62,8 @@ func computeJobXfer(fromTo common.FromTo) newJobXfer { return FileToLocal case common.EFromTo.LocalFile(): // upload from local file system to Azure File return LocalToFile + case common.EFromTo.FileTrash(): + return DeleteFilePrologue } panic(fmt.Errorf("Unrecognized FromTo: %q", fromTo.String())) } From a50d9c20baa0da013498f58c0eab6a390586f433 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Mon, 14 May 2018 13:38:01 -0700 Subject: [PATCH 02/23] added support for canceling job by writing cancel on stdin --- cmd/cancel.go | 39 ++++++++++++++++++++++++++++++++++++++- cmd/copy.go | 3 +-- cmd/resume.go | 3 +-- cmd/sync.go | 3 +-- common/fe-ste-models.go | 2 +- main.go | 2 ++ 6 files changed, 44 insertions(+), 8 deletions(-) diff --git a/cmd/cancel.go b/cmd/cancel.go index 1659865d0..fb8d7dce1 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -25,8 +25,16 @@ import ( "fmt" "github.com/Azure/azure-storage-azcopy/common" "github.com/spf13/cobra" + "bufio" + "os" + "strings" ) +// created a signal channel to receive the Interrupt and Kill signal send to OS +// this channel is shared by copy, resume, sync and an independent go routine reading stdin +// for cancel command +var cancelChannel = make(chan os.Signal, 1) + type rawCancelCmdArgs struct { jobID string } @@ -51,7 +59,6 @@ type cookedCancelCmdArgs struct { func (cca cookedCancelCmdArgs) process() error { var cancelJobResponse common.CancelPauseResumeResponse Rpc(common.ERpcCmd.CancelJob(), cca.jobID, &cancelJobResponse) - if !cancelJobResponse.CancelledPauseResumed { return fmt.Errorf("job cannot be cancelled because %s", cancelJobResponse.ErrorMsg) } @@ -95,3 +102,33 @@ func init() { } rootCmd.AddCommand(cancelCmd) } + +// ReadStandardInputToCancelJob is a function that reads the standard Input +// If Input given is "cancel", it cancels the current job. +func ReadStandardInputToCancelJob() { + for { + consoleReader := bufio.NewReader(os.Stdin) + // ReadString reads input until the first occurrence of \n in the input, + input, err := consoleReader.ReadString('\n') + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + //remove the delimiter "\n" + input = strings.Trim(input, "\n") + // remove trailing white spaces + input = strings.Trim(input, " ") + // converting the input characters to lower case characters + // this is done to avoid case sensitiveness. + input = strings.ToLower(input) + + switch input { + case "cancel": + // send a kill signal to the cancel channel. + cancelChannel <- os.Kill + default: + panic(fmt.Errorf("command %s not supported by azcopy", input)) + } + } +} \ No newline at end of file diff --git a/cmd/copy.go b/cmd/copy.go index 999a8626d..d326028d6 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -432,8 +432,7 @@ func (cca cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { } func (cca cookedCopyCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync.WaitGroup) { - // created a signal channel to receive the Interrupt and Kill signal send to OS - cancelChannel := make(chan os.Signal, 1) + // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals signal.Notify(cancelChannel, os.Interrupt, os.Kill) diff --git a/cmd/resume.go b/cmd/resume.go index 42fde3301..89f793fc1 100644 --- a/cmd/resume.go +++ b/cmd/resume.go @@ -58,8 +58,7 @@ func init() { } func waitUntilJobCompletion(jobID common.JobID) { - // created a signal channel to receive the Interrupt and Kill signal send to OS - cancelChannel := make(chan os.Signal, 1) + // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals signal.Notify(cancelChannel, os.Interrupt, os.Kill) diff --git a/cmd/sync.go b/cmd/sync.go index 45025f1c6..227a4ca7b 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -101,8 +101,7 @@ func (cca cookedSyncCmdArgs) process() (err error) { } func (cca cookedSyncCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync.WaitGroup) { - // created a signal channel to receive the Interrupt and Kill signal send to OS - cancelChannel := make(chan os.Signal, 1) + // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals signal.Notify(cancelChannel, os.Interrupt, os.Kill) diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 1982d88e1..7396a6e47 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -141,7 +141,7 @@ func (JobStatus) Paused() JobStatus { return JobStatus(1) } func (JobStatus) Cancelled() JobStatus { return JobStatus(2) } func (JobStatus) Completed() JobStatus { return JobStatus(3) } func (js JobStatus) String() string { - return EnumHelper{}.StringInteger(uint32(js), reflect.TypeOf(js)) + return EnumHelper{}.StringInteger(js, reflect.TypeOf(js)) } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/main.go b/main.go index 5d33c92f2..1e8a8b555 100644 --- a/main.go +++ b/main.go @@ -46,8 +46,10 @@ func mainWithExitCode() exitCode { cmd.Execute() return eexitCode.success() } + go cmd.ReadStandardInputToCancelJob() azcopyAppPathFolder := GetAzCopyAppPath() go ste.MainSTE(300, 500, azcopyAppPathFolder) cmd.Execute() return eexitCode.success() } + From 3afffd9abc0a505396ff3d4e8b6bf1741d7a82ca Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Mon, 14 May 2018 15:15:48 -0700 Subject: [PATCH 03/23] log level input changed to string --- cmd/copy.go | 12 ++++++------ cmd/remove.go | 4 +--- cmd/sync.go | 10 ++++++---- common/fe-ste-models.go | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index d326028d6..80555cb68 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -33,8 +33,6 @@ import ( "sync" "sync/atomic" "time" - - "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/spf13/cobra" @@ -85,7 +83,7 @@ type rawCopyCmdArgs struct { background bool outputJson bool acl string - logVerbosity byte + logVerbosity string } // validates and transform raw input into cooked input @@ -117,6 +115,10 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { if err != nil { return cooked, err } + err = cooked.logVerbosity.Parse(raw.logVerbosity) + if err != nil{ + return cooked, err + } cooked.metadata = raw.metadata cooked.contentType = raw.contentType cooked.contentEncoding = raw.contentEncoding @@ -125,8 +127,6 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { cooked.background = raw.background cooked.outputJson = raw.outputJson cooked.acl = raw.acl - cooked.logVerbosity = common.LogLevel(raw.logVerbosity) - return cooked, nil } @@ -553,5 +553,5 @@ Usage: cpCmd.PersistentFlags().BoolVar(&raw.background, "background-op", false, "true if user has to perform the operations as a background operation") cpCmd.PersistentFlags().BoolVar(&raw.outputJson, "output-json", false, "true if user wants the output in Json format") cpCmd.PersistentFlags().StringVar(&raw.acl, "acl", "", "Access conditions to be used when uploading/downloading from Azure Storage.") - cpCmd.PersistentFlags().Uint8Var(&raw.logVerbosity, "Logging", uint8(pipeline.LogWarning), "defines the log verbosity to be saved to log file") + cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "Logging", "None", "defines the log verbosity to be saved to log file") } diff --git a/cmd/remove.go b/cmd/remove.go index da934c7f5..2f1935a3e 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -22,8 +22,6 @@ package cmd import ( "fmt" - "github.com/Azure/azure-pipeline-go/pipeline" - //"github.com/Azure/azure-storage-azcopy/common" "github.com/spf13/cobra" "github.com/Azure/azure-storage-azcopy/common" ) @@ -70,6 +68,6 @@ func init() { rootCmd.AddCommand(deleteCmd) deleteCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when deleting from container.") - deleteCmd.PersistentFlags().Uint8Var(&raw.logVerbosity, "Logging", uint8(pipeline.LogWarning), "defines the log verbosity to be saved to log file") + deleteCmd.PersistentFlags().StringVar(&raw.logVerbosity, "Logging", "None", "defines the log verbosity to be saved to log file") deleteCmd.PersistentFlags().BoolVar(&raw.outputJson, "output-json", false, "true if user wants the output in Json format") } diff --git a/cmd/sync.go b/cmd/sync.go index 227a4ca7b..173924d48 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -22,7 +22,6 @@ package cmd import ( "fmt" - "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" "github.com/spf13/cobra" "os" @@ -37,7 +36,7 @@ type syncCommandArguments struct { recursive bool // options from flags blockSize uint32 - logVerbosity byte + logVerbosity string } // validates and transform raw input into cooked input @@ -56,7 +55,10 @@ func (raw syncCommandArguments) cook() (cookedSyncCmdArgs, error) { cooked.blockSize = raw.blockSize - cooked.logVerbosity = common.LogLevel(raw.logVerbosity) + err := cooked.logVerbosity.Parse(raw.logVerbosity) + if err != nil{ + return cooked, err + } cooked.recursive = raw.recursive @@ -164,5 +166,5 @@ func init() { rootCmd.AddCommand(syncCmd) syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when syncing destination to source.") syncCmd.PersistentFlags().Uint32Var(&raw.blockSize, "block-size", 100*1024*1024, "Use this block size when source to Azure Storage or from Azure Storage.") - syncCmd.PersistentFlags().Uint8Var(&raw.logVerbosity, "Logging", uint8(pipeline.LogWarning), "defines the log verbosity to be saved to log file") + syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "Logging", "None", "defines the log verbosity to be saved to log file") } diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 7396a6e47..19c787e8f 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -28,6 +28,8 @@ import ( "sync/atomic" "time" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" + "fmt" + "strings" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -80,6 +82,37 @@ type Status uint32 type LogLevel byte +func (ll *LogLevel) Parse(s string) error { + // converting string s to all lower case string + // to avoid case sensitiveness + s = strings.ToLower(s) + switch s { + case "none": + *ll = 0 + return nil + case "fatal": + *ll = 1 + return nil + case "error": + *ll = 2 + return nil + case "panic": + *ll = 3 + return nil + case "warning": + *ll = 4 + return nil + case "info": + *ll = 5 + return nil + case "debug": + *ll = 6 + return nil + default: + return fmt.Errorf("invaild log type %s passed. Azcopy supports none, fatal, error, panic, warning, info and debug log levels",s) + } +} + func (ll LogLevel) ToPipelineLogLevel() pipeline.LogLevel { // This assumes that pipeline's LogLevel values can fit in a byte (which they can) return pipeline.LogLevel(ll) From 7e3fd31333a4375aecfb83a4197d634bea90292c Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Mon, 14 May 2018 17:57:16 -0700 Subject: [PATCH 04/23] fixed test cases for the recent change 'log level input changed to string' --- testSuite/scripts/init.py | 4 +- testSuite/scripts/test_azcopy_operations.py | 8 ++-- testSuite/scripts/test_blob_download.py | 18 ++++---- testSuite/scripts/test_file_download.py | 32 +++++++------- testSuite/scripts/test_file_upload.py | 22 +++++----- testSuite/scripts/test_upload_block_blob.py | 46 ++++++++++++--------- testSuite/scripts/test_upload_page_blob.py | 20 ++++----- testSuite/scripts/utility.py | 42 ++++++++++++++++++- 8 files changed, 119 insertions(+), 73 deletions(-) diff --git a/testSuite/scripts/init.py b/testSuite/scripts/init.py index 2f217f07b..4220631dc 100644 --- a/testSuite/scripts/init.py +++ b/testSuite/scripts/init.py @@ -8,7 +8,7 @@ import configparser import platform -def execute_user_scenario_1() : +def execute_user_scenario_blob_1() : test_1kb_blob_upload() test_63mb_blob_upload() test_n_1kb_blob_upload(5) @@ -162,7 +162,7 @@ def cleanup(): def main(): init() execute_user_scenario_azcopy_op() - execute_user_scenario_1() + execute_user_scenario_blob_1() #execute_user_scenario_2() #execute_user_scenario_file_1() #temp_adhoc_scenario() diff --git a/testSuite/scripts/test_azcopy_operations.py b/testSuite/scripts/test_azcopy_operations.py index c5d42a726..a75c966a3 100644 --- a/testSuite/scripts/test_azcopy_operations.py +++ b/testSuite/scripts/test_azcopy_operations.py @@ -9,7 +9,7 @@ def test_cancel_job(): file_path = create_test_file(file_name, 1024*1024*1024) # execute the azcopy upload job in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "5").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -41,7 +41,7 @@ def test_pause_resume_job_95Mb_file(): file_path = create_test_file(file_name, 95*1024*1024) # execute azcopy file upload in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "5").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -85,7 +85,7 @@ def test_pause_resume_job_200Mb_file(): file_path = create_test_file(file_name, 200*1024*1024) # execute azcopy file upload in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "5").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -131,7 +131,7 @@ def test_remove_virtual_directory(): # execute azcopy command result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_remove_virtual_directory failed while uploading ", dir_n_files_path, " files to the container") return diff --git a/testSuite/scripts/test_blob_download.py b/testSuite/scripts/test_blob_download.py index aac9c786a..c355cc63b 100644 --- a/testSuite/scripts/test_blob_download.py +++ b/testSuite/scripts/test_blob_download.py @@ -13,7 +13,7 @@ def test_download_1kb_blob() : src = file_path dest = util.test_container_url result = util.Command("copy").add_arguments(src).add_arguments(dest). \ - add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the container") return @@ -32,7 +32,7 @@ def test_download_1kb_blob() : # downloading the uploaded file src = util.get_resource_sas(filename) dest = util.test_directory_path + "/test_1kb_blob_download.txt" - result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("Logging", "5").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_download_1kb_blob test case failed") @@ -56,7 +56,7 @@ def test_blob_download_preserve_last_modified_time() : # upload file through azcopy. destination_sas = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the container") return @@ -71,7 +71,7 @@ def test_blob_download_preserve_last_modified_time() : # download file through azcopy with flag preserve-last-modified-time set to true download_file_name = util.test_directory_path + "/test_download_preserve_last_mtime.txt" - result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name).add_flags("Logging", "5").add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name).add_flags("Logging", "info").add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command() if not result: print("test_download_perserve_last_modified_time test case failed") return @@ -92,7 +92,7 @@ def test_blob_download_63mb_in_4mb(): # uploading file through azcopy with flag block-size set to 4194304 i.e 4mb destination_sas = util.get_resource_sas(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").add_flags("block-size", "4194304").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").add_flags("block-size", "4194304").execute_azcopy_copy_command() if not result: print("error uploading 63 mb file. test_blob_download_63mb_in_4mb test case failed") return @@ -105,7 +105,7 @@ def test_blob_download_63mb_in_4mb(): # downloading the created parallely in blocks of 4mb file through azcopy. download_file = util.test_directory_path + "/test_63mb_in4mb_download.txt" - result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file).add_flags("Logging", "5").add_flags("block-size", "4194304").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file).add_flags("Logging", "info").add_flags("block-size", "4194304").execute_azcopy_copy_command() if not result: print("error downloading the 63mb file. test_blob_download_63mb_in_4mb test case failed") return @@ -125,7 +125,7 @@ def test_recursive_download_blob(): dir1_path = util.create_test_n_files(1024, 5, dir_name) # upload the directory to container through azcopy with recursive set to true. - result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_container_url).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_container_url).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("error uploading recursive dir ", dir1_path) return @@ -144,7 +144,7 @@ def test_recursive_download_blob(): return # downloading the directory created from container through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("error download recursive dir ", dir1_path) return @@ -165,7 +165,7 @@ def test_blob_download_with_special_characters(): print("error creating blob ", filename_special_characters, " with special characters") return # downloading the blob created above. - result = util.Command("copy").add_arguments(resource_url).add_arguments(util.test_directory_path).add_flags("Logging", "5").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(resource_url).add_arguments(util.test_directory_path).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("error downloading the file with special characters ", filename_special_characters) return diff --git a/testSuite/scripts/test_file_download.py b/testSuite/scripts/test_file_download.py index 360292a4e..7fd694a3f 100644 --- a/testSuite/scripts/test_file_download.py +++ b/testSuite/scripts/test_file_download.py @@ -13,7 +13,7 @@ def test_upload_download_1kb_file_fullname() : src = file_path dest = util.test_share_url result = util.Command("copy").add_arguments(src).add_arguments(dest). \ - add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the share") return @@ -32,7 +32,7 @@ def test_upload_download_1kb_file_fullname() : # downloading the uploaded file src = util.get_resource_sas_from_share(filename) dest = util.test_directory_path + "/test_1kb_file_download.txt" - result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("Logging", "5").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_upload_download_1kb_file_fullname test case failed") @@ -56,7 +56,7 @@ def test_upload_download_1kb_file_wildcard_all_files() : # Upload 1KB file using azcopy. result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url). \ - add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the share") return @@ -84,7 +84,7 @@ def test_upload_download_1kb_file_wildcard_all_files() : finally: os.makedirs(dest) - result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(dest).add_flags("Logging", "5").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(dest).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_upload_download_1kb_file_wildcard_all_files test case failed") @@ -109,7 +109,7 @@ def test_upload_download_1kb_file_wildcard_several_files() : # Upload 1KB file using azcopy. result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url). \ - add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the share") return @@ -137,7 +137,7 @@ def test_upload_download_1kb_file_wildcard_several_files() : finally: os.makedirs(dest) - result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(dest).add_flags("Logging", "5").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(dest).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_upload_download_1kb_file_wildcard_several_files test case failed") @@ -169,7 +169,7 @@ def test_n_1kb_file_in_dir_upload_download_share(number_of_files): # execute azcopy command dest_share = util.test_share_url result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_share).\ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_n_1kb_file_in_dir_upload_download_share failed while uploading ", number_of_files, " files to the share") return @@ -193,7 +193,7 @@ def test_n_1kb_file_in_dir_upload_download_share(number_of_files): os.makedirs(download_local_dest_dir) # downloading the directory created from azure file share through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments(download_local_dest_dir).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments(download_local_dest_dir).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("error download recursive dir ", download_azure_src_dir) return @@ -237,7 +237,7 @@ def test_n_1kb_file_in_dir_upload_download_azure_directory(number_of_files, recu # execute azcopy command result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_azure_dir).\ - add_flags("recursive", recursive).add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", recursive).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_n_1kb_file_in_dir_upload_download_azure_directory failed while uploading ", number_of_files, " files to the share") return @@ -261,7 +261,7 @@ def test_n_1kb_file_in_dir_upload_download_azure_directory(number_of_files, recu os.makedirs(download_local_dest_dir) # downloading the directory created from azure file directory through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments(download_local_dest_dir).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments(download_local_dest_dir).add_flags("Logging", "info").\ add_flags("recursive", recursive).execute_azcopy_copy_command() if not result: print("error download recursive dir ", download_azure_src_dir) @@ -286,7 +286,7 @@ def test_download_perserve_last_modified_time() : # upload file through azcopy. destination_sas = util.get_resource_sas_from_share(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the share") return @@ -301,7 +301,7 @@ def test_download_perserve_last_modified_time() : # download file through azcopy with flag preserve-last-modified-time set to true download_file_name = util.test_directory_path + "/test_download_preserve_last_mtime.txt" - result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name).add_flags("Logging", "5").add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file_name).add_flags("Logging", "info").add_flags("preserve-last-modified-time", "true").execute_azcopy_copy_command() if not result: print("test_download_perserve_last_modified_time test case failed") return @@ -322,7 +322,7 @@ def test_file_download_63mb_in_4mb(): # uploading file through azcopy with flag block-size set to 4194304 i.e 4mb destination_sas = util.get_resource_sas_from_share(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").add_flags("block-size", "4194304").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").add_flags("block-size", "4194304").execute_azcopy_copy_command() if not result: print("error uploading 63 mb file. test_file_download_63mb_in_4mb test case failed") return @@ -335,7 +335,7 @@ def test_file_download_63mb_in_4mb(): # downloading the created parallely in blocks of 4mb file through azcopy. download_file = util.test_directory_path + "/test_63mb_in4mb_download.txt" - result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file).add_flags("Logging", "5").add_flags("block-size", "4194304").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(download_file).add_flags("Logging", "info").add_flags("block-size", "4194304").execute_azcopy_copy_command() if not result: print("error downloading the 63mb file. test_file_download_63mb_in_4mb test case failed") return @@ -355,7 +355,7 @@ def test_recursive_download_file(): dir1_path = util.create_test_n_files(1024, 5, dir_name) # upload the directory to share through azcopy with recursive set to true. - result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_share_url).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_share_url).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("error uploading recursive dir ", dir1_path) return @@ -374,7 +374,7 @@ def test_recursive_download_file(): return # downloading the directory created from share through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("error download recursive dir ", dir1_path) return diff --git a/testSuite/scripts/test_file_upload.py b/testSuite/scripts/test_file_upload.py index 163ea4fcc..9f3b42e96 100644 --- a/testSuite/scripts/test_file_upload.py +++ b/testSuite/scripts/test_file_upload.py @@ -8,7 +8,7 @@ def test_file_upload_1mb_fullname(): # execute azcopy upload. destination = util.get_resource_sas_from_share(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -29,7 +29,7 @@ def test_file_upload_1mb_wildcard(): # execute azcopy upload. destination = util.get_resource_sas_from_share(file_name) wildcard_path = file_path.replace(file_name, "*") - result = util.Command("copy").add_arguments(wildcard_path).add_arguments(destination).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(wildcard_path).add_arguments(destination).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -51,7 +51,7 @@ def test_file_range_for_complete_sparse_file(): # execute azcopy file upload. destination_sas = util.get_resource_sas_from_share(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -74,7 +74,7 @@ def test_file_upload_partial_sparse_file(): # execute azcopy file upload. destination_sas = util.get_resource_sas_from_share(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -104,7 +104,7 @@ def test_n_1kb_file_in_dir_upload_to_share(number_of_files): # execute azcopy command result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_share_url).\ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_n_1kb_file_upload failed while uploading ", number_of_files, " files to the share") return @@ -137,7 +137,7 @@ def test_n_1kb_file_in_dir_upload_to_azure_directory(number_of_files, recursive) # execute azcopy command result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(directory_url).\ - add_flags("recursive", recursive).add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", recursive).add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_n_1kb_file_upload failed while uploading ", number_of_files, " files to the share") return @@ -161,7 +161,7 @@ def test_metaData_content_encoding_content_type(): # execute azcopy upload command. destination_sas = util.get_resource_sas_from_share(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).\ - add_flags("Logging", "5").add_flags("recursive", "true").add_flags("metadata", "author=jiac;viewport=width;description=test file").\ + add_flags("Logging", "info").add_flags("recursive", "true").add_flags("metadata", "author=jiac;viewport=width;description=test file").\ add_flags("content-type", "testctype").add_flags("content-encoding", "testenc").add_flags("no-guess-mime-type", "true").execute_azcopy_copy_command() if not result: print("uploading 2KB file with metadata, content type and content-encoding failed") @@ -185,14 +185,14 @@ def test_guess_mime_type(): # execute azcopy upload of html file. destination_sas = util.get_resource_sas_from_share(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("uploading file ", filename, " failed") return # execute the validator to verify the content-type. - result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("testFile").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("recursive", "true") if not result: print("test_guess_mime_type test failed") @@ -208,7 +208,7 @@ def test_9mb_file_upload(): # execute azcopy copy upload. dest = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(dest)\ - .add_flags("Logging", "5").add_flags("block-size", "104857600").add_flags("recursive", "true").\ + .add_flags("Logging", "info").add_flags("block-size", "104857600").add_flags("recursive", "true").\ execute_azcopy_copy_command() if not result: print("failed uploading file", filename, " to the share") @@ -230,7 +230,7 @@ def test_1GB_file_upload(): # execute azcopy upload. destination_sas = util.get_resource_sas_from_share(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5"). \ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info"). \ add_flags("block-size", "104857600").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1G file", filename, " to the share") diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index f5530b8ff..fdd90e61f 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -14,7 +14,7 @@ def test_1kb_blob_upload(): src = file_path dest = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(src).add_arguments(dest).\ - add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1KB file to the container") return @@ -37,7 +37,7 @@ def test_63mb_blob_upload(): # execute azcopy copy upload. dest = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(dest)\ - .add_flags("Logging", "5").add_flags("block-size", "104857600").add_flags("recursive", "true").\ + .add_flags("Logging", "info").add_flags("block-size", "104857600").add_flags("recursive", "true").\ execute_azcopy_copy_command() if not result: print("failed uploading file", filename, " to the container") @@ -59,7 +59,7 @@ def test_n_1kb_blob_upload(number_of_files): # execute azcopy command result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url).\ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_n_1kb_blob_upload failed while uploading ", number_of_files, " files to the container") return @@ -83,7 +83,7 @@ def test_blob_metaData_content_encoding_content_type(): # execute azcopy upload command. destination_sas = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).\ - add_flags("Logging", "5").add_flags("recursive", "true").add_flags("metadata", "author=prjain;viewport=width;description=test file").\ + add_flags("Logging", "info").add_flags("recursive", "true").add_flags("metadata", "author=prjain;viewport=width;description=test file").\ add_flags("content-type", "testctype").add_flags("content-encoding", "testenc").add_flags("no-guess-mime-type", "true").execute_azcopy_copy_command() if not result: print("uploading 2KB file with metadata, content type and content-encoding failed") @@ -107,7 +107,7 @@ def test_1GB_blob_upload(): # execute azcopy upload. destination_sas = util.get_resource_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5"). \ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info"). \ add_flags("block-size", "104857600").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading 1G file", filename, " to the container") @@ -133,7 +133,7 @@ def test_block_size(block_size): # execute azcopy upload of 63 Mb file. destination_sas = util.get_resource_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5"). \ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info"). \ add_flags("block-size", str(block_size)).add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("failed uploading file", filename, " with block size 4MB to the container") @@ -160,14 +160,14 @@ def test_guess_mime_type(): # execute azcopy upload of html file. destination_sas = util.get_resource_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("uploading file ", filename, " failed") return # execute the validator to verify the content-type. - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("recursive", "true") if not result: print("test_guess_mime_type test failed") @@ -182,7 +182,7 @@ def test_set_block_blob_tier(): # uploading the file file_hot_block_blob_tier using azcopy and setting the block-blob-tier to Hot destination_sas = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("block-blob-tier", "Hot").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("block-blob-tier", "Hot").execute_azcopy_copy_command() if not result: print("uploading file with block-blob-tier set to Hot failed. ") return @@ -200,7 +200,7 @@ def test_set_block_blob_tier(): # uploading the file file_cool_block_blob_tier using azcopy and setting the block-blob-tier to Cool. destination_sas = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("block-blob-tier", "Cool").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("block-blob-tier", "Cool").execute_azcopy_copy_command() if not result: print("uploading file with block-blob-tier set to Cool failed.") return @@ -218,7 +218,7 @@ def test_set_block_blob_tier(): # uploading the file file_archive_block_blob_tier using azcopy and setting the block-blob-tier to Archive. destination_sas = util.get_resource_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("block-blob-tier", "archive").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("block-blob-tier", "archive").execute_azcopy_copy_command() if not result: print("uploading file with block-blob-tier set to Cool failed.") return @@ -230,13 +230,14 @@ def test_set_block_blob_tier(): return print("test_set_block_blob_tier successfully passed") + def test_force_flag_set_to_false_upload(): # creating directory with 20 files in it. dir_name = "dir_force_flag_set_upload" dir_n_files_path = util.create_test_n_files(1024, 20, dir_name) # uploading the directory with 20 files in it. result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_upload failed while uploading ", 20, "files in to dir_force_flag_set_upload the container") return @@ -249,12 +250,14 @@ def test_force_flag_set_to_false_upload(): # uploading the directory again with force flag set to false. result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("force", "false").add_flags("Logging", "5").\ + add_flags("recursive", "true").add_flags("force", "false").add_flags("Logging", "info").\ add_flags("output-json","true").execute_azcopy_copy_command_get_output() if not result: print("test_force_flag_set_to_false_upload failed while uploading ", 20, "files in to dir_force_flag_set_upload the container with force flag set to false") return + # parsing the json and comparing the number of failed and successful transfers. + result = util.parseAzcopyOutput(result) x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) if x.TransfersFailed is not 20 and x.TransfersCompleted is not 0 : print("test_force_flag_set_to_false_upload failed with difference in the number of failed and successful transfers") @@ -265,7 +268,7 @@ def test_force_flag_set_to_false_upload(): sub_dir_n_files_path = util.create_test_n_files(1024, 20, sub_dir_name) result = util.Command("copy").add_arguments(sub_dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_upload failed while uploading ", 20, "files in " + sub_dir_force_flag_set) return @@ -280,14 +283,14 @@ def test_force_flag_set_to_false_upload(): # removing the sub directory. result = util.Command("rm").add_arguments(sub_directory_resource_sas).\ - add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_upload failed removing ", sub_dir_n_files_path) return # uploading the directory again with force flag set to false. result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("force", "false").add_flags("Logging", "5"). \ + add_flags("recursive", "true").add_flags("force", "false").add_flags("Logging", "info"). \ add_flags("output-json","true").execute_azcopy_copy_command_get_output() if not result: print("test_force_flag_set_to_false_upload failed while uploading ", 20, "files in to dir_force_flag_set the container with force flag set to false") @@ -295,6 +298,7 @@ def test_force_flag_set_to_false_upload(): # parsing the json and comparing the number of failed and successful transfers. # Number of failed transfers should be 20 and number of successful transfer should be 20. + result = util.parseAzcopyOutput(result) x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) if x.TransfersFailed is not 20 and x.TransfersCompleted is not 20 : print("test_force_flag_set_to_false_upload failed with difference in the number of failed and successful transfers") @@ -307,7 +311,7 @@ def test_force_flag_set_to_false_download(): dir_n_files_path = util.create_test_n_files(1024, 20, dir_name) # uploading the directory with 20 files in it. result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("Logging", "5").execute_azcopy_copy_command() + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_download failed while uploading ", 20, "files in to dir_force_flag_set_download the container") return @@ -326,7 +330,7 @@ def test_force_flag_set_to_false_download(): return # downloading the directory created from container through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "5").add_flags("recursive", "true").execute_azcopy_copy_command() + result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "info").add_flags("recursive", "true").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_download failed downloading dir ", dir_name) return @@ -338,8 +342,9 @@ def test_force_flag_set_to_false_download(): return # downloading the directory created from container through azcopy with recursive flag to true and force flag set to false. - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "info").\ add_flags("recursive", "true").add_flags("force", "false").add_flags("output-json","true").execute_azcopy_copy_command_get_output() + result = util.parseAzcopyOutput(result) x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) if x.TransfersFailed is not 20 and x.TransfersCompleted is not 0 : print("test_force_flag_set_to_false_download failed with difference in the number of failed and successful transfers") @@ -356,8 +361,9 @@ def test_force_flag_set_to_false_download(): # downloading the directory created from container through azcopy with recursive flag to true and force flag set to false. # 5 deleted files should be downloaded. Number of failed transfer should be 15 and number of completed transfer should be 5 - result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "5"). \ + result = util.Command("copy").add_arguments(destination).add_arguments(util.test_directory_path).add_flags("Logging", "info"). \ add_flags("recursive", "true").add_flags("force", "false").add_flags("output-json","true").execute_azcopy_copy_command_get_output() + result = util.parseAzcopyOutput(result) x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) if x.TransfersFailed is not 15 and x.TransfersCompleted is not 5 : print("test_force_flag_set_to_false_download failed with difference in the number of failed and successful transfers") diff --git a/testSuite/scripts/test_upload_page_blob.py b/testSuite/scripts/test_upload_page_blob.py index 4a14e9135..b8309b00f 100644 --- a/testSuite/scripts/test_upload_page_blob.py +++ b/testSuite/scripts/test_upload_page_blob.py @@ -9,7 +9,7 @@ def test_page_blob_upload_1mb(): # execute azcopy upload. destination = util.get_resource_sas(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -31,7 +31,7 @@ def test_page_range_for_complete_sparse_file(): # execute azcopy page blob upload. destination_sas = util.get_resource_sas(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -54,7 +54,7 @@ def test_page_blob_upload_partial_sparse_file(): # execute azcopy pageblob upload. destination_sas = util.get_resource_sas(file_name) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "5").\ + result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas).add_flags("Logging", "info").\ add_flags("block-size","4194304").execute_azcopy_copy_command() if not result: print("uploading file ", file_name, " failed") @@ -78,7 +78,7 @@ def test_set_page_blob_tier(): destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P10").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P10 failed.") return @@ -95,7 +95,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P20").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P20 failed.") return @@ -112,7 +112,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P30").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P30 failed.") return @@ -129,7 +129,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P4").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P4 failed.") return @@ -146,7 +146,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P40").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P40 failed.") return @@ -163,7 +163,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P50").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P50 failed.") return @@ -180,7 +180,7 @@ def test_set_page_blob_tier(): file_path = util.create_test_file(filename, 100 * 1024) destination_sas = util.get_resource_sas_from_premium_container_sas(filename) result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("Logging", "5").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() + add_flags("Logging", "info").add_flags("page-blob-tier", "P6").execute_azcopy_copy_command() if not result: print("uploading file with page-blob-tier set to P6 failed.") return diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index 3355ad727..15b2a79d2 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -376,4 +376,44 @@ def get_suite_executable_name(): return "testSuite.exe" if osType == "LINUX": return "testSuite" - return "" \ No newline at end of file + return "" + +# parseAzcopyOutput parses the Azcopy Output in JSON format to give the final Azcopy Output in JSON Format +# Final Azcopy Output is the last JobSummary for the Job +# Azcopy Output can have more than one Summary for the Job +# parseAzcopyOutput returns the final JobSummary in JSON format. +def parseAzcopyOutput(s): + count = 0 + output = "" + final_output = "" + # Split the lines + lines = s.split('\n') + # Iterating through the output in reverse order since last summary has to be considered. + # Increment the count when line is "}" + # Reduce the count when line is "{" + # append the line to final output + # When the count is 0, it means the last Summary has been traversed + for line in reversed(lines): + # If the line is empty, then continue + if line == "": + continue + elif line is '}': + count = count + 1 + elif line is "{": + count = count - 1 + if count >= 0: + if len(output) > 0: + output = output + '\n' + line + else: + output = line + if count == 0: + break + lines = output.split('\n') + # Since the lines were iterated in reverse order revering them again and + # concatenating the lines to get the final JobSummary + for line in reversed(lines): + if len(final_output) > 0 : + final_output = final_output + '\n' + line + else: + final_output = line + return final_output \ No newline at end of file From e2671e7272f3a38fb98a7b20382e8d96babae7bd Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Tue, 15 May 2018 11:42:43 -0700 Subject: [PATCH 05/23] enum added for log level --- common/fe-ste-models.go | 49 ++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 19c787e8f..c9078933e 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -28,8 +28,6 @@ import ( "sync/atomic" "time" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" - "fmt" - "strings" ) //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -80,37 +78,28 @@ type Status uint32 //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -type LogLevel byte +type LogLevel uint8 + +var ELogLevel = LogLevel(0) + +func (LogLevel) None() LogLevel { return LogLevel(0)} +func (LogLevel) Fatal() LogLevel { return LogLevel(1)} +func (LogLevel) Panic() LogLevel { return LogLevel(2)} +func (LogLevel) Error() LogLevel { return LogLevel(3)} +func (LogLevel) Warn() LogLevel { return LogLevel(4)} +func (LogLevel) Info() LogLevel { return LogLevel(5)} +func (LogLevel) Debug() LogLevel { return LogLevel(6)} func (ll *LogLevel) Parse(s string) error { - // converting string s to all lower case string - // to avoid case sensitiveness - s = strings.ToLower(s) - switch s { - case "none": - *ll = 0 - return nil - case "fatal": - *ll = 1 - return nil - case "error": - *ll = 2 - return nil - case "panic": - *ll = 3 - return nil - case "warning": - *ll = 4 - return nil - case "info": - *ll = 5 - return nil - case "debug": - *ll = 6 - return nil - default: - return fmt.Errorf("invaild log type %s passed. Azcopy supports none, fatal, error, panic, warning, info and debug log levels",s) + val, err := EnumHelper{}.Parse(reflect.TypeOf(ll), s, true) + if err == nil { + *ll = val.(LogLevel) } + return err +} + +func (ll LogLevel) String() string { + return EnumHelper{}.StringInteger(ll, reflect.TypeOf(ll)) } func (ll LogLevel) ToPipelineLogLevel() pipeline.LogLevel { From d94e0d59731ea54bd07c6fe453a93c65d1b510de Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Tue, 15 May 2018 13:11:00 -0700 Subject: [PATCH 06/23] fixed the test_force_flag_set_to_false_upload test case --- testSuite/scripts/test_upload_block_blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index fdd90e61f..957a594ec 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -267,7 +267,7 @@ def test_force_flag_set_to_false_upload(): sub_dir_name = os.path.join(dir_name + "/sub_dir_force_flag_set_upload") sub_dir_n_files_path = util.create_test_n_files(1024, 20, sub_dir_name) - result = util.Command("copy").add_arguments(sub_dir_n_files_path).add_arguments(util.test_container_url). \ + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() if not result: print("test_force_flag_set_to_false_upload failed while uploading ", 20, "files in " + sub_dir_force_flag_set) From d7c30dc31af258b71933bec5c82efc1de8e73fab Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Wed, 16 May 2018 14:39:15 -0700 Subject: [PATCH 07/23] temp front-end changes --- cmd/copyDownloadBlobEnumerator.go | 240 +++++++++++++++++++++++++++++- cmd/copyUtil.go | 62 +++++++- 2 files changed, 289 insertions(+), 13 deletions(-) diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index 79146a998..ab5794725 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -15,6 +15,233 @@ import ( type copyDownloadBlobEnumerator common.CopyJobPartOrderRequest +func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecursiveOn bool, destinationPath string, + wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + util := copyHandlerUtil{} + + p := azblob.NewPipeline( + azblob.NewAnonymousCredential(), + azblob.PipelineOptions{ + Retry: azblob.RetryOptions{ + Policy: azblob.RetryPolicyExponential, + MaxTries: ste.UploadMaxTries, + TryTimeout: ste.UploadTryTimeout, + RetryDelay: ste.UploadRetryDelay, + MaxRetryDelay: ste.UploadMaxRetryDelay, + }, + }) + + // attempt to parse the source url + sourceUrl, err := url.Parse(sourceUrlString) + if err != nil { + return errors.New("cannot parse source URL") + } + + // get the blob parts + blobUrlParts := azblob.NewBlobURLParts(*sourceUrl) + + // first check if source blob exists + blobUrl := azblob.NewBlobURL(*sourceUrl, p) + blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + + // for a single blob, the destination can either be a file or a directory + var singleBlobDestinationPath string + if util.isPathDirectory(destinationPath) { + blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) + // check for special characters and get blobName without special character. + blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) + singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) + } else { + singleBlobDestinationPath = destinationPath + } + + // if the single blob exists, download it + if err == nil { + e.addTransfer(common.CopyTransfer{ + Source: sourceUrl.String(), + Destination: singleBlobDestinationPath, + LastModifiedTime: blobProperties.LastModified(), + SourceSize: blobProperties.ContentLength(), + }, wg, waitUntilJobCompletion) + + return nil + } + + // check if the given url is a container + if util.urlIsContainerOrShare(sourceUrl) { + // if path ends with a /, then append * + if strings.HasSuffix(sourceUrl.Path, "/") { + sourceUrl.Path += "*" + } else { // if path is just /container_name, '/*' needs to be appended + sourceUrl.Path += "/*" + } + } + + literalContainerUrl := util.getContainerUrl(blobUrlParts) + containerUrl := azblob.NewContainerURL(literalContainerUrl, p) + + searchPrefix := util. + numOfStarInUrlPath := util.numOfStarInUrl(sourceUrl.Path) + if numOfStarInUrlPath == 1 { // prefix search + + // the * must be at the end of the path + if strings.LastIndex(sourceUrl.Path, "*") != len(sourceUrl.Path)-1 { + return errors.New("the * in the source URL must be at the end of the path") + } + + // the destination must be a directory, otherwise we don't know where to put the files + if !util.isPathDirectory(destinationPath) { + return errors.New("the destination must be an existing directory in this download scenario") + } + + // get the search prefix to query the service + searchPrefix := util.blobNameFromUrl(blobUrlParts) + searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end + + closestVirtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) + + // strip away the leading / in the closest virtual directory + if len(closestVirtualDirectory) > 0 && closestVirtualDirectory[0:1] == "/" { + closestVirtualDirectory = closestVirtualDirectory[1:] + } + + // perform a list blob + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix + listBlob, err := containerUrl.ListBlobsFlatSegment(context.TODO(), marker, + azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) + } + + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If the blob is not valid as per the conditions mentioned in the + // api isBlobValid, then skip the blob. + if !util.isBlobValid(blobInfo) { + continue + } + blobNameAfterPrefix := blobInfo.Name[len(closestVirtualDirectory):] + if !isRecursiveOn && strings.Contains(blobNameAfterPrefix, "/") { + continue + } + + // check for special characters and get the blob without special characters. + blobNameAfterPrefix = util.blobPathWOSpecialCharacters(blobNameAfterPrefix) + e.addTransfer(common.CopyTransfer{ + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), + Destination: util.generateLocalPath(destinationPath, blobNameAfterPrefix), + LastModifiedTime: blobInfo.Properties.LastModified, + SourceSize: *blobInfo.Properties.ContentLength}, + wg, waitUntilJobCompletion) + } + + marker = listBlob.NextMarker + if err != nil { + return err + } + } + + err = e.dispatchFinalPart() + if err != nil { + return err + } + + } else if numOfStarInUrlPath == 0 { // no prefix search + // see if source blob exists + blobUrl := azblob.NewBlobURL(*sourceUrl, p) + blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + + // for a single blob, the destination can either be a file or a directory + var singleBlobDestinationPath string + if util.isPathDirectory(destinationPath) { + blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) + // check for special characters and get blobName without special character. + blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) + singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) + } else { + singleBlobDestinationPath = destinationPath + } + + // if the single blob exists, download it + if err == nil { + e.addTransfer(common.CopyTransfer{ + Source: sourceUrl.String(), + Destination: singleBlobDestinationPath, + LastModifiedTime: blobProperties.LastModified(), + SourceSize: blobProperties.ContentLength(), + }, wg, waitUntilJobCompletion) + + //err = e.dispatchPart(false) + if err != nil { + return err + } + } else if err != nil && !isRecursiveOn { + return errors.New("cannot get source blob properties, make sure it exists, for virtual directory download please use --recursive") + } + + // if recursive happens to be turned on, then we will attempt to download a virtual directory + if isRecursiveOn { + // recursively download everything that is under the given path, that is a virtual directory + searchPrefix := util.blobNameFromUrl(blobUrlParts) + + // if the user did not specify / at the end of the virtual directory, add it before doing the prefix search + if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 { + searchPrefix += "/" + } + + // the destination must be a directory, otherwise we don't know where to put the files + if !util.isPathDirectory(destinationPath) { + return errors.New("the destination must be an existing directory in this download scenario") + } + + // perform a list blob + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up + listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, + azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) + } + + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If the blob is not valid as per the conditions mentioned in the + // api isBlobValid, then skip the blob. + if !util.isBlobValid(blobInfo) { + continue + } + blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") + // check for the special character in blob relative path and get path without special character. + blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) + e.addTransfer(common.CopyTransfer{ + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), + Destination: util.generateLocalPath(destinationPath, blobRelativePath), + LastModifiedTime: blobInfo.Properties.LastModified, + SourceSize: *blobInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) + } + + marker = listBlob.NextMarker + //err = e.dispatchPart(false) + if err != nil { + return err + } + } + } + + err = e.dispatchFinalPart() + if err != nil { + return err + } + + } else { // more than one * is not supported + return errors.New("only one * is allowed in the source URL") + } + return nil +} + // this function accepts a url (with or without *) to blobs for download and processes them func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { @@ -49,7 +276,8 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi } // get the container url to be used later for listing - literalContainerUrl := util.getContainerURLFromString(*sourceUrl) + blobUrlParts := azblob.NewBlobURLParts(*sourceUrl) + literalContainerUrl := util.getContainerUrl(blobUrlParts) containerUrl := azblob.NewContainerURL(literalContainerUrl, p) numOfStarInUrlPath := util.numOfStarInUrl(sourceUrl.Path) @@ -66,7 +294,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi } // get the search prefix to query the service - searchPrefix := util.getBlobNameFromURL(sourceUrl.Path) + searchPrefix := util.blobNameFromUrl(blobUrlParts) searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end closestVirtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) @@ -100,7 +328,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // check for special characters and get the blob without special characters. blobNameAfterPrefix = util.blobPathWOSpecialCharacters(blobNameAfterPrefix) e.addTransfer(common.CopyTransfer{ - Source: util.generateBlobUrl(literalContainerUrl, blobInfo.Name), + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), Destination: util.generateLocalPath(destinationPath, blobNameAfterPrefix), LastModifiedTime: blobInfo.Properties.LastModified, SourceSize: *blobInfo.Properties.ContentLength}, @@ -126,7 +354,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // for a single blob, the destination can either be a file or a directory var singleBlobDestinationPath string if util.isPathDirectory(destinationPath) { - blobNameFromUrl := util.getBlobNameFromURL(sourceUrl.Path) + blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) // check for special characters and get blobName without special character. blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) @@ -154,7 +382,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // if recursive happens to be turned on, then we will attempt to download a virtual directory if isRecursiveOn { // recursively download everything that is under the given path, that is a virtual directory - searchPrefix := util.getBlobNameFromURL(sourceUrl.Path) + searchPrefix := util.blobNameFromUrl(blobUrlParts) // if the user did not specify / at the end of the virtual directory, add it before doing the prefix search if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 { @@ -186,7 +414,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // check for the special character in blob relative path and get path without special character. blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) e.addTransfer(common.CopyTransfer{ - Source: util.generateBlobUrl(literalContainerUrl, blobInfo.Name), + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), Destination: util.generateLocalPath(destinationPath, blobRelativePath), LastModifiedTime: blobInfo.Properties.LastModified, SourceSize: *blobInfo.Properties.ContentLength}, diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 02bcd023c..8660bd189 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -37,6 +37,8 @@ import ( "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" + "mime/multipart" + "math" ) const ( @@ -156,13 +158,55 @@ func (util copyHandlerUtil) getDirNameFromSource(path string) (sourcePathWithout return } +func (util copyHandlerUtil) firstIndexOfWildCard(name string) int{ + sIndex := strings.Index(name, "*") + if sIndex == -1 { + sIndex = math.MaxInt64 + } + qIndex := strings.Index(name, "?") + if qIndex == -1 { + qIndex = math.MaxInt64 + } + obIndex := strings.Index(name, "[") + if obIndex == -1 { + obIndex = math.MaxInt64 + } + cbIndex := strings.Index(name, "]") + if (cbIndex == -1){ + cbIndex = math.MaxInt64 + } + return math.Min(math.Min(math.Min(sIndex, qIndex), obIndex), cbIndex) +} func (util copyHandlerUtil) getContainerURLFromString(url url.URL) url.URL { - //blobParts := azblob.NewBlobURLParts(url) - //blobParts.BlobName = "" - //return blobParts.URL() - containerName := strings.SplitAfterN(url.Path[1:], "/", 2)[0] - url.Path = "/" + containerName - return url + blobParts := azblob.NewBlobURLParts(url) + blobParts.BlobName = "" + return blobParts.URL() + //containerName := strings.SplitAfterN(url.Path[1:], "/", 2)[0] + //url.Path = "/" + containerName + //return url +} + +func (util copyHandlerUtil) getContainerUrl(blobParts azblob.BlobURLParts) url.URL { + blobParts.BlobName = "" + return blobParts.URL() +} + +func (util copyHandlerUtil) blobNameFromUrl(blobParts azblob.BlobURLParts) string { + return blobParts.BlobName +} + +func (util copyHandlerUtil) createBlobUrlFromContainer(blobUrlParts azblob.BlobURLParts, blobName string) string { + blobUrlParts.BlobName = blobName + blobUrl := blobUrlParts.URL() + return blobUrl.String() +} + +func (util copyHandlerUtil) searchPrefixFromUrl(parts azblob.BlobURLParts) (prefix, pattern string){ + if parts.BlobName == "" { + return + } + blobName := parts.BlobName + } func (util copyHandlerUtil) getConatinerUrlAndSuffix(url url.URL) (containerUrl, suffix string) { @@ -177,7 +221,11 @@ func (util copyHandlerUtil) getConatinerUrlAndSuffix(url url.URL) (containerUrl, } func (util copyHandlerUtil) generateBlobUrl(containerUrl url.URL, blobName string) string { - containerUrl.Path = containerUrl.Path + blobName + if containerUrl.Path[len(containerUrl.Path)-1] != '/'{ + containerUrl.Path = containerUrl.Path + "/"+ blobName + }else{ + containerUrl.Path = containerUrl.Path + blobName + } return containerUrl.String() } From 9c896d125d33c62b7eee983b527d25043c097680 Mon Sep 17 00:00:00 2001 From: prjain-msft Date: Thu, 17 May 2018 11:13:38 -0700 Subject: [PATCH 08/23] readstdinforcancel changes; log enum changes; changes in copy download enumeration --- cmd/cancel.go | 2 +- cmd/copyDownloadBlobEnumerator.go | 220 +++++++----------------------- cmd/copyUtil.go | 45 +++--- common/fe-ste-models.go | 18 +-- 4 files changed, 83 insertions(+), 202 deletions(-) diff --git a/cmd/cancel.go b/cmd/cancel.go index fb8d7dce1..7171657b6 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -105,7 +105,7 @@ func init() { // ReadStandardInputToCancelJob is a function that reads the standard Input // If Input given is "cancel", it cancels the current job. -func ReadStandardInputToCancelJob() { +func ReadStandardInputToCancelJob(cancelChannel chan <- os.Signal) { for { consoleReader := bufio.NewReader(os.Stdin) // ReadString reads input until the first occurrence of \n in the input, diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index ab5794725..7af34cbac 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" + "path/filepath" ) type copyDownloadBlobEnumerator common.CopyJobPartOrderRequest @@ -44,19 +45,21 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs blobUrl := azblob.NewBlobURL(*sourceUrl, p) blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) - // for a single blob, the destination can either be a file or a directory - var singleBlobDestinationPath string - if util.isPathDirectory(destinationPath) { - blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) - // check for special characters and get blobName without special character. - blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) - singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) - } else { - singleBlobDestinationPath = destinationPath - } - + //TODO: Examples // if the single blob exists, download it if err == nil { + + // for a single blob, the destination can either be a file or a directory + var singleBlobDestinationPath string + if util.isPathDirectory(destinationPath) { + blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) + // check for special characters and get blobName without special character. + blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) + singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) + } else { + singleBlobDestinationPath = destinationPath + } + e.addTransfer(common.CopyTransfer{ Source: sourceUrl.String(), Destination: singleBlobDestinationPath, @@ -67,177 +70,60 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs return nil } - // check if the given url is a container - if util.urlIsContainerOrShare(sourceUrl) { - // if path ends with a /, then append * - if strings.HasSuffix(sourceUrl.Path, "/") { - sourceUrl.Path += "*" - } else { // if path is just /container_name, '/*' needs to be appended - sourceUrl.Path += "/*" - } + + // the destination must be a directory, otherwise we don't know where to put the files + if !util.isPathDirectory(destinationPath) { + return errors.New("the destination must be an existing directory in this download scenario") } literalContainerUrl := util.getContainerUrl(blobUrlParts) containerUrl := azblob.NewContainerURL(literalContainerUrl, p) - searchPrefix := util. - numOfStarInUrlPath := util.numOfStarInUrl(sourceUrl.Path) - if numOfStarInUrlPath == 1 { // prefix search - - // the * must be at the end of the path - if strings.LastIndex(sourceUrl.Path, "*") != len(sourceUrl.Path)-1 { - return errors.New("the * in the source URL must be at the end of the path") - } - - // the destination must be a directory, otherwise we don't know where to put the files - if !util.isPathDirectory(destinationPath) { - return errors.New("the destination must be an existing directory in this download scenario") - } - - // get the search prefix to query the service - searchPrefix := util.blobNameFromUrl(blobUrlParts) - searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end - - closestVirtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) - - // strip away the leading / in the closest virtual directory - if len(closestVirtualDirectory) > 0 && closestVirtualDirectory[0:1] == "/" { - closestVirtualDirectory = closestVirtualDirectory[1:] - } + searchPrefix, blobNamePattern := util.searchPrefixFromUrl(blobUrlParts) - // perform a list blob - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerUrl.ListBlobsFlatSegment(context.TODO(), marker, - azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Blobs.Blob { - // If the blob is not valid as per the conditions mentioned in the - // api isBlobValid, then skip the blob. - if !util.isBlobValid(blobInfo) { - continue - } - blobNameAfterPrefix := blobInfo.Name[len(closestVirtualDirectory):] - if !isRecursiveOn && strings.Contains(blobNameAfterPrefix, "/") { - continue - } - - // check for special characters and get the blob without special characters. - blobNameAfterPrefix = util.blobPathWOSpecialCharacters(blobNameAfterPrefix) - e.addTransfer(common.CopyTransfer{ - Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), - Destination: util.generateLocalPath(destinationPath, blobNameAfterPrefix), - LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, - wg, waitUntilJobCompletion) - } - - marker = listBlob.NextMarker - if err != nil { - return err - } - } - - err = e.dispatchFinalPart() + // perform a list blob + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up + listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, + azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) if err != nil { - return err + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) } - } else if numOfStarInUrlPath == 0 { // no prefix search - // see if source blob exists - blobUrl := azblob.NewBlobURL(*sourceUrl, p) - blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) - - // for a single blob, the destination can either be a file or a directory - var singleBlobDestinationPath string - if util.isPathDirectory(destinationPath) { - blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) - // check for special characters and get blobName without special character. - blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) - singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) - } else { - singleBlobDestinationPath = destinationPath - } - - // if the single blob exists, download it - if err == nil { - e.addTransfer(common.CopyTransfer{ - Source: sourceUrl.String(), - Destination: singleBlobDestinationPath, - LastModifiedTime: blobProperties.LastModified(), - SourceSize: blobProperties.ContentLength(), - }, wg, waitUntilJobCompletion) - - //err = e.dispatchPart(false) - if err != nil { - return err + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If the blob is not valid as per the conditions mentioned in the + // api doesBlobRepresentAFolder, then skip the blob. + if util.doesBlobRepresentAFolder(blobInfo) { + continue } - } else if err != nil && !isRecursiveOn { - return errors.New("cannot get source blob properties, make sure it exists, for virtual directory download please use --recursive") - } - - // if recursive happens to be turned on, then we will attempt to download a virtual directory - if isRecursiveOn { - // recursively download everything that is under the given path, that is a virtual directory - searchPrefix := util.blobNameFromUrl(blobUrlParts) - - // if the user did not specify / at the end of the virtual directory, add it before doing the prefix search - if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 { - searchPrefix += "/" + // TODO: add a function to perform the match + matched, err := filepath.Match(blobNamePattern, blobInfo.Name) + if err != nil { + panic(err) } - - // the destination must be a directory, otherwise we don't know where to put the files - if !util.isPathDirectory(destinationPath) { - return errors.New("the destination must be an existing directory in this download scenario") + if !matched { + continue } - // perform a list blob - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up - listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, - azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Blobs.Blob { - // If the blob is not valid as per the conditions mentioned in the - // api isBlobValid, then skip the blob. - if !util.isBlobValid(blobInfo) { - continue - } - blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") - // check for the special character in blob relative path and get path without special character. - blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) - e.addTransfer(common.CopyTransfer{ - Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), - Destination: util.generateLocalPath(destinationPath, blobRelativePath), - LastModifiedTime: blobInfo.Properties.LastModified, - SourceSize: *blobInfo.Properties.ContentLength}, - wg, - waitUntilJobCompletion) - } - - marker = listBlob.NextMarker - //err = e.dispatchPart(false) - if err != nil { - return err - } - } + blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") + // check for the special character in blob relative path and get path without special character. + blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) + e.addTransfer(common.CopyTransfer{ + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), + Destination: util.generateLocalPath(destinationPath, blobRelativePath), + LastModifiedTime: blobInfo.Properties.LastModified, + SourceSize: *blobInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) } - err = e.dispatchFinalPart() + marker = listBlob.NextMarker + //err = e.dispatchPart(false) if err != nil { return err } - - } else { // more than one * is not supported - return errors.New("only one * is allowed in the source URL") } return nil } @@ -316,8 +202,8 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) for _, blobInfo := range listBlob.Blobs.Blob { // If the blob is not valid as per the conditions mentioned in the - // api isBlobValid, then skip the blob. - if !util.isBlobValid(blobInfo) { + // api doesBlobRepresentAFolder, then skip the blob. + if !util.doesBlobRepresentAFolder(blobInfo) { continue } blobNameAfterPrefix := blobInfo.Name[len(closestVirtualDirectory):] @@ -406,8 +292,8 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) for _, blobInfo := range listBlob.Blobs.Blob { // If the blob is not valid as per the conditions mentioned in the - // api isBlobValid, then skip the blob. - if !util.isBlobValid(blobInfo) { + // api doesBlobRepresentAFolder, then skip the blob. + if !util.doesBlobRepresentAFolder(blobInfo) { continue } blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 8660bd189..c39a51567 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -159,23 +159,7 @@ func (util copyHandlerUtil) getDirNameFromSource(path string) (sourcePathWithout } func (util copyHandlerUtil) firstIndexOfWildCard(name string) int{ - sIndex := strings.Index(name, "*") - if sIndex == -1 { - sIndex = math.MaxInt64 - } - qIndex := strings.Index(name, "?") - if qIndex == -1 { - qIndex = math.MaxInt64 - } - obIndex := strings.Index(name, "[") - if obIndex == -1 { - obIndex = math.MaxInt64 - } - cbIndex := strings.Index(name, "]") - if (cbIndex == -1){ - cbIndex = math.MaxInt64 - } - return math.Min(math.Min(math.Min(sIndex, qIndex), obIndex), cbIndex) + return strings.Index(name, "*") } func (util copyHandlerUtil) getContainerURLFromString(url url.URL) url.URL { blobParts := azblob.NewBlobURLParts(url) @@ -203,10 +187,24 @@ func (util copyHandlerUtil) createBlobUrlFromContainer(blobUrlParts azblob.BlobU func (util copyHandlerUtil) searchPrefixFromUrl(parts azblob.BlobURLParts) (prefix, pattern string){ if parts.BlobName == "" { + pattern = "*" return } - blobName := parts.BlobName - + wildCardIndex := util.firstIndexOfWildCard(parts.BlobName) + if wildCardIndex < 0 { // no wildcard exists + prefix = parts.BlobName + // check for separator at the end of virtual directory + if prefix[len(prefix)-1] != '/'{ + prefix += "/" + } + pattern = "*" + return + } + // wild card exists prefix will be the content of blob name till the wildcard index + //TODO: example add + prefix = parts.BlobName[:wildCardIndex] + pattern = parts.BlobName + return } func (util copyHandlerUtil) getConatinerUrlAndSuffix(url url.URL) (containerUrl, suffix string) { @@ -303,17 +301,14 @@ func (util copyHandlerUtil) blobPathWOSpecialCharacters(blobPath string) string return bnwc } -// isBlobValid verifies whether blob is valid or not. +// doesBlobRepresentAFolder verifies whether blob is valid or not. // Used to handle special scenarios or conditions. -func (util copyHandlerUtil) isBlobValid(bInfo azblob.Blob) bool { +func (util copyHandlerUtil) doesBlobRepresentAFolder(bInfo azblob.Blob) bool { // this condition is to handle the WASB V1 directory structure. // HDFS driver creates a blob for the empty directories (let’s call it ‘myfolder’) // and names all the blobs under ‘myfolder’ as such: ‘myfolder/myblob’ // The empty directory has meta-data 'hdi_isfolder = true' - if bInfo.Metadata["hdi_isfolder"] == "true" { - return false - } - return true + return bInfo.Metadata["hdi_isfolder"] == "true" } func (copyHandlerUtil) fetchJobStatus(jobID common.JobID, startTime *time.Time, bytesTransferredInLastInterval *uint64, outputJson bool) common.JobStatus { diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index c9078933e..396e6e5f0 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -80,15 +80,15 @@ type Status uint32 type LogLevel uint8 -var ELogLevel = LogLevel(0) - -func (LogLevel) None() LogLevel { return LogLevel(0)} -func (LogLevel) Fatal() LogLevel { return LogLevel(1)} -func (LogLevel) Panic() LogLevel { return LogLevel(2)} -func (LogLevel) Error() LogLevel { return LogLevel(3)} -func (LogLevel) Warn() LogLevel { return LogLevel(4)} -func (LogLevel) Info() LogLevel { return LogLevel(5)} -func (LogLevel) Debug() LogLevel { return LogLevel(6)} +var ELogLevel = LogLevel(pipeline.LogNone) + +func (LogLevel) None() LogLevel { return LogLevel(pipeline.LogNone)} +func (LogLevel) Fatal() LogLevel { return LogLevel(pipeline.LogFatal)} +func (LogLevel) Panic() LogLevel { return LogLevel(pipeline.LogPanic)} +func (LogLevel) Error() LogLevel { return LogLevel(pipeline.LogError)} +func (LogLevel) Warning() LogLevel { return LogLevel(pipeline.LogWarning)} +func (LogLevel) Info() LogLevel { return LogLevel(pipeline.LogInfo)} +func (LogLevel) Debug() LogLevel { return LogLevel(pipeline.LogDebug)} func (ll *LogLevel) Parse(s string) error { val, err := EnumHelper{}.Parse(reflect.TypeOf(ll), s, true) From f34115d9801b1bfcdec7316f08d61a78f2cefeb3 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 17 May 2018 11:21:16 -0700 Subject: [PATCH 09/23] fixed compilation errors --- cmd/cancel.go | 2 +- cmd/copy.go | 8 ++++---- cmd/copyUtil.go | 2 -- cmd/resume.go | 8 ++++---- cmd/sync.go | 8 ++++---- main.go | 2 +- 6 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cmd/cancel.go b/cmd/cancel.go index 7171657b6..0f185fb28 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -33,7 +33,7 @@ import ( // created a signal channel to receive the Interrupt and Kill signal send to OS // this channel is shared by copy, resume, sync and an independent go routine reading stdin // for cancel command -var cancelChannel = make(chan os.Signal, 1) +var CancelChannel = make(chan os.Signal, 1) type rawCancelCmdArgs struct { jobID string diff --git a/cmd/copy.go b/cmd/copy.go index 80555cb68..c03270db9 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -433,16 +433,16 @@ func (cca cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { func (cca cookedCopyCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync.WaitGroup) { - // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals - signal.Notify(cancelChannel, os.Interrupt, os.Kill) + // CancelChannel will be notified when os receives os.Interrupt and os.Kill signals + signal.Notify(CancelChannel, os.Interrupt, os.Kill) - // waiting for signals from either cancelChannel or timeOut Channel. + // waiting for signals from either CancelChannel or timeOut Channel. // if no signal received, will fetch/display a job status update then sleep for a bit startTime := time.Now() bytesTransferredInLastInterval := uint64(0) for { select { - case <-cancelChannel: + case <-CancelChannel: fmt.Println("Cancelling Job") cookedCancelCmdArgs{jobID: jobID}.process() os.Exit(1) diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index c39a51567..11ce90add 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -37,8 +37,6 @@ import ( "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" - "mime/multipart" - "math" ) const ( diff --git a/cmd/resume.go b/cmd/resume.go index 89f793fc1..7e2269d11 100644 --- a/cmd/resume.go +++ b/cmd/resume.go @@ -59,16 +59,16 @@ func init() { func waitUntilJobCompletion(jobID common.JobID) { - // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals - signal.Notify(cancelChannel, os.Interrupt, os.Kill) + // CancelChannel will be notified when os receives os.Interrupt and os.Kill signals + signal.Notify(CancelChannel, os.Interrupt, os.Kill) - // waiting for signals from either cancelChannel or timeOut Channel. + // waiting for signals from either CancelChannel or timeOut Channel. // if no signal received, will fetch/display a job status update then sleep for a bit startTime := time.Now() bytesTransferredInLastInterval := uint64(0) for { select { - case <-cancelChannel: + case <-CancelChannel: fmt.Println("Cancelling Job") cookedCancelCmdArgs{jobID: jobID}.process() os.Exit(1) diff --git a/cmd/sync.go b/cmd/sync.go index 173924d48..e475e801d 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -104,16 +104,16 @@ func (cca cookedSyncCmdArgs) process() (err error) { func (cca cookedSyncCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync.WaitGroup) { - // cancelChannel will be notified when os receives os.Interrupt and os.Kill signals - signal.Notify(cancelChannel, os.Interrupt, os.Kill) + // CancelChannel will be notified when os receives os.Interrupt and os.Kill signals + signal.Notify(CancelChannel, os.Interrupt, os.Kill) - // waiting for signals from either cancelChannel or timeOut Channel. + // waiting for signals from either CancelChannel or timeOut Channel. // if no signal received, will fetch/display a job status update then sleep for a bit startTime := time.Now() bytesTransferredInLastInterval := uint64(0) for { select { - case <-cancelChannel: + case <-CancelChannel: fmt.Println("Cancelling Job") cookedCancelCmdArgs{jobID: jobID}.process() os.Exit(1) diff --git a/main.go b/main.go index 1e8a8b555..ed0177eee 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,7 @@ func mainWithExitCode() exitCode { cmd.Execute() return eexitCode.success() } - go cmd.ReadStandardInputToCancelJob() + go cmd.ReadStandardInputToCancelJob(cmd.CancelChannel) azcopyAppPathFolder := GetAzCopyAppPath() go ste.MainSTE(300, 500, azcopyAppPathFolder) cmd.Execute() From d16493ccd94b0d4277cfaf2962d3e9b42d69b708 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 17 May 2018 17:06:20 -0700 Subject: [PATCH 10/23] code refactored in copydownloadblobenumerator --- cmd/copyDownloadBlobEnumerator.go | 83 +++++++++++++++++++------------ cmd/copyDownloadFileEnumerator.go | 6 +-- cmd/copyUtil.go | 38 +++++++++++--- cmd/removeFileEnumerator.go | 2 +- 4 files changed, 85 insertions(+), 44 deletions(-) diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index 7af34cbac..19c04e827 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -11,15 +11,15 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" - "path/filepath" ) type copyDownloadBlobEnumerator common.CopyJobPartOrderRequest -func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecursiveOn bool, destinationPath string, +func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { util := copyHandlerUtil{} + // Create Pipeline to Get the Blob Properties or List Blob Segment p := azblob.NewPipeline( azblob.NewAnonymousCredential(), azblob.PipelineOptions{ @@ -41,48 +41,70 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs // get the blob parts blobUrlParts := azblob.NewBlobURLParts(*sourceUrl) - // first check if source blob exists + // First Check if source blob exists + // This check is in place to avoid listing of the blobs and matching the given blob against it + // For example given source is https:///a? and there exists other blobs aa and aab + // Listing the blobs with prefix /a will list other blob as well blobUrl := azblob.NewBlobURL(*sourceUrl, p) blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) - //TODO: Examples - // if the single blob exists, download it + // If the source blob exists, then queue transfer and return + // Example: https:///? if err == nil { - - // for a single blob, the destination can either be a file or a directory - var singleBlobDestinationPath string - if util.isPathDirectory(destinationPath) { + // For a single blob, destination provided can be either a directory or file. + // If the destination is directory, then name of blob is preserved + // If the destination is file, then blob will be downloaded as the given file name + // Example1: Downloading https:///a? to directory C:\\Users\\User1 + // will download the blob as C:\\Users\\User1\\a + // Example2: Downloading https:///a? to directory C:\\Users\\User1\\b + // (b is not a directory) will download blob as C:\\Users\\User1\\b + var blobLocalPath string + if util.isPathALocalDirectory(destinationPath) { blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) // check for special characters and get blobName without special character. blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) - singleBlobDestinationPath = util.generateLocalPath(destinationPath, blobNameFromUrl) + blobLocalPath = util.generateLocalPath(destinationPath, blobNameFromUrl) } else { - singleBlobDestinationPath = destinationPath + blobLocalPath = destinationPath } - + // Add the transfer to CopyJobPartOrderRequest e.addTransfer(common.CopyTransfer{ Source: sourceUrl.String(), - Destination: singleBlobDestinationPath, + Destination: blobLocalPath, LastModifiedTime: blobProperties.LastModified(), SourceSize: blobProperties.ContentLength(), }, wg, waitUntilJobCompletion) - + // only one transfer for this Job, dispatch the JobPart + err := e.dispatchFinalPart() + if err != nil{ + return err + } return nil } - - // the destination must be a directory, otherwise we don't know where to put the files - if !util.isPathDirectory(destinationPath) { + // Since the given source url doesn't represent an existing blob + // it is either a container or a virtual directory, so it need to be + // downloaded to an existing directory + // Check if the given destination path is a directory or not. + if !util.isPathALocalDirectory(destinationPath) { return errors.New("the destination must be an existing directory in this download scenario") } literalContainerUrl := util.getContainerUrl(blobUrlParts) containerUrl := azblob.NewContainerURL(literalContainerUrl, p) + // searchPrefix is the used in listing blob inside a container + // all the blob listed should have the searchPrefix as the prefix + // blobNamePattern represents the regular expression which the blobName should Match searchPrefix, blobNamePattern := util.searchPrefixFromUrl(blobUrlParts) - - // perform a list blob + // If blobNamePattern is "*", means that all the contents inside the given source url needs to be downloaded + // It means that source url provided is either a container or a virtual directory + // All the blobs inside a container or virtual directory will be downloaded only when the recursive flag is set to true + if blobNamePattern == "*" && !isRecursiveOn{ + return fmt.Errorf("cannot download the enitre container / virtual directory. Please use recursive flag for this download scenario") + } + // perform a list blob with search prefix for marker := (azblob.Marker{}); marker.NotDone(); { // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, @@ -93,17 +115,14 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) for _, blobInfo := range listBlob.Blobs.Blob { - // If the blob is not valid as per the conditions mentioned in the + // If the blob represents a folder as per the conditions mentioned in the // api doesBlobRepresentAFolder, then skip the blob. if util.doesBlobRepresentAFolder(blobInfo) { continue } - // TODO: add a function to perform the match - matched, err := filepath.Match(blobNamePattern, blobInfo.Name) - if err != nil { - panic(err) - } - if !matched { + // If the blobName doesn't matches the blob name pattern, then blob is not included + // queued for transfer + if !util.blobNameMatchesThePattern(blobNamePattern, blobInfo.Name){ continue } @@ -118,9 +137,9 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs wg, waitUntilJobCompletion) } - marker = listBlob.NextMarker - //err = e.dispatchPart(false) + // dispatch the JobPart as Final Part of the Job + err = e.dispatchFinalPart() if err != nil { return err } @@ -129,7 +148,7 @@ func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecurs } // this function accepts a url (with or without *) to blobs for download and processes them -func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, +func (e *copyDownloadBlobEnumerator) enumerate1(sourceUrlString string, isRecursiveOn bool, destinationPath string, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { util := copyHandlerUtil{} @@ -175,7 +194,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi } // the destination must be a directory, otherwise we don't know where to put the files - if !util.isPathDirectory(destinationPath) { + if !util.isPathALocalDirectory(destinationPath) { return errors.New("the destination must be an existing directory in this download scenario") } @@ -239,7 +258,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi // for a single blob, the destination can either be a file or a directory var singleBlobDestinationPath string - if util.isPathDirectory(destinationPath) { + if util.isPathALocalDirectory(destinationPath) { blobNameFromUrl := util.blobNameFromUrl(blobUrlParts) // check for special characters and get blobName without special character. blobNameFromUrl = util.blobPathWOSpecialCharacters(blobNameFromUrl) @@ -276,7 +295,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi } // the destination must be a directory, otherwise we don't know where to put the files - if !util.isPathDirectory(destinationPath) { + if !util.isPathALocalDirectory(destinationPath) { return errors.New("the destination must be an existing directory in this download scenario") } diff --git a/cmd/copyDownloadFileEnumerator.go b/cmd/copyDownloadFileEnumerator.go index fd35cdfed..7b612a7e9 100644 --- a/cmd/copyDownloadFileEnumerator.go +++ b/cmd/copyDownloadFileEnumerator.go @@ -74,7 +74,7 @@ func (e *copyDownloadFileEnumerator) enumerate(sourceURLString string, isRecursi if doPrefixSearch { // Case 1: Do prefix search, the file pattern would be [AnyLetter]+\* // The destination must be a directory, otherwise we don't know where to put the files. - if !util.isPathDirectory(destinationPath) { + if !util.isPathALocalDirectory(destinationPath) { return fmt.Errorf("the destination must be an existing directory in this download scenario") } @@ -123,7 +123,7 @@ func (e *copyDownloadFileEnumerator) enumerate(sourceURLString string, isRecursi if fileURL != nil { // Single file. var singleFileDestinationPath string - if util.isPathDirectory(destinationPath) { + if util.isPathALocalDirectory(destinationPath) { singleFileDestinationPath = util.generateLocalPath(destinationPath, util.getPossibleFileNameFromURL(sourceURL.Path)) } else { singleFileDestinationPath = destinationPath @@ -141,7 +141,7 @@ func (e *copyDownloadFileEnumerator) enumerate(sourceURLString string, isRecursi } else { // Directory. // The destination must be a directory, otherwise we don't know where to put the files. - if !util.isPathDirectory(destinationPath) { + if !util.isPathALocalDirectory(destinationPath) { return fmt.Errorf("the destination must be an existing directory in this download scenario") } diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 11ce90add..2acb85c20 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -37,6 +37,7 @@ import ( "github.com/Azure/azure-storage-azcopy/ste" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" + "path/filepath" ) const ( @@ -112,7 +113,7 @@ func (copyHandlerUtil) getRelativePath(rootPath, filePath string, pathSep string } // this function can tell if a path represents a directory (must exist) -func (util copyHandlerUtil) isPathDirectory(pathString string) bool { +func (util copyHandlerUtil) isPathALocalDirectory(pathString string) bool { // check if path exists destinationInfo, err := os.Stat(pathString) @@ -125,7 +126,6 @@ func (util copyHandlerUtil) isPathDirectory(pathString string) bool { func (util copyHandlerUtil) generateLocalPath(directoryPath, fileName string) string { var result string - // check if the directory path ends with the path separator if strings.LastIndex(directoryPath, string(os.PathSeparator)) == len(directoryPath)-1 { result = fmt.Sprintf("%s%s", directoryPath, fileName) @@ -133,10 +133,10 @@ func (util copyHandlerUtil) generateLocalPath(directoryPath, fileName string) st result = fmt.Sprintf("%s%s%s", directoryPath, string(os.PathSeparator), fileName) } - if os.PathSeparator == '\\' { - return strings.Replace(result, "/", "\\", -1) - } - return result + // blob name has "/" as Path Separator. + // To preserve the path in blob name on local disk, replace "/" with OS Path Separator + // For Example blob name = "blob-1/blob-2/blob-2" will be "blob-1\\blob-2\\blob-3" for windows + return strings.Replace(result, "/", string(os.PathSeparator), -1) } func (util copyHandlerUtil) getBlobNameFromURL(path string) string { @@ -183,23 +183,45 @@ func (util copyHandlerUtil) createBlobUrlFromContainer(blobUrlParts azblob.BlobU return blobUrl.String() } +func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName string) (bool){ + matched, err := filepath.Match(pattern, blobName) + if err != nil { + panic(err) + } + return matched +} + func (util copyHandlerUtil) searchPrefixFromUrl(parts azblob.BlobURLParts) (prefix, pattern string){ + // If the blobName is empty, it means the url provided is of a container, + // then all blobs inside containers needs to be included, so pattern is set to * if parts.BlobName == "" { pattern = "*" return } + // Check for wildcards and get the index of first wildcard + // If the wild card does not exists, then index returned is -1 wildCardIndex := util.firstIndexOfWildCard(parts.BlobName) - if wildCardIndex < 0 { // no wildcard exists + if wildCardIndex < 0 { + // If no wild card exits and url represents a virtual directory + // prefix is the path of virtual directory after the container. + // Example: https:///vd-1?, prefix = /vd-1 + // Example: https:///vd-1/vd-2?, prefix = /vd-1/vd-2 prefix = parts.BlobName // check for separator at the end of virtual directory if prefix[len(prefix)-1] != '/'{ prefix += "/" } + // since the url is a virtual directory, then all blobs inside the virtual directory + // needs to be downloaded, so the pattern is "*" + // pattern being "*", all blobNames when matched with "*" will be true + // so all blobs inside the virtual dir will be included pattern = "*" return } // wild card exists prefix will be the content of blob name till the wildcard index - //TODO: example add + // Example: https:///vd-1/vd-2/abc* + // prefix = /vd-1/vd-2/abc and pattern = /vd-1/vd-2/abc* + // All the blob inside the container in virtual dir vd-2 that have the prefix "abc" prefix = parts.BlobName[:wildCardIndex] pattern = parts.BlobName return diff --git a/cmd/removeFileEnumerator.go b/cmd/removeFileEnumerator.go index 9886f30f3..85bb62ed9 100644 --- a/cmd/removeFileEnumerator.go +++ b/cmd/removeFileEnumerator.go @@ -76,7 +76,7 @@ func (e *removeFileEnumerator) enumerate(sourceURLString string, isRecursiveOn b if doPrefixSearch { // Case 1: Do prefix search, the file pattern would be [AnyLetter]+\* // The destination must be a directory, otherwise we don't know where to put the files. - if !util.isPathDirectory(destinationPath) { + if !util.isPathALocalDirectory(destinationPath) { return fmt.Errorf("the destination must be an existing directory in this remove scenario") } From f7fbf259d73389e91e44fddd403d15e4d0e7f60c Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Tue, 22 May 2018 12:23:20 -0700 Subject: [PATCH 11/23] fixed blobNameMatchesThePattern for linux --- cmd/copyUtil.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 2acb85c20..126d70175 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -184,6 +184,12 @@ func (util copyHandlerUtil) createBlobUrlFromContainer(blobUrlParts azblob.BlobU } func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName string) (bool){ + // Since filePath.Match matches "*" with any sequence of non-separator characters + // it will return false when "*" matched with "a/b" on linux or "a\\b" on windows + // Hence hard-coded check added for "*" + if pattern == "*" { + return true + } matched, err := filepath.Match(pattern, blobName) if err != nil { panic(err) From 6e48b7daa0e9f443d5dd7cdc1dde6d25a8c1ac71 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Wed, 23 May 2018 18:14:04 -0700 Subject: [PATCH 12/23] include/exclude flag added for copy / resume command; test case added include/exclude flag for copy command --- cmd/copy.go | 43 ++++- cmd/copyDownloadBlobEnumerator.go | 108 ++++++++++- cmd/copyUploadEnumerator.go | 126 ++++++++++++- cmd/copyUtil.go | 45 +++++ cmd/resume.go | 63 +++++-- cmd/rpc.go | 2 +- common/rpc-models.go | 8 + main_windows.go | 4 +- ste/JobPartPlanFileName.go | 3 +- ste/init.go | 7 +- ste/mgr-JobMgr.go | 15 +- ste/mgr-JobPartMgr.go | 36 +++- testSuite/scripts/init.py | 16 +- testSuite/scripts/test_upload_block_blob.py | 188 +++++++++++++++++++- testSuite/scripts/utility.py | 9 + 15 files changed, 627 insertions(+), 46 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index c03270db9..40ff3aecd 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -36,6 +36,7 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/spf13/cobra" + "strings" ) // upload related @@ -63,6 +64,7 @@ type rawCopyCmdArgs struct { //blobUrlForRedirection string // filters from flags + include string exclude string recursive bool followSymlinks bool @@ -100,7 +102,6 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { cooked.fromTo = fromTo // copy&transform flags to type-safety - cooked.exclude = raw.exclude cooked.recursive = raw.recursive cooked.followSymlinks = raw.followSymlinks cooked.withSnapshots = raw.withSnapshots @@ -119,6 +120,39 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) { if err != nil{ return cooked, err } + + // initialize the include map which contains the list of files to be included + // parse the string passed in include flag + // more than one file are expected to be separated by ';' + cooked.include = make(map[string]int) + if len(raw.include) > 0 { + files := strings.Split(raw.include, ";") + for index := range files { + // If split of the include string leads to an empty string + // not include that string + if len(files[index]) == 0 { + continue + } + cooked.include[files[index]] = index + } + } + + // initialize the exclude map which contains the list of files to be excluded + // parse the string passed in exclude flag + // more than one file are expected to be separated by ';' + cooked.exclude = make(map[string]int) + if len(raw.exclude) > 0 { + files := strings.Split(raw.exclude, ";") + for index := range files { + // If split of the include string leads to an empty string + // not include that string + if len(files[index]) == 0 { + continue + } + cooked.exclude[files[index]] = index + } + } + cooked.metadata = raw.metadata cooked.contentType = raw.contentType cooked.contentEncoding = raw.contentEncoding @@ -138,7 +172,8 @@ type cookedCopyCmdArgs struct { fromTo common.FromTo // filters from flags - exclude string + include map[string]int + exclude map[string]int recursive bool followSymlinks bool withSnapshots bool @@ -367,6 +402,8 @@ func (cca cookedCopyCmdArgs) processCopyJobPartOrders() (err error) { ForceWrite: cca.forceWrite, Priority: common.EJobPriority.Normal(), LogLevel: cca.logVerbosity, + Include:cca.include, + Exclude:cca.exclude, BlobAttributes: common.BlobTransferAttributes{ BlockSizeInBytes: cca.blockSize, ContentType: cca.contentType, @@ -535,6 +572,8 @@ Usage: // define the flags relevant to the cp command // filters + cpCmd.PersistentFlags().StringVar(&raw.include, "include", "", "Filter: only include these files when copying. " + + "Support use of *. More than one file are separated by ';'") cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "Filter: Exclude these files when copying. Support use of *.") cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Filter: Look into sub-directories recursively when uploading from local file system.") cpCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "Filter: Follow symbolic links when uploading from local file system.") diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index 19c04e827..6f8423cb4 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -81,7 +81,6 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi } return nil } - // Since the given source url doesn't represent an existing blob // it is either a container or a virtual directory, so it need to be // downloaded to an existing directory @@ -92,6 +91,109 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi literalContainerUrl := util.getContainerUrl(blobUrlParts) containerUrl := azblob.NewContainerURL(literalContainerUrl, p) + + // If the files to be downloaded are mentioned in the include flag + // Download the blobs or virtual directory mentioned with the include flag + if len(e.Include) > 0 { + for blob, _ := range e.Include { + // Get the blobUrl by appending the blob name to the given source Url + // blobName is the name after the container in the appended blobUrl + blobUrl, blobName := util.appendBlobNameToUrl(blobUrlParts, blob) + if blob[len(blob) - 1] != '/' { + // If there is no separator at the end of blobName, then it is consider to be a blob + // For Example src = https://? include = "file1.txt" + // blobUrl = https:///file1.txt? ; blobName = file1.txt + bUrl := azblob.NewBlobURL(blobUrl, p) + bProperties, err := bUrl.GetProperties(context.TODO(), azblob.BlobAccessConditions{}) + if err != nil { + return fmt.Errorf("invalid blob name %s passed in include flag", blob) + } + // check for special characters and get blobName without special character. + blobName = util.blobPathWOSpecialCharacters(blobName) + blobLocalPath := util.generateLocalPath(destinationPath, blobName) + e.addTransfer(common.CopyTransfer{ + Source: blobUrl.String(), + Destination: blobLocalPath, + LastModifiedTime: bProperties.LastModified(), + SourceSize: bProperties.ContentLength(), + }, wg, waitUntilJobCompletion) + }else { + // If there is a separator at the end of blobName, then it is consider to be a virtual directory in the container + // all blobs inside this virtual directory needs to downloaded + // For Example: src = https://? include = "dir1/" + // blobName = dir1/ searchPrefix = dir1/ + // all blob starting with dir1/ will be listed + searchPrefix := blobName + pattern := "*" + // perform a list blob with search prefix + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up + listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, + azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) + } + + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If the blob represents a folder as per the conditions mentioned in the + // api doesBlobRepresentAFolder, then skip the blob. + if util.doesBlobRepresentAFolder(blobInfo) { + continue + } + // If the blobName doesn't matches the blob name pattern, then blob is not included + // queued for transfer + if !util.blobNameMatchesThePattern(pattern, blobInfo.Name){ + continue + } + + blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") + // check for the special character in blob relative path and get path without special character. + blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) + e.addTransfer(common.CopyTransfer{ + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), + Destination: util.generateLocalPath(destinationPath, blobRelativePath), + LastModifiedTime: blobInfo.Properties.LastModified, + SourceSize: *blobInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) + } + marker = listBlob.NextMarker + } + } + } + // dispatch the JobPart as Final Part of the Job + err = e.dispatchFinalPart() + if err != nil { + return err + } + return nil + } + + // If some blobs are mentioned with exclude flag + // Iterate through each blob and append to the source url passed. + // The blob name after appending to the source url is stored in the map + // For Example: src = https:// exclude ="file.txt" + // blobNameToExclude will be dir/file.txt + if len(e.Exclude) > 0 { + destinationBlobName := blobUrlParts.BlobName + if len(destinationBlobName) > 0 && destinationBlobName[len(destinationBlobName)-1] != '/'{ + destinationBlobName += "/" + } + for blob, index := range e.Exclude { + blobNameToExclude := destinationBlobName + blob + // If the blob name passed with the exclude flag is a virtual directory + // Append * at the end of the blobNameToExclude so blobNameToExclude matches + // the name of all blob inside the virtual dir + // For Example: src = https:// exclude ="dir/" + // blobNameToExclude will be "dir/*" + if blobNameToExclude[len(blobNameToExclude)-1] == '/' { + blobNameToExclude += "*" + } + delete(e.Exclude, blob) + e.Exclude[blobNameToExclude] = index + } + } // searchPrefix is the used in listing blob inside a container // all the blob listed should have the searchPrefix as the prefix @@ -125,7 +227,9 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi if !util.blobNameMatchesThePattern(blobNamePattern, blobInfo.Name){ continue } - + if util.resourceShouldBeExcluded(e.Exclude, blobInfo.Name) { + continue + } blobRelativePath := util.getRelativePath(searchPrefix, blobInfo.Name, "/") // check for the special character in blob relative path and get path without special character. blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath) diff --git a/cmd/copyUploadEnumerator.go b/cmd/copyUploadEnumerator.go index c99204ebf..c86b7203e 100644 --- a/cmd/copyUploadEnumerator.go +++ b/cmd/copyUploadEnumerator.go @@ -40,6 +40,12 @@ func (e *copyUploadEnumerator) enumerate(src string, isRecursiveOn bool, dst str } if !f.IsDir() { + // Check if the files are passed with include flag + // then source needs to be directory, if it is a file + // then error is returned + if len(e.Include) > 0 { + return fmt.Errorf("for the use of include flag, source needs to be a directory") + } // append file name as blob name in case the given URL is a container if (e.FromTo == common.EFromTo.LocalBlob() && util.urlIsContainerOrShare(destinationURL)) || (e.FromTo == common.EFromTo.LocalFile() && util.urlIsAzureFileDirectory(ctx, destinationURL)) { @@ -59,12 +65,125 @@ func (e *copyUploadEnumerator) enumerate(src string, isRecursiveOn bool, dst str return e.dispatchFinalPart() } } - // if the user specifies a virtual directory ex: /container_name/extra_path // then we should extra_path as a prefix while uploading // temporarily save the path of the container cleanContainerPath := destinationURL.Path + // If the files have been explicitly mentioned in the Include flag + // then we do not need to iterate through the entire directory + // Iterate through the files or sub-dir in the include flag + // and queue them for transfer + if len(e.Include) > 0 { + for file, _ := range e.Include { + // append the file name in the include flag to the soure directory + // For Example: src = C:\User\new-User include = "file.txt;file2.txt" + // currentFile = C:\User\new\User\file.txt + currentFile := src + string(os.PathSeparator) + file + // temporary saving the destination Url to later modify it + // to get the resource Url + currentDestinationUrl := *destinationURL + f , err := os.Stat(currentFile) + if err != nil { + return fmt.Errorf("invalid file name %s. It doesn't exists inside the directory %s", file, src) + } + // When the string in include flag is a file + // add it to the transfer list. + // Example: currentFile = C:\User\new\User\file.txt + if !f.IsDir() { + currentDestinationUrl.Path = util.generateObjectPath(currentDestinationUrl.Path, + util.getRelativePath(src, currentFile, string(os.PathSeparator))) + e.addTransfer(common.CopyTransfer{ + Source: currentFile, + Destination: currentDestinationUrl.String(), + LastModifiedTime: f.ModTime(), + SourceSize: f.Size(), + }, wg, waitUntilJobCompletion) + }else { + // When the string in include flag is a sub-directory + // Example: currentFile = C:\User\new\User\dir1 + if !isRecursiveOn { + // If the recursive flag is not set to true + // then Ignore the files inside sub-dir + continue + } + // walk through each file in sub directory + err = filepath.Walk(currentFile, func(pathToFile string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir(){ + // If the file inside sub-dir is again a sub-dir + // then skip it, since files inside sub-dir will be + // considered by walk func + return nil + }else{ + // create the remote Url of file inside sub-dir + currentDestinationUrl.Path = util.generateObjectPath(cleanContainerPath, + util.getRelativePath(src, pathToFile, string(os.PathSeparator))) + err = e.addTransfer(common.CopyTransfer{ + Source: pathToFile, + Destination: currentDestinationUrl.String(), + LastModifiedTime: f.ModTime(), + SourceSize: f.Size(), + }, wg, waitUntilJobCompletion) + if err != nil { + return err + } + } + return nil + }) + // TODO: Eventually permission error won't be returned and CopyTransfer will carry the error to the transfer engine. + if err != nil { + return err + } + } + } + // dispatch the final part + e.dispatchFinalPart() + return nil + } + + // Iterate through each file mentioned in the exclude flag + // Verify if the file exists inside the source directory or not. + // Replace the file entry in the exclude map with entire path of file. + if len(e.Exclude) > 0 { + for file, index := range e.Exclude { + var filePath = "" + // If the source directory doesn't have a separator at the end + // place a separator between the source and file + if src[len(src)-1] != os.PathSeparator { + filePath = fmt.Sprintf("%s%s%s", src, string(os.PathSeparator), file) + }else { + filePath = fmt.Sprintf("%s%s", src, file) + } + // Get the file info to verify file exists or not. + f, err := os.Stat(filePath) + if err != nil { + return fmt.Errorf("file %s mentioned in the exclude doesn't exists inside the source dir %s", file, src) + } + + // If the file passed is a sub-directory inside the source directory + // append '*' at the end of the path of sub-dir + // '*' is added so that sub-dir path matches the path of all the files inside this sub-dir + // while enumeration + // For Example: Src = C:\\User\user-1 exclude = "dir1" + // filePath = C:\User\user-1\dir1\* + // filePath matches with Path of C:\User\user-1\dir1\a.txt; C:\User\user-1\dir1\b.txt + if f.IsDir() { + // If the filePath doesn't have a separator at the end + // place a separator between filePath and '*' + if filePath[len(filePath)-1] == os.PathSeparator { + filePath = fmt.Sprintf("%s%s%s", filePath, string(os.PathSeparator), "*") + }else { + filePath = fmt.Sprintf("%s%s", filePath, "*") + } + } + delete(e.Exclude, file) + e.Exclude[filePath] = index + } + } + // walk through every file and directory // upload every file // upload directory recursively if recursive option is on @@ -81,7 +200,10 @@ func (e *copyUploadEnumerator) enumerate(src string, isRecursiveOn bool, dst str if f.IsDir() { // skip the subdirectories, we only care about files return nil - } else { // upload the files + } else { + // Check if the file should be excluded or not. + if util.resourceShouldBeExcluded(e.Exclude, pathToFile) { return nil} + // upload the files // the path in the blob name started at the given fileOrDirectoryPath // example: fileOrDirectoryPath = "/dir1/dir2/dir3" pathToFile = "/dir1/dir2/dir3/file1.txt" result = "dir3/file1.txt" destinationURL.Path = util.generateObjectPath(cleanContainerPath, diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index 126d70175..cf16c8ebb 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -89,10 +89,42 @@ func (copyHandlerUtil) generateObjectPath(destinationPath, fileName string) stri return fmt.Sprintf("%s/%s", destinationPath, fileName) } +// resourceShouldBeExcluded decides whether the file at given filePath should be excluded from the transfer or not. +// First, checks whether filePath exists in the Map or not. +// Then iterates through each entry of the map and check whether the given filePath matches the expression of any +// entry of the map. +func (util copyHandlerUtil) resourceShouldBeExcluded(excludedFilePathMap map[string]int, filePath string) bool{ + // Check if the given filePath exists as an entry in the map + _, ok := excludedFilePathMap[filePath] + if ok { + return true + } + // Iterate through each entry of the Map + // Matches the given filePath against map entry pattern + // This is to handle case when user passed a sub-dir inside + // source to exclude. All the files inside that sub-directory + // should be excluded. + // For Example: src = C:\User\user-1 exclude = "dir1" + // Entry in Map = C:\User\user-1\dir1\* will match the filePath C:\User\user-1\dir1\file1.txt + for key, _ := range excludedFilePathMap { + matched, err := filepath.Match(key, filePath) + if err != nil { + panic(err) + } + if matched { + return true + } + } + return false +} + // get relative path given a root path func (copyHandlerUtil) getRelativePath(rootPath, filePath string, pathSep string) string { // root path contains the entire absolute path to the root directory, so we need to take away everything except the root directory from filePath // example: rootPath = "/dir1/dir2/dir3" filePath = "/dir1/dir2/dir3/file1.txt" result = "dir3/file1.txt" scrubAway="/dir1/dir2/" + if len(rootPath) == 0 { + return filePath + } var scrubAway string // test if root path finishes with a /, if yes, ignore it @@ -183,6 +215,19 @@ func (util copyHandlerUtil) createBlobUrlFromContainer(blobUrlParts azblob.BlobU return blobUrl.String() } +func (util copyHandlerUtil) appendBlobNameToUrl(blobUrlParts azblob.BlobURLParts, blobName string) (url.URL, string){ + if blobUrlParts.BlobName == "" { + blobUrlParts.BlobName = blobName + }else { + if blobUrlParts.BlobName[len(blobUrlParts.BlobName)-1] == '/' { + blobUrlParts.BlobName += blobName + }else{ + blobUrlParts.BlobName += "/" + blobName + } + } + return blobUrlParts.URL(), blobUrlParts.BlobName +} + func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName string) (bool){ // Since filePath.Match matches "*" with any sequence of non-separator characters // it will return false when "*" matched with "a/b" on linux or "a\\b" on windows diff --git a/cmd/resume.go b/cmd/resume.go index 7e2269d11..c6cd9b71b 100644 --- a/cmd/resume.go +++ b/cmd/resume.go @@ -28,11 +28,14 @@ import ( "os" "os/signal" "time" + "strings" ) func init() { var commandLineInput = "" - + var includeTransfer = "" + var excludeTransfer = "" + rawResumeJobCommand := common.ResumeJob{} // resumeCmd represents the resume command resumeCmd := &cobra.Command{ Use: "resume", @@ -51,10 +54,53 @@ func init() { return nil }, Run: func(cmd *cobra.Command, args []string) { - HandleResumeCommand(commandLineInput) + jobID, err := common.ParseJobID(commandLineInput) + if err != nil { + fmt.Println(fmt.Sprintf("error parsing the jobId %s. Failed with error %s", commandLineInput, err.Error())) + os.Exit(1) + } + rawResumeJobCommand.JobID = jobID + rawResumeJobCommand.IncludeTransfer = make(map[string]int) + rawResumeJobCommand.ExcludeTransfer = make(map[string]int) + + // If the transfer has been provided with the include + // parse the transfer list + if len(includeTransfer) > 0 { + // Split the Include Transfer using ';' + transfers := strings.Split(includeTransfer, ";") + for index := range transfers { + if len(transfers[index]) == 0 { + // If the transfer provided is empty + // skip the transfer + // This is to handle the misplaced ';' + continue + } + rawResumeJobCommand.IncludeTransfer[transfers[index]] = index + } + } + // If the transfer has been provided with the exclude + // parse the transfer list + if len(excludeTransfer) > 0 { + // Split the Exclude Transfer using ';' + transfers := strings.Split(excludeTransfer, ";") + for index := range transfers { + if len(transfers[index]) == 0 { + // If the transfer provided is empty + // skip the transfer + // This is to handle the misplaced ';' + continue + } + rawResumeJobCommand.ExcludeTransfer[transfers[index]] = index + } + } + HandleResumeCommand(rawResumeJobCommand) }, } rootCmd.AddCommand(resumeCmd) + rootCmd.PersistentFlags().StringVar(&includeTransfer, "include", "", "Filter: only include these failed transfer will be resumed while resuming the job " + + "More than one file are separated by ';'") + rootCmd.PersistentFlags().StringVar(&excludeTransfer, "exclude", "", "Filter: exclude these failed transfer while resuming the job " + + "More than one file are separated by ';'") } func waitUntilJobCompletion(jobID common.JobID) { @@ -89,20 +135,13 @@ func waitUntilJobCompletion(jobID common.JobID) { // handles the resume command // dispatches the resume Job order to the storage engine -func HandleResumeCommand(jobIdString string) { - // parsing the given JobId to validate its format correctness - jobID, err := common.ParseJobID(jobIdString) - if err != nil { - // If parsing gives an error, hence it is not a valid JobId format - fmt.Println("invalid jobId string passed. Failed while parsing string to jobId") - return - } +func HandleResumeCommand(resJobOrder common.ResumeJob) { var resumeJobResponse common.CancelPauseResumeResponse - Rpc(common.ERpcCmd.ResumeJob(), jobID, &resumeJobResponse) + Rpc(common.ERpcCmd.ResumeJob(), resJobOrder, &resumeJobResponse) if !resumeJobResponse.CancelledPauseResumed { fmt.Println(resumeJobResponse.ErrorMsg) return } - waitUntilJobCompletion(jobID) + waitUntilJobCompletion(resJobOrder.JobID) } diff --git a/cmd/rpc.go b/cmd/rpc.go index 72af98bbe..2b175aa70 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -50,7 +50,7 @@ func inprocSend(rpcCmd common.RpcCmd, requestData interface{}, responseData inte responseData = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Cancelled()) case common.ERpcCmd.ResumeJob(): - *(responseData.(*common.CancelPauseResumeResponse)) = ste.ResumeJobOrder(requestData.(common.JobID)) + *(responseData.(*common.CancelPauseResumeResponse)) = ste.ResumeJobOrder(requestData.(common.ResumeJob)) default: panic(fmt.Errorf("Unrecognized RpcCmd: %q", rpcCmd.String())) diff --git a/common/rpc-models.go b/common/rpc-models.go index 5a3506324..f2b6ada1f 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -43,6 +43,8 @@ type CopyJobPartOrderRequest struct { ForceWrite bool // to determine if the existing needs to be overwritten or not. If set to true, existing blobs are overwritten Priority JobPriority // priority of the task FromTo FromTo + Include map[string]int + Exclude map[string]int Transfers []CopyTransfer LogLevel LogLevel BlobAttributes BlobTransferAttributes @@ -115,6 +117,12 @@ type ListJobTransfersRequest struct { OfStatus TransferStatus } +type ResumeJob struct { + JobID JobID + IncludeTransfer map[string]int + ExcludeTransfer map[string]int +} + // represents the Details and details of a single transfer type TransferDetail struct { Src string diff --git a/main_windows.go b/main_windows.go index 03bbeb423..e7f60ad46 100644 --- a/main_windows.go +++ b/main_windows.go @@ -23,8 +23,8 @@ package main import ( "os" "os/exec" - "path" "syscall" + "fmt" ) func osModifyProcessCommand(cmd *exec.Cmd) *exec.Cmd { @@ -39,7 +39,7 @@ func osModifyProcessCommand(cmd *exec.Cmd) *exec.Cmd { // GetAzCopyAppPath returns the path of Azcopy in local appdata. func GetAzCopyAppPath() string { localAppData := os.Getenv("LOCALAPPDATA") - azcopyAppDataFolder := path.Join(localAppData, "\\Azcopy") + azcopyAppDataFolder := fmt.Sprintf("%s%s%s", localAppData,string(os.PathSeparator), "Azcopy") if err := os.Mkdir(azcopyAppDataFolder, os.ModeDir); err != nil && !os.IsExist(err) { return "" } diff --git a/ste/JobPartPlanFileName.go b/ste/JobPartPlanFileName.go index 9b5d8c4bb..759bd771c 100644 --- a/ste/JobPartPlanFileName.go +++ b/ste/JobPartPlanFileName.go @@ -7,7 +7,6 @@ import ( "github.com/Azure/azure-storage-azcopy/common" "io" "os" - "path" "reflect" "strings" "time" @@ -17,7 +16,7 @@ import ( type JobPartPlanFileName string func (jppfn *JobPartPlanFileName) GetJobPartPlanPath() string { - return path.Join(JobsAdmin.AppPathFolder(), "/"+string(*jppfn)) + return fmt.Sprintf("%s%s%s", JobsAdmin.AppPathFolder(), string(os.PathSeparator), string(*jppfn)) } const jobPartPlanFileNameFormat = "%v--%05d.steV%d" diff --git a/ste/init.go b/ste/init.go index b2ade58b8..37d40ed2b 100644 --- a/ste/init.go +++ b/ste/init.go @@ -111,7 +111,7 @@ func MainSTE(concurrentConnections int, targetRateInMBps int64, azcopyAppPathFol }) http.HandleFunc(common.ERpcCmd.ResumeJob().Pattern(), func(writer http.ResponseWriter, request *http.Request) { - var payload common.JobID + var payload common.ResumeJob deserialize(request, &payload) serialize(ResumeJobOrder(payload), writer) }) @@ -264,7 +264,8 @@ func CancelPauseJobOrder(jobID common.JobID, desiredJobStatus common.JobStatus) // } // return jr //} -func ResumeJobOrder(jobID common.JobID) common.CancelPauseResumeResponse { +func ResumeJobOrder(resJobOrder common.ResumeJob) common.CancelPauseResumeResponse { + jobID := resJobOrder.JobID jm, found := JobsAdmin.JobMgr(jobID) // Find Job being resumed if !found { return common.CancelPauseResumeResponse{ @@ -296,7 +297,7 @@ func ResumeJobOrder(jobID common.JobID) common.CancelPauseResumeResponse { if jm.ShouldLog(pipeline.LogInfo) { jm.Log(pipeline.LogInfo, fmt.Sprintf("JobID=%v resumed", jobID)) } - jm.ResumeTransfers(steCtx) // Reschedule all job part's transfers + jm.ResumeTransfers(steCtx, resJobOrder.IncludeTransfer, resJobOrder.ExcludeTransfer) // Reschedule all job part's transfers jr = common.CancelPauseResumeResponse{ CancelledPauseResumed: true, ErrorMsg: "", diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index ff7e22266..b85a60501 100644 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -38,7 +38,7 @@ type IJobMgr interface { JobPartMgr(partNum PartNumber) (IJobPartMgr, bool) //Throughput() XferThroughput AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, scheduleTransfers bool) IJobPartMgr - ResumeTransfers(appCtx context.Context) + ResumeTransfers(appCtx context.Context, includeTransfer map[string]int, excludeTransfer map[string]int) PipelineLogInfo() pipeline.LogOptions ReportJobPartDone() uint32 Cancel() @@ -101,7 +101,7 @@ func (jm *jobMgr) JobPartMgr(partNumber PartNumber) (IJobPartMgr, bool) { func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, scheduleTransfers bool) IJobPartMgr { jpm := &jobPartMgr{jobMgr: jm, filename: planFile, pacer: JobsAdmin.(*jobsAdmin).pacer} if scheduleTransfers { - jpm.ScheduleTransfers(jm.ctx) + jpm.ScheduleTransfers(jm.ctx, make(map[string]int), make(map[string]int)) } else { // If the transfer not scheduled, then Map the part file. jpm.planMMF = jpm.filename.Map() @@ -112,18 +112,11 @@ func (jm *jobMgr) AddJobPart(partNum PartNumber, planFile JobPartPlanFileName, s } // ScheduleTransfers schedules this job part's transfers. It is called when a new job part is ordered & is also called to resume a paused Job -func (jm *jobMgr) ResumeTransfers(appCtx context.Context) { +func (jm *jobMgr) ResumeTransfers(appCtx context.Context, includeTransfer map[string]int, excludeTransfer map[string]int) { jm.reset(appCtx) jm.jobPartMgrs.Iterate(false, func(p common.PartNumber, jpm IJobPartMgr) { - jpm.ScheduleTransfers(jm.ctx) + jpm.ScheduleTransfers(jm.ctx, includeTransfer, excludeTransfer) }) - //for p := common.PartNumber(0); true; p++ { // Schedule the transfer all of this job's parts - // jpm, found := jm.JobPartMgr(p) - // if !found { - // break - // } - // jpm.ScheduleTransfers(jm.ctx) - //} } // ReportJobPartDone is called to report that a job part completed or failed diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 520035e39..209779dd7 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -18,7 +18,7 @@ var _ IJobPartMgr = &jobPartMgr{} type IJobPartMgr interface { Plan() *JobPartPlanHeader - ScheduleTransfers(jobCtx context.Context) + ScheduleTransfers(jobCtx context.Context, includeTransfer map[string]int, excludeTransfer map[string]int) StartJobXfer(jptm IJobPartTransferMgr) ReportTransferDone() uint32 IsForceWriteTrue() bool @@ -150,7 +150,7 @@ type jobPartMgr struct { func (jpm *jobPartMgr) Plan() *JobPartPlanHeader { return jpm.planMMF.Plan() } // ScheduleTransfers schedules this job part's transfers. It is called when a new job part is ordered & is also called to resume a paused Job -func (jpm *jobPartMgr) ScheduleTransfers(jobCtx context.Context) { +func (jpm *jobPartMgr) ScheduleTransfers(jobCtx context.Context, includeTransfer map[string]int, excludeTransfer map[string]int) { jpm.atomicTransfersDone = 0 // Reset the # of transfers done back to 0 jpm.planMMF = jpm.filename.Map() // Open the job part plan file & memory-map it in plan := jpm.planMMF.Plan() @@ -203,10 +203,42 @@ func (jpm *jobPartMgr) ScheduleTransfers(jobCtx context.Context) { jpm.AddToBytesDone(jppt.SourceSize) // Since transfer is not scheduled, hence increasing the continue } + + // If the list of transfer to be included is passed + // then check current transfer exists in the list of included transfer + // If it doesn't exists, skip the transfer + if len(includeTransfer) > 0 { + // Get the source string from the part plan header + src, _ := plan.TransferSrcDstStrings(t) + // If source doesn't exists, skip the transfer + _, ok := includeTransfer[src] + if !ok { + jpm.ReportTransferDone() // Don't schedule transfer which is not mentioned to be included + jpm.AddToBytesDone(jppt.SourceSize) // Since transfer is not scheduled, hence increasing the number of bytes done + continue + } + } + // If the list of transfer to be excluded is passed + // then check the current transfer in the list of excluded transfer + // If it exists, then skip the transfer + if len(excludeTransfer) > 0 { + // Get the source string from the part plan header + src, _ := plan.TransferSrcDstStrings(t) + // If the source exists in the list of excluded transfer + // skip the transfer + _, ok := excludeTransfer[src] + if ok { + jpm.ReportTransferDone() // Don't schedule transfer which is mentioned to be excluded + jpm.AddToBytesDone(jppt.SourceSize) // Since transfer is not scheduled, hence increasing the number of bytes done + continue + } + } + // If the transfer was failed, then while rescheduling the transfer marking it Started. if ts == common.ETransferStatus.Failed() { jppt.SetTransferStatus(common.ETransferStatus.Started(), true) } + // Each transfer gets its own context (so any chunk can cancel the whole transfer) based off the job's context transferCtx, transferCancel := context.WithCancel(jobCtx) jptm := &jobPartTransferMgr{ diff --git a/testSuite/scripts/init.py b/testSuite/scripts/init.py index 4220631dc..fcbe82c98 100644 --- a/testSuite/scripts/init.py +++ b/testSuite/scripts/init.py @@ -35,11 +35,15 @@ def temp_adhoc_scenario() : #test_upload_download_1kb_file_wildcard_several_files() def execute_user_scenario_azcopy_op(): - test_remove_virtual_directory() - test_set_block_blob_tier() - test_set_page_blob_tier() - test_force_flag_set_to_false_upload() - test_force_flag_set_to_false_download() + test_download_blob_exclude_flag() + test_download_blob_include_flag() + test_upload_block_blob_include_flag() + test_upload_block_blob_exclude_flag() + # test_remove_virtual_directory() + # test_set_block_blob_tier() + # test_set_page_blob_tier() + # test_force_flag_set_to_false_upload() + # test_force_flag_set_to_false_download() def execute_user_scenario_file_1() : ### @@ -162,7 +166,7 @@ def cleanup(): def main(): init() execute_user_scenario_azcopy_op() - execute_user_scenario_blob_1() + #execute_user_scenario_blob_1() #execute_user_scenario_2() #execute_user_scenario_file_1() #temp_adhoc_scenario() diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index 957a594ec..d7c28bbee 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -368,4 +368,190 @@ def test_force_flag_set_to_false_download(): if x.TransfersFailed is not 15 and x.TransfersCompleted is not 5 : print("test_force_flag_set_to_false_download failed with difference in the number of failed and successful transfers") return - print("test_force_flag_set_to_false_download successfully passed") \ No newline at end of file + print("test_force_flag_set_to_false_download successfully passed") + +# test_upload_block_blob_include_flag tests the include flag in the upload scenario +def test_upload_block_blob_include_flag(): + dir_name = "dir_include_flag_set_upload" + # create 10 files inside the directory + dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) + + # create sub-directory inside the dir_include_flag_set_upload + sub_dir_name = os.path.join(dir_name, "sub_dir_include_flag_set_upload") + # create 10 files inside the sub-dir + sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) + + # uploading the directory with 2 files in the include flag. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info")\ + .add_flags("include", "test101024_2.txt;test101024_3.txt").add_flags("output-json", "true").execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of successful transfer should be 2 and there should be not a failed transfer + if x.TransfersCompleted is not 2 and x.TransfersFailed is not 0 : + print("test_upload_block_blob_include_flag failed with difference in the number of failed and successful transfers with 2 files in include flag") + return + + # uploading the directory with sub-dir in the include flag. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info") \ + .add_flags("include", "sub_dir_include_flag_set_upload").add_flags("output-json", "true").execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of successful transfer should be 10 and there should be not failed transfer + if x.TransfersCompleted is not 10 and x.TransfersFailed is not 0 : + print("test_upload_block_blob_include_flag failed with difference in the number of failed and successful transfers with sub-dir in include flag") + return + print("test_upload_block_blob_include_flag successfully passed") + +# test_upload_block_blob_exclude_flag tests the exclude flag in the upload scenario +def test_upload_block_blob_exclude_flag(): + dir_name = "dir_exclude_flag_set_upload" + # create 10 files inside the directory + dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) + + # create sub-directory inside the dir_exclude_flag_set_upload + sub_dir_name = os.path.join(dir_name, "sub_dir_exclude_flag_set_upload") + # create 10 files inside the sub-dir + sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) + + # uploading the directory with 2 files in the exclude flag. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info") \ + .add_flags("exclude", "test101024_2.txt;test101024_3.txt").add_flags("output-json", "true").execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of successful transfer should be 18 and there should be not failed transfer + # Since total number of files inside dir_exclude_flag_set_upload is 20 and 2 files are set + # to exclude, so total number of transfer should be 18 + if x.TransfersCompleted is not 18 and x.TransfersFailed is not 0 : + print("test_upload_block_blob_exclude_flag failed with difference in the number of failed and successful transfers with two files in exclude flag") + return + + # uploading the directory with sub-dir in the exclude flag. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info") \ + .add_flags("exclude", "sub_dir_exclude_flag_set_upload").add_flags("output-json", "true").execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of successful transfer should be 10 and there should be not failed transfer + # Since the total number of files in dir_exclude_flag_set_upload is 20 and sub_dir_exclude_flag_set_upload + # sub-dir is set to exclude, total number of transfer will be 10 + if x.TransfersCompleted is not 10 and x.TransfersFailed is not 0 : + print("test_upload_block_blob_exclude_flag failed with difference in the number of failed and successful transfers with sub-dir in exclude flag") + return + print("test_upload_block_blob_exclude_flag successfully passed") + +def test_download_blob_include_flag(): + # create dir and 10 files of size 1024 inside it + dir_name = "dir_include_flag_set_download" + dir_n_files_path = util.create_test_n_files(1024, 10 , dir_name) + + # create sub-dir inside dir dir_include_flag_set_download + # create 10 files inside the sub-dir of size 1024 + sub_dir_name = os.path.join(dir_name, "sub_dir_include_flag_set_download") + sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) + + # uploading the directory with 20 files in it. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() + if not result: + print("test_download_blob_include_flag failed while uploading ", 20, "files in dir_include_flag_set_download in the container") + return + # execute the validator and validating the uploaded directory. + destination = util.get_resource_sas(dir_name) + result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ + add_flags("is-object-dir","true").execute_azcopy_verify() + if not result: + print("test_download_blob_include_flag test case failed while validating the directory uploaded") + + #download from container with include flags + destination_sas = util.get_resource_sas(dir_name) + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).\ + add_flags("recursive", "true").add_flags("Logging", "info").add_flags("output-json", "true").\ + add_flags("include", "test101024_1.txt;test101024_2.txt;test101024_3.txt"). \ + execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + if x.TransfersCompleted is not 2 and x.TransfersFailed is not 0 : + print("test_download_blob_include_flag failed with difference in the number of failed and successful transfers with 2 files in include flag") + return + + #download from container with sub-dir in include flags + destination_sas = util.get_resource_sas(dir_name) + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path). \ + add_flags("recursive", "true").add_flags("Logging", "info").add_flags("output-json", "true"). \ + add_flags("include", "sub_dir_include_flag_set_download/"). \ + execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + if x.TransfersCompleted is not 10 and x.TransfersFailed is not 0 : + print("test_download_blob_include_flag failed with difference in the number of failed and successful transfers with sub-dir in include flag") + return + print("test_download_blob_include_flag successfully passed") + +def test_download_blob_exclude_flag(): + # create dir and 10 files of size 1024 inside it + dir_name = "dir_exclude_flag_set_download" + dir_n_files_path = util.create_test_n_files(1024, 10 , dir_name) + + # create sub-dir inside dir dir_exclude_flag_set_download + # create 10 files inside the sub-dir of size 1024 + sub_dir_name = os.path.join(dir_name, "sub_dir_exclude_flag_set_download") + sub_dir_n_file_path = util.create_test_n_files(1024, 10, sub_dir_name) + + # uploading the directory with 20 files in it. + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("recursive", "true").add_flags("Logging", "info").execute_azcopy_copy_command() + if not result: + print("test_download_blob_exclude_flag failed while uploading ", 20, "files in dir_exclude_flag_set_download in the container") + return + # execute the validator and validating the uploaded directory. + destination = util.get_resource_sas(dir_name) + result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ + add_flags("is-object-dir","true").execute_azcopy_verify() + if not result: + print("test_download_blob_exclude_flag test case failed while validating the directory uploaded") + + #download from container with exclude flags + destination_sas = util.get_resource_sas(dir_name) + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path). \ + add_flags("recursive", "true").add_flags("Logging", "info").add_flags("output-json", "true"). \ + add_flags("exclude", "test101024_1.txt;test101024_2.txt;test101024_3.txt"). \ + execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of expected successful transfer should be 18 since two files in directory are set to exclude + if x.TransfersCompleted is not 17 and x.TransfersFailed is not 0 : + print("test_download_blob_include_flag failed with difference in the number of failed and successful transfers with 2 files in include flag") + return + + #download from container with sub-dir in exclude flags + destination_sas = util.get_resource_sas(dir_name) + result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path). \ + add_flags("recursive", "true").add_flags("Logging", "info").add_flags("output-json", "true"). \ + add_flags("exclude", "sub_dir_include_flag_set_download/"). \ + execute_azcopy_copy_command_get_output() + # parse the result to get the last job progress summary + result = util.parseAzcopyOutput(result) + # parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Number of Expected Transfer should be 10 since sub-dir is to exclude which has 10 files in it. + if x.TransfersCompleted is not 10 and x.TransfersFailed is not 0 : + print("test_download_blob_include_flag failed with difference in the number of failed and successful transfers with sub-dir in include flag") + return + print("test_download_blob_exclude_flag successfully passed") \ No newline at end of file diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index 15b2a79d2..a7be13e99 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -196,6 +196,15 @@ def create_test_html_file(filename): f.close() return file_path +# creates a dir with given inside test directory +def create_test_dir(dir_name): + dir_path = os.path.join(test_directory_path, dir_name) + try: + os.mkdir(dir_path) + except: + raise Exception("error creating directory ", dir_path) + return dir_path + # create_test_n_files creates given number of files for given size # inside directory inside test directory. # returns the path of directory in which n files are created. From 98f5d01c51974fcd3c366f751bc15e7a461dcb47 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 24 May 2018 13:59:44 -0700 Subject: [PATCH 13/23] fixed infer argument location api; enabled the test case in init test script --- cmd/validators.go | 9 ++++----- testSuite/scripts/init.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/validators.go b/cmd/validators.go index f25fd5b5a..b570fb216 100644 --- a/cmd/validators.go +++ b/cmd/validators.go @@ -27,7 +27,6 @@ import ( "net/url" "reflect" "strings" - "os" ) func validateFromTo(src, dst string, userSpecifiedFromTo string) (common.FromTo, error) { @@ -124,10 +123,10 @@ func inferArgumentLocation(arg string) Location { } } else { // If we successfully get the argument's file stats, then we'll infer that this argument is a local file - _, err := os.Stat(arg) - if err != nil && !os.IsNotExist(err){ - return ELocation.Unknown() - } + //_, err := os.Stat(arg) + //if err != nil && !os.IsNotExist(err){ + // return ELocation.Unknown() + //} return ELocation.Local() } diff --git a/testSuite/scripts/init.py b/testSuite/scripts/init.py index fcbe82c98..951de61db 100644 --- a/testSuite/scripts/init.py +++ b/testSuite/scripts/init.py @@ -39,11 +39,11 @@ def execute_user_scenario_azcopy_op(): test_download_blob_include_flag() test_upload_block_blob_include_flag() test_upload_block_blob_exclude_flag() - # test_remove_virtual_directory() - # test_set_block_blob_tier() - # test_set_page_blob_tier() - # test_force_flag_set_to_false_upload() - # test_force_flag_set_to_false_download() + test_remove_virtual_directory() + test_set_block_blob_tier() + test_set_page_blob_tier() + test_force_flag_set_to_false_upload() + test_force_flag_set_to_false_download() def execute_user_scenario_file_1() : ### @@ -166,9 +166,9 @@ def cleanup(): def main(): init() execute_user_scenario_azcopy_op() - #execute_user_scenario_blob_1() - #execute_user_scenario_2() - #execute_user_scenario_file_1() + execute_user_scenario_blob_1() + execute_user_scenario_2() + execute_user_scenario_file_1() #temp_adhoc_scenario() cleanup() From b7ce0b7ea54a2bfefce421f6694f5c77065e98dc Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 24 May 2018 15:16:43 -0700 Subject: [PATCH 14/23] fixed blob download --- cmd/copyDownloadBlobEnumerator.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/copyDownloadBlobEnumerator.go b/cmd/copyDownloadBlobEnumerator.go index 6f8423cb4..2cddcf582 100644 --- a/cmd/copyDownloadBlobEnumerator.go +++ b/cmd/copyDownloadBlobEnumerator.go @@ -242,11 +242,11 @@ func (e *copyDownloadBlobEnumerator) enumerate(sourceUrlString string, isRecursi waitUntilJobCompletion) } marker = listBlob.NextMarker - // dispatch the JobPart as Final Part of the Job - err = e.dispatchFinalPart() - if err != nil { - return err - } + } + // dispatch the JobPart as Final Part of the Job + err = e.dispatchFinalPart() + if err != nil { + return err } return nil } From 0b3ad8da254da5813902ab01a844adad5252c856 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 24 May 2018 16:56:50 -0700 Subject: [PATCH 15/23] wildcard support added in remove; test case added for removing with wildcards --- cmd/removeBlobEnumerator.go | 189 ++++++++------------ ste/mgr-JobPartMgr.go | 6 +- ste/pacer.go | 2 +- testSuite/scripts/init.py | 6 +- testSuite/scripts/test_azcopy_operations.py | 142 +++++++++++++-- testSuite/scripts/utility.py | 13 ++ 6 files changed, 226 insertions(+), 132 deletions(-) diff --git a/cmd/removeBlobEnumerator.go b/cmd/removeBlobEnumerator.go index b20f81528..2ef388fa4 100644 --- a/cmd/removeBlobEnumerator.go +++ b/cmd/removeBlobEnumerator.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "net/url" - "strings" "sync" "github.com/Azure/azure-storage-azcopy/common" @@ -15,22 +14,22 @@ import ( type removeBlobEnumerator common.CopyJobPartOrderRequest -// this function accepts a url (with or without *) to blobs for download and processes them func (e *removeBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn bool, destinationPath string, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { util := copyHandlerUtil{} - p := azblob.NewPipeline( - azblob.NewAnonymousCredential(), + // Create Pipeline to Get the Blob Properties or List Blob Segment + p := ste.NewBlobPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{ - Retry: azblob.RetryOptions{ - Policy: azblob.RetryPolicyExponential, - MaxTries: ste.UploadMaxTries, - TryTimeout: ste.UploadTryTimeout, - RetryDelay: ste.UploadRetryDelay, - MaxRetryDelay: ste.UploadMaxRetryDelay, - }, - }) + Telemetry: azblob.TelemetryOptions{Value: "azcopy-V2"}, + }, + ste.XferRetryOptions{ + Policy: 0, + MaxTries: ste.UploadMaxTries, + TryTimeout: ste.UploadTryTimeout, + RetryDelay: ste.UploadRetryDelay, + MaxRetryDelay: ste.UploadMaxRetryDelay, + }, nil) // attempt to parse the source url sourceUrl, err := url.Parse(sourceUrlString) @@ -38,117 +37,83 @@ func (e *removeBlobEnumerator) enumerate(sourceUrlString string, isRecursiveOn b return errors.New("cannot parse source URL") } - // get the container url to be used later for listing - literalContainerUrl := util.getContainerURLFromString(*sourceUrl) - containerUrl := azblob.NewContainerURL(literalContainerUrl, p) - - // check if the given url is a container - if util.urlIsContainerOrShare(sourceUrl) { - return errors.New("cannot remove an entire container, use prefix match with a * at the end of path instead") - } - - numOfStarInUrlPath := util.numOfStarInUrl(sourceUrl.Path) - if numOfStarInUrlPath == 1 { // prefix search - - // the * must be at the end of the path - if strings.LastIndex(sourceUrl.Path, "*") != len(sourceUrl.Path)-1 { - return errors.New("the * in the source URL must be at the end of the path") + // get the blob parts + blobUrlParts := azblob.NewBlobURLParts(*sourceUrl) + + // First Check if source blob exists + // This check is in place to avoid listing of the blobs and matching the given blob against it + // For example given source is https:///a? and there exists other blobs aa and aab + // Listing the blobs with prefix /a will list other blob as well + blobUrl := azblob.NewBlobURL(*sourceUrl, p) + blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + + // If the source blob exists, then queue transfer for deletion and return + // Example: https:///? + if err == nil { + e.addTransfer(common.CopyTransfer{ + Source: sourceUrl.String(), + SourceSize: blobProperties.ContentLength(), + }, wg, waitUntilJobCompletion) + // only one transfer for this Job, dispatch the JobPart + err := e.dispatchFinalPart() + if err != nil{ + return err } + return nil + } - // get the search prefix to query the service - searchPrefix := util.getBlobNameFromURL(sourceUrl.Path) - searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end + // save the container Url in order to list the blobs further + literalContainerUrl := util.getContainerUrl(blobUrlParts) + containerUrl := azblob.NewContainerURL(literalContainerUrl, p) - closestVirtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) + // searchPrefix is the used in listing blob inside a container + // all the blob listed should have the searchPrefix as the prefix + // blobNamePattern represents the regular expression which the blobName should Match + // For Example: src = https:///user-1? searchPrefix = user-1/ + // For Example: src = https:///user-1/file*? searchPrefix = user-1/file + searchPrefix, blobNamePattern := util.searchPrefixFromUrl(blobUrlParts) + + // If blobNamePattern is "*", means that all the contents inside the given source url needs to be downloaded + // It means that source url provided is either a container or a virtual directory + // All the blobs inside a container or virtual directory will be downloaded only when the recursive flag is set to true + if blobNamePattern == "*" && !isRecursiveOn{ + return fmt.Errorf("cannot download the enitre container / virtual directory. Please use recursive flag for this download scenario") + } - // strip away the leading / in the closest virtual directory - if len(closestVirtualDirectory) > 0 && closestVirtualDirectory[0:1] == "/" { - closestVirtualDirectory = closestVirtualDirectory[1:] + // perform a list blob with search prefix + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up + listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, + azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) } - // perform a list blob - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix - listBlob, err := containerUrl.ListBlobsFlatSegment(context.TODO(), marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If the blob represents a folder as per the conditions mentioned in the + // api doesBlobRepresentAFolder, then skip the blob. + if util.doesBlobRepresentAFolder(blobInfo) { + continue } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Blobs.Blob { - blobNameAfterPrefix := blobInfo.Name[len(closestVirtualDirectory):] - if !isRecursiveOn && strings.Contains(blobNameAfterPrefix, "/") { - continue - } - - e.addTransfer(common.CopyTransfer{ - Source: util.generateBlobUrl(literalContainerUrl, blobInfo.Name), - SourceSize: *blobInfo.Properties.ContentLength, - }, - wg, waitUntilJobCompletion) + // If the blobName doesn't matches the blob name pattern, then blob is not included + // queued for transfer + if !util.blobNameMatchesThePattern(blobNamePattern, blobInfo.Name){ + continue } - marker = listBlob.NextMarker - } - err = e.dispatchFinalPart() - if err != nil { - return err - } - } else if numOfStarInUrlPath == 0 { // no prefix search - // see if source blob exists - blobUrl := azblob.NewBlobURL(*sourceUrl, p) - blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) - - // if the single blob exists, remove it - if err == nil { e.addTransfer(common.CopyTransfer{ - Source: sourceUrl.String(), - SourceSize: blobProperties.ContentLength()}, - wg, waitUntilJobCompletion) - - } else if err != nil && !isRecursiveOn { - return errors.New("cannot get source blob properties, make sure it exists, for virtual directory remove please use --recursive") - } - - // if recursive happens to be turned on, then we will attempt to download a virtual directory - if isRecursiveOn { - // recursively download everything that is under the given path, that is a virtual directory - searchPrefix := util.getBlobNameFromURL(sourceUrl.Path) - - // if the user did not specify / at the end of the virtual directory, add it before doing the prefix search - if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 { - searchPrefix += "/" - } - - // perform a list blob - for marker := (azblob.Marker{}); marker.NotDone(); { - // look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up - listBlob, err := containerUrl.ListBlobsFlatSegment(context.Background(), marker, - azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) - if err != nil { - return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) - } - - // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) - for _, blobInfo := range listBlob.Blobs.Blob { - e.addTransfer(common.CopyTransfer{ - Source: util.generateBlobUrl(literalContainerUrl, blobInfo.Name), - SourceSize: *blobInfo.Properties.ContentLength}, - wg, - waitUntilJobCompletion) - } - - marker = listBlob.NextMarker - } - } - err = e.dispatchFinalPart() - if err != nil { - return err + Source: util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name), + SourceSize: *blobInfo.Properties.ContentLength}, + wg, + waitUntilJobCompletion) } - - } else { // more than one * is not supported - return errors.New("only one * is allowed in the source URL") + marker = listBlob.NextMarker + } + // dispatch the JobPart as Final Part of the Job + err = e.dispatchFinalPart() + if err != nil { + return err } return nil } diff --git a/ste/mgr-JobPartMgr.go b/ste/mgr-JobPartMgr.go index 209779dd7..cd1d66e60 100644 --- a/ste/mgr-JobPartMgr.go +++ b/ste/mgr-JobPartMgr.go @@ -59,8 +59,8 @@ func NewVersionPolicyFactory() pipeline.Factory { }) } -// newBlobPipeline creates a Pipeline using the specified credentials and options. -func newBlobPipeline(c azblob.Credential, o azblob.PipelineOptions, r XferRetryOptions, p *pacer) pipeline.Pipeline { +// NewBlobPipeline creates a Pipeline using the specified credentials and options. +func NewBlobPipeline(c azblob.Credential, o azblob.PipelineOptions, r XferRetryOptions, p *pacer) pipeline.Pipeline { if c == nil { panic("c can't be nil") } @@ -275,7 +275,7 @@ func (jpm *jobPartMgr) createPipeline() { case common.EFromTo.BlobLocal(): // download from Azure Blob to local file system fallthrough case common.EFromTo.LocalBlob(): // upload from local file system to Azure blob - jpm.pipeline = newBlobPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{ + jpm.pipeline = NewBlobPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{ Log: jpm.jobMgr.PipelineLogInfo(), Telemetry: azblob.TelemetryOptions{Value: "azcopy-V2"}, }, diff --git a/ste/pacer.go b/ste/pacer.go index 4ab2c3b6c..e0d419c0c 100644 --- a/ste/pacer.go +++ b/ste/pacer.go @@ -71,7 +71,7 @@ func NewPacerPolicyFactory(p *pacer) pipeline.Factory { return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc { return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { resp, err := next.Do(ctx, request) - if err == nil { + if p != nil && err == nil { // Reducing the pacer's rate limit by 10 s for every 503 error. p.updateTargetRate( (resp.Response().StatusCode != http.StatusServiceUnavailable) && diff --git a/testSuite/scripts/init.py b/testSuite/scripts/init.py index 951de61db..66439e84d 100644 --- a/testSuite/scripts/init.py +++ b/testSuite/scripts/init.py @@ -34,6 +34,9 @@ def temp_adhoc_scenario() : test_3_1kb_file_in_dir_upload_download_azure_directory_recursive() #test_upload_download_1kb_file_wildcard_several_files() +def execute_user_scenario_wildcards_op(): + test_remove_files_with_Wildcard() + def execute_user_scenario_azcopy_op(): test_download_blob_exclude_flag() test_download_blob_include_flag() @@ -165,11 +168,12 @@ def cleanup(): def main(): init() + execute_user_scenario_wildcards_op() execute_user_scenario_azcopy_op() execute_user_scenario_blob_1() execute_user_scenario_2() execute_user_scenario_file_1() - #temp_adhoc_scenario() + temp_adhoc_scenario() cleanup() main() diff --git a/testSuite/scripts/test_azcopy_operations.py b/testSuite/scripts/test_azcopy_operations.py index a75c966a3..1dad9393e 100644 --- a/testSuite/scripts/test_azcopy_operations.py +++ b/testSuite/scripts/test_azcopy_operations.py @@ -1,15 +1,16 @@ import utility as util - +import json import time +from collections import namedtuple # test_cancel_job verifies the cancel functionality of azcopy def test_cancel_job(): # create test file. file_name = "test_cancel_file.txt" - file_path = create_test_file(file_name, 1024*1024*1024) + file_path = util.create_test_file(file_name, 1024*1024*1024) # execute the azcopy upload job in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = util.Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -20,14 +21,14 @@ def test_cancel_job(): jobId = output_split[len(output_split)-1] # execute azcopy cancel job. - output = Command("cancel").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("cancel").add_arguments(jobId).execute_azcopy_operation_get_output() if output is None: print("error cancelling job with JobId ", jobId) return # execute list job progress summary. # expected behavior is it should fail. - output = Command("list").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("list").add_arguments(jobId).execute_azcopy_operation_get_output() if output is not None: print("error cancelling the job") print("test_cancel_job test failed") @@ -38,10 +39,10 @@ def test_cancel_job(): def test_pause_resume_job_95Mb_file(): # create test file of 20 MB file_name = "test_pause_resume_file_95.txt" - file_path = create_test_file(file_name, 95*1024*1024) + file_path = util.create_test_file(file_name, 95*1024*1024) # execute azcopy file upload in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = util.Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -52,13 +53,13 @@ def test_pause_resume_job_95Mb_file(): jobId = output_split[len(output_split)-1] # execute azcopy pause job with jobId. - output = Command("pause").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("pause").add_arguments(jobId).execute_azcopy_operation_get_output() if output is None: print("error while pausing job with JobId ", jobId) return # execute azcopy resume job with JobId. - output = Command("resume").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("resume").add_arguments(jobId).execute_azcopy_operation_get_output() if output is None: print("error while resuming job with JobId ", jobId) return @@ -68,7 +69,7 @@ def test_pause_resume_job_95Mb_file(): # validated in loop with sleep of 1 min after each try. retry_count = 10 for x in range (0, retry_count): - result = Command("testBlob").add_arguments(file_name).execute_azcopy_verify() + result = util.Command("testBlob").add_arguments(file_name).execute_azcopy_verify() if not result: if x == (retry_count-1): print("the job could not resume successfully. test_pause_resume_job failed") @@ -82,10 +83,10 @@ def test_pause_resume_job_95Mb_file(): def test_pause_resume_job_200Mb_file(): # create test file of 20 MB file_name = "test_pause_resume_file.txt" - file_path = create_test_file(file_name, 200*1024*1024) + file_path = util.create_test_file(file_name, 200*1024*1024) # execute azcopy file upload in background. - output = Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() + output = util.Command("copy").add_arguments(file_path).add_flags("Logging", "info").add_flags("recursive", "true").add_flags("background-op", "true").execute_azcopy_copy_command_get_output() if output is None: print("error copy file ", file_name, " in background mode") print("test_cancel_job test failed") @@ -96,13 +97,13 @@ def test_pause_resume_job_200Mb_file(): jobId = output_split[len(output_split)-1] # execute azcopy pause job with jobId. - output = Command("pause").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("pause").add_arguments(jobId).execute_azcopy_operation_get_output() if output is None: print("error while pausing job with JobId ", jobId) return # execute azcopy resume job with JobId. - output = Command("resume").add_arguments(jobId).execute_azcopy_operation_get_output() + output = util.Command("resume").add_arguments(jobId).execute_azcopy_operation_get_output() if output is None: print("error while resuming job with JobId ", jobId) return @@ -112,7 +113,7 @@ def test_pause_resume_job_200Mb_file(): # validated in loop with sleep of 1 min after each try. retry_count = 10 for x in range (0, retry_count): - result = Command("testBlob").add_arguments(file_name).execute_azcopy_verify() + result = util.Command("testBlob").add_arguments(file_name).execute_azcopy_verify() if not result: if x == (retry_count-1): print("the job could not resume successfully. test_pause_resume_job failed") @@ -148,6 +149,117 @@ def test_remove_virtual_directory(): return print("test_remove_virtual_directory passed") +def test_remove_files_with_Wildcard(): + # create dir dir_remove_files_with_wildcard + # create 40 files inside the dir + dir_name = "dir_remove_files_with_wildcard" + dir_n_files_path = util.create_test_n_files(1024, 40, dir_name) + + #Upload the directory by azcopy + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url).\ + add_flags("Logging", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() + if not result: + print("test_remove_files_with_Wildcard failed uploading directory dir_remove_files_with_wildcard to the container") + return + + # destination is the remote URl of the uploaded dir + destination = util.get_resource_sas(dir_name) + #Verify the Uploaded directory + # execute the validator. + result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ + add_flags("is-object-dir","true").execute_azcopy_verify() + if not result: + print("test_remove_files_with_Wildcard failed validating the uploaded dir dir_remove_files_with_wildcard") + return + + #removes the files that ends with 4.txt + destination_sas_with_wildcard = util.append_text_path_resource_sas(destination, "*4.txt") + result = util.Command("rm").add_arguments(destination_sas_with_wildcard).add_flags("Logging", "Info").\ + add_flags("recursive", "true").add_flags("output-json", "true").execute_azcopy_operation_get_output() + # Get the latest Job Summary + result = util.parseAzcopyOutput(result) + # Parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + if x.TransfersFailed is not 0 and x.TransfersCompleted is not 4 : + print("test_remove_files_with_Wildcard failed with difference in the number of failed and successful transfers") + return + + #removes the files that starts with test + destination_sas_with_wildcard = util.append_text_path_resource_sas(destination, "test*") + result = util.Command("rm").add_arguments(destination_sas_with_wildcard).add_flags("Logging", "Info"). \ + add_flags("recursive", "true").add_flags("output-json", "true").execute_azcopy_operation_get_output() + # Get the latest Job Summary + result = util.parseAzcopyOutput(result) + # Parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Expected number of successful transfer will be 36 since 4 files have already been deleted + if x.TransfersFailed is not 0 and x.TransfersCompleted is not 36 : + print("test_remove_files_with_Wildcard failed with difference in the number of failed and successful transfers") + return + + # Create directory dir_remove_all_files_with_wildcard + dir_name = "dir_remove_all_files_with_wildcard" + dir_n_files_path = util.create_test_n_files(1024, 40, dir_name) + + # Upload the directory using Azcopy + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url).\ + add_flags("Logging", "Info").add_flags("recursive", "true").execute_azcopy_copy_command() + if not result: + print("test_remove_files_with_Wildcard failed uploading dir dir_remove_all_files_with_wildcard") + + # destination is the remote URl of the uploaded dir + destination = util.get_resource_sas(dir_name) + # Validate the Uploaded directory + # execute the validator. + result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ + add_flags("is-object-dir","true").execute_azcopy_verify() + if not result: + print("test_remove_files_with_Wildcard failed validating the uploaded dir dir_remove_files_with_wildcard") + return + # add * at the end of destination sas + # destination_sas_with_wildcard = https:////*? + destination_sas_with_wildcard = util.append_text_path_resource_sas(destination, "*") + result = util.Command("rm").add_arguments(destination_sas_with_wildcard).add_flags("Logging", "Info").\ + add_flags("recursive","true").add_flags("output-json", "true").execute_azcopy_operation_get_output() + # Get the latest Job Summary + result = util.parseAzcopyOutput(result) + # Parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Expected number of successful transfer will be 40 since all files will be deleted + if x.TransfersFailed is not 0 and x.TransfersCompleted is not 36 : + print("test_remove_files_with_Wildcard failed with difference in the number of failed and successful transfers") + return + # removing multiple directories with use of WildCards + for i in range(1,4): + dir_name = "rdir"+str(i) + dir_n_files_path = util.create_test_n_files(1024, 40, dir_name) + # Upload the directory + result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ + add_flags("Logging", "Info").add_flags("recursive","true").execute_azcopy_copy_command() + if not result: + print("test_remove_files_with_Wildcard failed uploading ",dir_name, " to the container") + return + # execute the validator + destination = util.get_resource_sas(dir_name) + result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(destination). \ + add_flags("is-object-dir","true").execute_azcopy_verify() + if not result: + print("test_remove_files_with_Wildcard failed validating the uploaded dir ", dir_name) + return + destination_sas_with_wildcard = util.append_text_path_resource_sas(util.test_container_url, "rdir*") + result = util.Command("rm").add_arguments(destination_sas_with_wildcard).add_flags("Logging", "Info").\ + add_flags("output-json", "true").add_flags("recursive", "true").execute_azcopy_operation_get_output() + # Get the latest Job Summary + result = util.parseAzcopyOutput(result) + # Parse the Json Output + x = json.loads(result, object_hook=lambda d: namedtuple('X', d.keys())(*d.values())) + # Expected number of successful transfer will be 40 since all files will be deleted + if x.TransfersFailed is not 0 and x.TransfersCompleted is not 90 : + print("test_remove_files_with_Wildcard failed with difference in the number of failed and successful transfers") + return + print("test_remove_files_with_Wildcard passed successfully") + + diff --git a/testSuite/scripts/utility.py b/testSuite/scripts/utility.py index a7be13e99..d51012334 100644 --- a/testSuite/scripts/utility.py +++ b/testSuite/scripts/utility.py @@ -343,6 +343,19 @@ def get_resource_sas(resource_name): resource_sas = url_parts[0] + "/" +resource_name + '?' + url_parts[1] return resource_sas +def append_text_path_resource_sas(resource_sas, text): + # Splitting the resource sas to add the text to the SAS + url_parts = resource_sas.split("?") + + # adding the text to the blob name of the resource sas + if url_parts[0].endswith("/"): + # If there is a separator at the end of blob name + # no need to append "/" before the text after the blob name + resource_sas = url_parts[0] + text + '?' + url_parts[1] + else: + resource_sas = url_parts[0] + "/" + text + '?' + url_parts[1] + return resource_sas + # get_resource_sas_from_share return the shared access signature for the given resource # based on the share url. def get_resource_sas_from_share(resource_name): From a817efacf77c68027601bebd55fca52e29b6bf6a Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 24 May 2018 17:24:19 -0700 Subject: [PATCH 16/23] relace filepath.Match with regexp Match --- cmd/copyUtil.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index cf16c8ebb..bb75e2aa5 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -38,6 +38,7 @@ import ( "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" "path/filepath" + "regexp" ) const ( @@ -235,7 +236,7 @@ func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName if pattern == "*" { return true } - matched, err := filepath.Match(pattern, blobName) + matched, err := regexp.MatchString(pattern, blobName) if err != nil { panic(err) } From 58528e4ccfa9f6149acc74e11c87552669f0db4a Mon Sep 17 00:00:00 2001 From: prjain-msft Date: Fri, 25 May 2018 01:48:56 -0700 Subject: [PATCH 17/23] fix for blob name match with pattern on linux --- cmd/copyUtil.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index bb75e2aa5..f731a99b0 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -38,7 +38,6 @@ import ( "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" "github.com/Azure/azure-storage-file-go/2017-07-29/azfile" "path/filepath" - "regexp" ) const ( @@ -236,7 +235,18 @@ func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName if pattern == "*" { return true } - matched, err := regexp.MatchString(pattern, blobName) + // BlobName has "/" as path separators + // filePath.Match matches "*" with any sequence of non-separator characters + // since path separator on linux and blobName is same + // Replace "/" with its url encoded value "%2F" + // This is to handle cases like matching "dir* and dir/a.txt" + // or matching "dir/* and dir/a/b.txt" + if string(os.PathSeparator) == "/" { + pattern = strings.Replace(pattern, "/", "%2F", -1) + blobName = strings.Replace(blobName, "/", "%2F", -1) + } + + matched, err := filepath.Match(pattern, blobName) if err != nil { panic(err) } From 8e096cf8b3de142c1569ad225b127a46df2f890c Mon Sep 17 00:00:00 2001 From: prjain-msft Date: Fri, 25 May 2018 18:11:02 -0700 Subject: [PATCH 18/23] wildcard in sync implemented --- cmd/syncDownloadEnumerator.go | 437 ++++++++++++++++++++++++++-------- cmd/syncUploadEnumerator.go | 231 ++++++++++-------- common/rpc-models.go | 4 + 3 files changed, 481 insertions(+), 191 deletions(-) diff --git a/cmd/syncDownloadEnumerator.go b/cmd/syncDownloadEnumerator.go index b6dd9ce6d..576ee1d9f 100644 --- a/cmd/syncDownloadEnumerator.go +++ b/cmd/syncDownloadEnumerator.go @@ -3,8 +3,6 @@ package cmd import ( "context" "fmt" - "io" - "io/ioutil" "net/http" "net/url" "os" @@ -15,101 +13,15 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" + "path/filepath" ) type syncDownloadEnumerator common.SyncJobPartOrderRequest -// accepts a new transfer which is to delete the blob on container. -func (e *syncDownloadEnumerator) addTransferToDelete(transfer common.CopyTransfer, wg *sync.WaitGroup, - waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { - // If the existing transfers in DeleteJobRequest is equal to NumOfFilesPerDispatchJobPart, - // then send the JobPartOrder to transfer engine. - if len(e.DeleteJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.DeleteJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.DeleteJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then start fetching the Job Progress summary. - if e.PartNumber == 0 { - wg.Add(1) - go waitUntilJobCompletion(e.JobID, wg) - } - e.DeleteJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.DeleteJobRequest.Transfers = append(e.DeleteJobRequest.Transfers, transfer) - return nil -} -// accept a new transfer, if the threshold is reached, dispatch a job part order -func (e *syncDownloadEnumerator) addTransferToUpload(transfer common.CopyTransfer, wg *sync.WaitGroup, - waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { - - if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { - resp := common.CopyJobPartOrderResponse{} - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - // if the current part order sent to engine is 0, then start fetching the Job Progress summary. - if e.PartNumber == 0 { - wg.Add(1) - go waitUntilJobCompletion(e.JobID, wg) - } - e.CopyJobRequest.Transfers = []common.CopyTransfer{} - e.PartNumber++ - } - e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) - return nil -} - -// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent -func (e *syncDownloadEnumerator) dispatchFinalPart() error { - numberOfCopyTransfers := len(e.CopyJobRequest.Transfers) - numberOfDeleteTransfers := len(e.DeleteJobRequest.Transfers) - if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { - return fmt.Errorf("cannot start job because there are no transfer to upload or delete. " + - "The source and destination are in sync") - } else if numberOfCopyTransfers > 0 && numberOfDeleteTransfers > 0 { - var resp common.CopyJobPartOrderResponse - e.CopyJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - e.PartNumber++ - e.DeleteJobRequest.IsFinalPart = true - e.DeleteJobRequest.PartNum = e.PartNumber - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.DeleteJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("delete job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - } else if numberOfCopyTransfers > 0 { - e.CopyJobRequest.IsFinalPart = true - e.CopyJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - } else { - e.DeleteJobRequest.IsFinalPart = true - e.DeleteJobRequest.PartNum = e.PartNumber - var resp common.CopyJobPartOrderResponse - Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.DeleteJobRequest), &resp) - if !resp.JobStarted { - return fmt.Errorf("delete job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) - } - } - return nil -} - -func (e *syncDownloadEnumerator) compareRemoteAgainstLocal( +/* +//TODO: Deprecated Api's. Need to delete the api's after unit test cases for sync are Inplace +func (e *syncDownloadEnumerator) compareRemoteAgainstLocal1( sourcePath string, isRecursiveOn bool, destinationUrlString string, p pipeline.Pipeline, wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { @@ -196,7 +108,7 @@ func (e *syncDownloadEnumerator) compareRemoteAgainstLocal( return nil } -func (e *syncDownloadEnumerator) compareLocalAgainstRemote(src string, isRecursiveOn bool, dst string, wg *sync.WaitGroup, p pipeline.Pipeline, +func (e *syncDownloadEnumerator) compareLocalAgainstRemote1(src string, isRecursiveOn bool, dst string, wg *sync.WaitGroup, p pipeline.Pipeline, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { util := copyHandlerUtil{} @@ -312,6 +224,323 @@ func (e *syncDownloadEnumerator) compareLocalAgainstRemote(src string, isRecursi return nil } return dirIterateFunction(src, "/") +}*/ + +// accept a new transfer, if the threshold is reached, dispatch a job part order +func (e *syncDownloadEnumerator) addTransferToUpload(transfer common.CopyTransfer, wg *sync.WaitGroup, + waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + + if len(e.CopyJobRequest.Transfers) == NumOfFilesPerDispatchJobPart { + resp := common.CopyJobPartOrderResponse{} + e.CopyJobRequest.PartNum = e.PartNumber + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) + + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) + } + // if the current part order sent to engine is 0, then start fetching the Job Progress summary. + if e.PartNumber == 0 { + wg.Add(1) + go waitUntilJobCompletion(e.JobID, wg) + } + e.CopyJobRequest.Transfers = []common.CopyTransfer{} + e.PartNumber++ + } + e.CopyJobRequest.Transfers = append(e.CopyJobRequest.Transfers, transfer) + return nil +} + +// we need to send a last part with isFinalPart set to true, along with whatever transfers that still haven't been sent +func (e *syncDownloadEnumerator) dispatchFinalPart() error { + numberOfCopyTransfers := len(e.CopyJobRequest.Transfers) + numberOfDeleteTransfers := len(e.DeleteJobRequest.Transfers) + // If the numberoftransfer to copy / delete both are 0 + // means no transfer has been to queue to send to STE + if numberOfCopyTransfers == 0 && numberOfDeleteTransfers == 0 { + // If there are some files that were deleted locally + // display the files + if e.FilesDeletedLocally > 0 { + return fmt.Errorf("%d files deleted locally. No transfer to upload or download ", e.FilesDeletedLocally) + }else { + return fmt.Errorf("cannot start job because there are no transfer to upload or delete. " + + "The source and destination are in sync") + } + } else if numberOfCopyTransfers > 0 && numberOfDeleteTransfers > 0 { + //If there are transfer to upload and download both + // Send the CopyJob Part Order first + // Increment the Part Number + // Send the DeleteJob Part are the final Part + var resp common.CopyJobPartOrderResponse + e.CopyJobRequest.PartNum = e.PartNumber + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) + } + e.PartNumber++ + e.DeleteJobRequest.IsFinalPart = true + e.DeleteJobRequest.PartNum = e.PartNumber + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.DeleteJobRequest), &resp) + if !resp.JobStarted { + return fmt.Errorf("delete job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) + } + } else if numberOfCopyTransfers > 0 { + // Only CopyJobPart Order needs to be sent + e.CopyJobRequest.IsFinalPart = true + e.CopyJobRequest.PartNum = e.PartNumber + var resp common.CopyJobPartOrderResponse + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.CopyJobRequest), &resp) + if !resp.JobStarted { + return fmt.Errorf("copy job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) + } + } else { + // Only DeleteJob Part Order needs to be sent + e.DeleteJobRequest.IsFinalPart = true + e.DeleteJobRequest.PartNum = e.PartNumber + var resp common.CopyJobPartOrderResponse + Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(&e.DeleteJobRequest), &resp) + if !resp.JobStarted { + return fmt.Errorf("delete job part order with JobId %s and part number %d failed because %s", e.JobID, e.PartNumber, resp.ErrorMsg) + } + } + return nil +} + +// compareRemoteAgainstLocal api compares the blob at given destination Url and +// compare with blobs locally. If the blobs locally doesn't exists, then destination +// blobs are downloaded locally. +func (e *syncDownloadEnumerator) compareRemoteAgainstLocal( + sourcePath string, isRecursiveOn bool, + destinationUrlString string, p pipeline.Pipeline, + wg *sync.WaitGroup, waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + + util := copyHandlerUtil{} + + destinationUrl, err := url.Parse(destinationUrlString) + if err != nil { + return fmt.Errorf("error parsing the destinatio url") + } + + blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) + containerUrl := util.getContainerUrl(blobUrlParts) + searchPrefix, pattern := util.searchPrefixFromUrl(blobUrlParts) + + containerBlobUrl := azblob.NewContainerURL(containerUrl, p) + // virtual directory is the entire virtual directory path before the blob name + // passed in the searchPrefix + // Example: dst = https:///vd-1? searchPrefix = vd-1/ + // virtualDirectory = vd-1 + // Example: dst = https:///vd-1/vd-2/fi*.txt? searchPrefix = vd-1/vd-2/fi*.txt + // virtualDirectory = vd-1/vd-2/ + virtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) + // strip away the leading / in the closest virtual directory + if len(virtualDirectory) > 0 && virtualDirectory[0:1] == "/" { + virtualDirectory = virtualDirectory[1:] + } + + for marker := (azblob.Marker{}); marker.NotDone(); { + // look for all blobs that start with the prefix + listBlob, err := containerBlobUrl.ListBlobsFlatSegment(context.TODO(), marker, + azblob.ListBlobsSegmentOptions{Prefix: searchPrefix}) + if err != nil { + return fmt.Errorf("cannot list blobs for download. Failed with error %s", err.Error()) + } + + // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) + for _, blobInfo := range listBlob.Blobs.Blob { + // If blob name doesn't match the pattern + // This check supports the Use wild cards + // SearchPrefix is used to list to all the blobs inside the destination + // and pattern is used to identify which blob to compare further + if !util.blobNameMatchesThePattern(pattern, blobInfo.Name) { + fmt.Println("pattern not matched ", pattern, " ", blobInfo.Name) + continue + } + // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded + // Example: src ="C:\User1\user-1" dst = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" + // realtivePathofBlobLocally = virtual-dir/a.txt + // remove the virtual directory from the realtivePathofBlobLocally + realtivePathofBlobLocally := util.getRelativePath(searchPrefix, blobInfo.Name, "/") + realtivePathofBlobLocally = strings.Replace(realtivePathofBlobLocally, virtualDirectory, "",1) + blobLocalPath := util.generateLocalPath(sourcePath, realtivePathofBlobLocally) + //fmt.Println("blob name ", blobInfo.Name) + //fmt.Println("blob loca path ", blobLocalPath) + // Check if the blob exists locally or not + _, err := os.Stat(blobLocalPath) + if err == nil { + // If the blob exists locally, then we don't need to compare the modified time + // since it has already been compared in compareLocalAgainstRemote api + continue + } + // if the blob doesn't exits locally, then we need to download blob. + if err != nil && os.IsNotExist(err) { + // delete the blob. + err = e.addTransferToUpload(common.CopyTransfer{ + Source: util.generateBlobUrl(containerUrl, blobInfo.Name), + Destination: blobLocalPath, + LastModifiedTime:blobInfo.Properties.LastModified, + SourceSize: *blobInfo.Properties.ContentLength, + }, wg, waitUntilJobCompletion) + if err != nil { + return err + } + } + } + marker = listBlob.NextMarker + } + return nil +} + +// compareLocalAgainstRemote iterates through each files/dir inside the source and compares +// them against blobs on container. If the blobs doesn't exists but exists locally, then delete +// the files locally +func (e *syncDownloadEnumerator) compareLocalAgainstRemote(src string, isRecursiveOn bool, dst string, wg *sync.WaitGroup, p pipeline.Pipeline, + waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) (error, bool) { + util := copyHandlerUtil{} + + // attempt to parse the destination url + destinationUrl, err := url.Parse(dst) + if err != nil { + // the destination should have already been validated, it would be surprising if it cannot be parsed at this point + panic(err) + } + blobUrl := azblob.NewBlobURL(*destinationUrl, p) + // Get the local file Info + f, ferr := os.Stat(src) + // Get the destination blob properties + bProperties, berr := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + // If the error occurs while fetching the fileInfo of the source + // return the error + if ferr != nil { + return fmt.Errorf("cannot access the source %s. Failed with error %s", src, err.Error()), false + } + // If the source is a file locally and destination is not a blob + // it means that it could be a virtual directory / container + // sync cannot happen between a file and a virtual directory / container + if !f.IsDir() && berr != nil { + return fmt.Errorf("cannot perform sync since source is a file and destination "+ + "is not a blob. Listing blob failed with error %s", berr.Error()), false + } + // If the destination is an existing blob and the source is a directory + // sync cannot happen between an existing blob and a local directory + if berr == nil && f.IsDir() { + return fmt.Errorf("cannot perform the sync since source %s "+ + "is a directory and destination %s is a blob", src, destinationUrl.String()), true + } + // If the source is a file and destination is a blob + // For Example: "src = C:\User\user-1\a.txt" && "dst = https:///vd-1/a.txt" + if berr == nil && !f.IsDir() { + // Get the blob name from the destination url + // blobName refers to the last name of the blob with which it is stored as file locally + // Example1: "dst = https:///blob1? blobName = blob1" + // Example1: "dst = https:///dir1/blob1? blobName = blob1" + blobName := destinationUrl.Path[strings.LastIndex(destinationUrl.Path, "/")+1:] + // Compare the blob name and file name + // blobName and filename should be same for sync to happen + if strings.Compare(blobName, f.Name()) != 0 { + return fmt.Errorf("sync cannot be done since blob %s and filename %s doesn't match", blobName, f.Name()), true + } + // If the modified time of file local is before than that of blob + // sync needs to happen. The transfer is queued + if f.ModTime().Before(bProperties.LastModified()) { + e.addTransferToUpload(common.CopyTransfer{ + Source: destinationUrl.String(), + Destination: src, + SourceSize: bProperties.ContentLength(), + LastModifiedTime:bProperties.LastModified(), + }, wg, waitUntilJobCompletion) + } + return nil, true + } + + blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) + + // checkAndQueue is an internal function which check the modified time of file locally + // and on container and then decideds whether to queue transfer for upload or not. + checkAndQueue := func(root string, pathToFile string, f os.FileInfo) error { + // localfileRelativePath is the path of file relative to root directory + // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt + // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt + localfileRelativePath := strings.Replace(pathToFile, root, "", 1) + // remove the path separator at the start of relative path + if len(localfileRelativePath) > 0 && localfileRelativePath[0] == os.PathSeparator { + localfileRelativePath = localfileRelativePath[1:] + } + // Appending the fileRelativePath to the destinationUrl + // root = C:\User\user1\dir-1 dst = https:///? + // fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt + // filedestinationUrl = https:////dir-2/a.txt? + filedestinationUrl, _ := util.appendBlobNameToUrl(blobUrlParts, localfileRelativePath) + + // Get the properties of given on container + blobUrl := azblob.NewBlobURL(filedestinationUrl, p) + blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + + if err != nil { + if stError, ok := err.(azblob.StorageError); !ok || (ok && stError.Response().StatusCode != http.StatusNotFound) { + return fmt.Errorf("error sync up the blob %s because it failed to get the properties. Failed with error %s", localfileRelativePath, err.Error()) + } + // If the blobUrl.GetProperties failed with StatusNotFound, it means blob doesn't exists + // delete the blob locally + if stError, ok := err.(azblob.StorageError); !ok || (ok && stError.Response().StatusCode == http.StatusNotFound) { + err := os.Remove(pathToFile) + if err != nil { + return fmt.Errorf("error deleting the file %s. Failed with error %s", pathToFile, err.Error()) + } + e.FilesDeletedLocally ++ + return nil + } + return err + } + // If the local file modified time was after the remote blob + // then sync is required + if err == nil && !blobProperties.LastModified().After(f.ModTime()) { + return nil + } + + // File exists locally but the modified time of file locally was before the modified + // time of blob, so sync is required + err = e.addTransferToUpload(common.CopyTransfer{ + Source: filedestinationUrl.String(), + Destination: pathToFile, + LastModifiedTime: blobProperties.LastModified(), + SourceSize: blobProperties.ContentLength(), + }, wg, waitUntilJobCompletion) + if err != nil { + return err + } + return nil + } + + listOfFilesAndDir, err := filepath.Glob(src) + + if err != nil { + return fmt.Errorf("error listing the file name inside the source %s", src), false + } + + // Iterate through each file / dir inside the source + // and then checkAndQueue + for _, fileOrDir := range listOfFilesAndDir { + f, err := os.Stat(fileOrDir) + if err == nil { + // directories are uploaded only if recursive is on + if f.IsDir() { + // walk goes through the entire directory tree + err = filepath.Walk(fileOrDir, func(pathToFile string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir(){ + return nil + } else { + return checkAndQueue(src, pathToFile, f) + } + }) + } else if !f.IsDir() { + err = checkAndQueue(src, fileOrDir, f) + } + } + } + return nil, false } // this function accepts the list of files/directories to transfer and processes them @@ -341,13 +570,27 @@ func (e *syncDownloadEnumerator) enumerate(src string, isRecursiveOn bool, dst s // FromTo of DeleteJobRequest will be BlobTrash. e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - err := e.compareLocalAgainstRemote(dst, isRecursiveOn, src, wg, p, waitUntilJobCompletion) + // set force wriet flag to true + e.CopyJobRequest.ForceWrite = true + + //Initialize the number of transfer deleted locally to Zero + e.FilesDeletedLocally = 0 + + //Set the log level + e.CopyJobRequest.LogLevel = e.LogLevel + e.DeleteJobRequest.LogLevel = e.LogLevel + + err, isSourceABlob := e.compareLocalAgainstRemote(dst, isRecursiveOn, src, wg, p, waitUntilJobCompletion) if err != nil { return nil } - err = e.compareRemoteAgainstLocal(dst, isRecursiveOn, src, p, wg, waitUntilJobCompletion) - if err != nil { - return err + // If the source provided is a blob, then remote doesn't needs to be compared against the local + // since single blob already has been compared against the file + if !isSourceABlob { + err = e.compareRemoteAgainstLocal(dst, isRecursiveOn, src, p, wg, waitUntilJobCompletion) + if err != nil { + return err + } } // No Job Part has been dispatched, then dispatch the JobPart. if e.PartNumber == 0 || diff --git a/cmd/syncUploadEnumerator.go b/cmd/syncUploadEnumerator.go index 5962dd6e7..3344f2e28 100644 --- a/cmd/syncUploadEnumerator.go +++ b/cmd/syncUploadEnumerator.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" "github.com/Azure/azure-storage-blob-go/2017-07-29/azblob" + "path/filepath" ) type syncUploadEnumerator common.SyncJobPartOrderRequest @@ -109,6 +110,9 @@ func (e *syncUploadEnumerator) dispatchFinalPart() error { return nil } +// compareRemoteAgainstLocal api compares the blob at given destination Url and +// compare with blobs locally. If the blobs locally doesn't exists, then destination +// blobs are deleted. func (e *syncUploadEnumerator) compareRemoteAgainstLocal( sourcePath string, isRecursiveOn bool, destinationUrlString string, p pipeline.Pipeline, @@ -120,29 +124,22 @@ func (e *syncUploadEnumerator) compareRemoteAgainstLocal( if err != nil { return fmt.Errorf("error parsing the destinatio url") } - var containerUrl url.URL - var searchPrefix string - if !util.urlIsContainerOrShare(destinationUrl) { - containerUrl = util.getContainerURLFromString(*destinationUrl) - // get the search prefix to query the service - searchPrefix = util.getBlobNameFromURL(destinationUrl.Path) - searchPrefix = searchPrefix[:len(searchPrefix)-1] // strip away the * at the end - } else { - containerUrl = *destinationUrl - searchPrefix = "" - } - // if the user did not specify / at the end of the virtual directory, add it before doing the prefix search - if strings.LastIndex(searchPrefix, "/") != len(searchPrefix)-1 { - searchPrefix += "/" - } + blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) + containerUrl := util.getContainerUrl(blobUrlParts) + searchPrefix, pattern := util.searchPrefixFromUrl(blobUrlParts) containerBlobUrl := azblob.NewContainerURL(containerUrl, p) - - closestVirtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) + // virtual directory is the entire virtual directory path before the blob name + // passed in the searchPrefix + // Example: dst = https:///vd-1? searchPrefix = vd-1/ + // virtualDirectory = vd-1 + // Example: dst = https:///vd-1/vd-2/fi*.txt? searchPrefix = vd-1/vd-2/fi*.txt + // virtualDirectory = vd-1/vd-2/ + virtualDirectory := util.getLastVirtualDirectoryFromPath(searchPrefix) // strip away the leading / in the closest virtual directory - if len(closestVirtualDirectory) > 0 && closestVirtualDirectory[0:1] == "/" { - closestVirtualDirectory = closestVirtualDirectory[1:] + if len(virtualDirectory) > 0 && virtualDirectory[0:1] == "/" { + virtualDirectory = virtualDirectory[1:] } for marker := (azblob.Marker{}); marker.NotDone(); { @@ -155,15 +152,22 @@ func (e *syncUploadEnumerator) compareRemoteAgainstLocal( // Process the blobs returned in this result segment (if the segment is empty, the loop body won't execute) for _, blobInfo := range listBlob.Blobs.Blob { - blobNameAfterPrefix := blobInfo.Name[len(closestVirtualDirectory):] - // If there is a "/" at the start of blobName, then strip "/" separator. - if len(blobNameAfterPrefix) > 0 && blobNameAfterPrefix[0:1] == "/" { - blobNameAfterPrefix = blobNameAfterPrefix[1:] - } - if !isRecursiveOn && strings.Contains(blobNameAfterPrefix, "/") { + // If blob name doesn't match the pattern + // This check supports the Use wild cards + // SearchPrefix is used to list to all the blobs inside the destination + // and pattern is used to identify which blob to compare further + if !util.blobNameMatchesThePattern(pattern, blobInfo.Name) { continue } - blobLocalPath := util.generateLocalPath(sourcePath, blobNameAfterPrefix) + + // realtivePathofBlobLocally is the local path relative to source at which blob should be downloaded + // Example: src ="C:\User1\user-1" dst = "https:///virtual-dir?" blob name = "virtual-dir/a.txt" + // realtivePathofBlobLocally = virtual-dir/a.txt + // remove the virtual directory from the realtivePathofBlobLocally + realtivePathofBlobLocally := util.getRelativePath(searchPrefix, blobInfo.Name, "/") + realtivePathofBlobLocally = strings.Replace(realtivePathofBlobLocally, virtualDirectory, "",1) + blobLocalPath := util.generateLocalPath(sourcePath, realtivePathofBlobLocally) + // Check if the blob exists locally or not _, err := os.Stat(blobLocalPath) if err == nil { continue @@ -179,16 +183,12 @@ func (e *syncUploadEnumerator) compareRemoteAgainstLocal( } } marker = listBlob.NextMarker - //err = e.dispatchPart(false) - if err != nil { - return err - } } return nil } func (e *syncUploadEnumerator) compareLocalAgainstRemote(src string, isRecursiveOn bool, dst string, wg *sync.WaitGroup, p pipeline.Pipeline, - waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) error { + waitUntilJobCompletion func(jobID common.JobID, wg *sync.WaitGroup)) (error, bool) { util := copyHandlerUtil{} // attempt to parse the destination url @@ -198,99 +198,134 @@ func (e *syncUploadEnumerator) compareLocalAgainstRemote(src string, isRecursive panic(err) } blobUrl := azblob.NewBlobURL(*destinationUrl, p) + // Get the local file Info f, ferr := os.Stat(src) + // Get the destination blob properties bProperties, berr := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + // If the error occurs while fetching the fileInfo of the source + // return the error if ferr != nil { - return fmt.Errorf("cannot access the source %s. Failed with error %s", src, err.Error()) + return fmt.Errorf("cannot access the source %s. Failed with error %s", src, err.Error()), false } + // If the source is a file locally and destination is not a blob + // it means that it could be a virtual directory / container + // sync cannot happen between a file and a virtual directory / container if !f.IsDir() && berr != nil { return fmt.Errorf("cannot perform sync since source is a file and destination "+ - "is not a blob. Listing blob failed with error %s", berr.Error()) + "is not a blob. Listing blob failed with error %s", berr.Error()), true } + // If the destination is an existing blob and the source is a directory + // sync cannot happen between an existing blob and a local directory if berr == nil && f.IsDir() { return fmt.Errorf("cannot perform the sync since source %s "+ - "is a directory and destination %s is a blob", src, destinationUrl.String()) + "is a directory and destination %s is a blob", src, destinationUrl.String()), false } // If the source is a file and destination is a blob + // For Example: "src = C:\User\user-1\a.txt" && "dst = https:///vd-1/a.txt" if berr == nil && !f.IsDir() { - blobName := destinationUrl.Path[strings.LastIndex(destinationUrl.Path, "/"):] + // Get the blob name from the destination url + // blobName refers to the last name of the blob with which it is stored as file locally + // Example1: "dst = https:///blob1? blobName = blob1" + // Example1: "dst = https:///dir1/blob1? blobName = blob1" + blobName := destinationUrl.Path[strings.LastIndex(destinationUrl.Path, "/")+1:] + // Compare the blob name and file name + // blobName and filename should be same for sync to happen if strings.Compare(blobName, f.Name()) != 0 { - return fmt.Errorf("sync cannot be done since blob %s and filename %s doesn't match", blobName, f.Name()) + return fmt.Errorf("sync cannot be done since blob %s and filename %s doesn't match", blobName, f.Name()), true } + // If the modified time of file local is later than that of blob + // sync needs to happen. The transfer is queued if f.ModTime().After(bProperties.LastModified()) { e.addTransferToUpload(common.CopyTransfer{ Source: src, Destination: destinationUrl.String(), SourceSize: f.Size(), + LastModifiedTime:f.ModTime(), }, wg, waitUntilJobCompletion) } - return nil + return nil, true } - // verify the source path provided is valid or not. - _, err = os.Stat(src) - if err != nil { - return fmt.Errorf("cannot find source to sync") - } + blobUrlParts := azblob.NewBlobURLParts(*destinationUrl) - var containerPath string - var destinationSuffixAfterContainer string + // checkAndQueue is an internal function which check the modified time of file locally + // and on container and then decideds whether to queue transfer for upload or not. + checkAndQueue := func(root string, pathToFile string, f os.FileInfo) error { + // localfileRelativePath is the path of file relative to root directory + // Example1: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\a.txt localfileRelativePath = \a.txt + // Example2: root = C:\User\user1\dir-1 fileAbsolutePath = :\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt + localfileRelativePath := strings.Replace(pathToFile, root, "", 1) + // remove the path separator at the start of relative path + if len(localfileRelativePath) > 0 && localfileRelativePath[0] == os.PathSeparator { + localfileRelativePath = localfileRelativePath[1:] + } + // Appending the fileRelativePath to the destinationUrl + // root = C:\User\user1\dir-1 dst = https:///? + // fileAbsolutePath = C:\User\user1\dir-1\dir-2\a.txt localfileRelativePath = \dir-2\a.txt + // filedestinationUrl = https:////dir-2/a.txt? + filedestinationUrl, _ := util.appendBlobNameToUrl(blobUrlParts, localfileRelativePath) - // If destination url is not container, then get container Url from destination string. - if !util.urlIsContainerOrShare(destinationUrl) { - containerPath, destinationSuffixAfterContainer = util.getConatinerUrlAndSuffix(*destinationUrl) - } else { - containerPath = util.getContainerURLFromString(*destinationUrl).Path - destinationSuffixAfterContainer = "" - } + // Get the properties of given on container + blobUrl := azblob.NewBlobURL(filedestinationUrl, p) + blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) - var dirIterateFunction func(dirPath string, currentDirString string) error - dirIterateFunction = func(dirPath string, currentDirString string) error { - files, err := ioutil.ReadDir(dirPath) + if err != nil { + if stError, ok := err.(azblob.StorageError); !ok || (ok && stError.Response().StatusCode != http.StatusNotFound) { + return fmt.Errorf("error sync up the blob %s because it failed to get the properties. Failed with error %s", localfileRelativePath, err.Error()) + } + } + // If the local file modified time was behind the remote + // then sync is not required + if err == nil && !f.ModTime().After(blobProperties.LastModified()) { + return nil + } + // Closing the blob Properties response body if not nil. + if blobProperties != nil && blobProperties.Response() != nil { + io.Copy(ioutil.Discard, blobProperties.Response().Body) + blobProperties.Response().Body.Close() + } + err = e.addTransferToUpload(common.CopyTransfer{ + Source: pathToFile, + Destination: filedestinationUrl.String(), + LastModifiedTime: f.ModTime(), + SourceSize: f.Size(), + }, wg, waitUntilJobCompletion) if err != nil { return err } - // Iterate through all files and directories. - for i := 0; i < len(files); i++ { - if files[i].IsDir() { - dirIterateFunction(dirPath+string(os.PathSeparator)+files[i].Name(), currentDirString+files[i].Name()+"/") - } else { - // the path in the blob name started at the given fileOrDirectoryPath - // example: fileOrDirectoryPath = "/dir1/dir2/dir3" pathToFile = "/dir1/dir2/dir3/file1.txt" result = "dir3/file1.txt" - destinationUrl.Path = containerPath + destinationSuffixAfterContainer + currentDirString + files[i].Name() - localFilePath := dirPath + string(os.PathSeparator) + files[i].Name() - blobUrl := azblob.NewBlobURL(*destinationUrl, p) - blobProperties, err := blobUrl.GetProperties(context.Background(), azblob.BlobAccessConditions{}) + return nil + } - if err != nil { - if stError, ok := err.(azblob.StorageError); !ok || (ok && stError.Response().StatusCode != http.StatusNotFound) { - return fmt.Errorf("error sync up the blob %s because it failed to get the properties. Failed with error %s", localFilePath, err.Error()) - } - } - if err == nil && !files[i].ModTime().After(blobProperties.LastModified()) { - continue - } + listOfFilesAndDir, err := filepath.Glob(src) - // Closing the blob Properties response body if not nil. - if blobProperties != nil && blobProperties.Response() != nil { - io.Copy(ioutil.Discard, blobProperties.Response().Body) - blobProperties.Response().Body.Close() - } + if err != nil { + return fmt.Errorf("error listing the file name inside the source %s", src), false + } - err = e.addTransferToUpload(common.CopyTransfer{ - Source: localFilePath, - Destination: destinationUrl.String(), - LastModifiedTime: files[i].ModTime(), - SourceSize: files[i].Size(), - }, wg, waitUntilJobCompletion) - if err != nil { - return err - } + // Iterate through each file / dir inside the source + // and then checkAndQueue + for _, fileOrDir := range listOfFilesAndDir { + f, err := os.Stat(fileOrDir) + if err == nil { + // directories are uploaded only if recursive is on + if f.IsDir() { + // walk goes through the entire directory tree + err = filepath.Walk(fileOrDir, func(pathToFile string, f os.FileInfo, err error) error { + if err != nil { + return err + } + if f.IsDir(){ + return nil + } else { + return checkAndQueue(src, pathToFile, f) + } + }) + } else if !f.IsDir() { + err = checkAndQueue(src, fileOrDir, f) } } - return nil } - return dirIterateFunction(src, "/") + return nil, false } // this function accepts the list of files/directories to transfer and processes them @@ -316,14 +351,22 @@ func (e *syncUploadEnumerator) enumerate(src string, isRecursiveOn bool, dst str // FromTo of DeleteJobRequest will be BlobTrash. e.DeleteJobRequest.FromTo = common.EFromTo.BlobTrash() - err := e.compareLocalAgainstRemote(src, isRecursiveOn, dst, wg, p, waitUntilJobCompletion) - if err != nil { - return nil - } - err = e.compareRemoteAgainstLocal(src, isRecursiveOn, dst, p, wg, waitUntilJobCompletion) + // Set the Log Level + e.CopyJobRequest.LogLevel = e.LogLevel + e.DeleteJobRequest.LogLevel = e.LogLevel + + err, isSourceAFile := e.compareLocalAgainstRemote(src, isRecursiveOn, dst, wg, p, waitUntilJobCompletion) if err != nil { return err } + // isSourceAFile defines whether source is a file or not. + // If source is a file and destination is a blob, then destination doesn't needs to be compared against local. + if !isSourceAFile { + err = e.compareRemoteAgainstLocal(src, isRecursiveOn, dst, p, wg, waitUntilJobCompletion) + if err != nil { + return err + } + } // No Job Part has been dispatched, then dispatch the JobPart. if e.PartNumber == 0 || len(e.CopyJobRequest.Transfers) > 0 || diff --git a/common/rpc-models.go b/common/rpc-models.go index f2b6ada1f..f91ded95a 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -58,6 +58,10 @@ type SyncJobPartOrderRequest struct { BlockSizeInBytes uint32 CopyJobRequest CopyJobPartOrderRequest DeleteJobRequest CopyJobPartOrderRequest + // FilesDeletedLocally is used to keep track of the file that are deleted locally + // Since local files to delete are not sent as transfer to STE + // the count of the local files deletion is tracked using it. + FilesDeletedLocally int } type CopyJobPartOrderResponse struct { From 0d5777a4e09af82d1f0a49022a757b5b1ee179d1 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 31 May 2018 14:14:12 -0700 Subject: [PATCH 19/23] fixed cancel command --- cmd/cancel.go | 3 +-- cmd/copy.go | 1 - cmd/rpc.go | 2 +- ste/init.go | 7 +++---- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/cmd/cancel.go b/cmd/cancel.go index 0f185fb28..168b1d4c1 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -111,8 +111,7 @@ func ReadStandardInputToCancelJob(cancelChannel chan <- os.Signal) { // ReadString reads input until the first occurrence of \n in the input, input, err := consoleReader.ReadString('\n') if err != nil { - fmt.Println(err) - os.Exit(1) + return } //remove the delimiter "\n" diff --git a/cmd/copy.go b/cmd/copy.go index 40ff3aecd..85870a0d1 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -485,7 +485,6 @@ func (cca cookedCopyCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync os.Exit(1) default: jobStatus := copyHandlerUtil{}.fetchJobStatus(jobID, &startTime, &bytesTransferredInLastInterval, cca.outputJson) - // happy ending to the front end if jobStatus == common.EJobStatus.Completed() { os.Exit(0) diff --git a/cmd/rpc.go b/cmd/rpc.go index 2b175aa70..2cbbde782 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -47,7 +47,7 @@ func inprocSend(rpcCmd common.RpcCmd, requestData interface{}, responseData inte responseData = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Paused()) case common.ERpcCmd.CancelJob(): - responseData = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Cancelled()) + *(responseData.(*common.CancelPauseResumeResponse)) = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Cancelled()) case common.ERpcCmd.ResumeJob(): *(responseData.(*common.CancelPauseResumeResponse)) = ste.ResumeJobOrder(requestData.(common.ResumeJob)) diff --git a/ste/init.go b/ste/init.go index 37d40ed2b..211100836 100644 --- a/ste/init.go +++ b/ste/init.go @@ -186,16 +186,15 @@ func CancelPauseJobOrder(jobID common.JobID, desiredJobStatus common.JobStatus) jpp0 := jpm.Plan() var jr common.CancelPauseResumeResponse switch jpp0.JobStatus() { // Current status - case common.EJobStatus.InProgress(): // Changing to InProgress/Paused/Canceled is OK - jpp0.SetJobStatus(desiredJobStatus) // Set status to paused/Canceled - case common.EJobStatus.Completed(): // You can't change state of a completed job jr = common.CancelPauseResumeResponse{ CancelledPauseResumed: false, ErrorMsg: fmt.Sprintf("Can't %s JobID=%v because it has already completed", verb, jobID), } - + case common.EJobStatus.InProgress(): + fallthrough case common.EJobStatus.Paused(): // Logically, It's OK to pause an already-paused job + fallthrough case common.EJobStatus.Cancelled(): jpp0.SetJobStatus(desiredJobStatus) msg := fmt.Sprintf("JobID=%v %s", jobID, From 8d54cc048d0bd0391ad56db19335c3fefd537a7c Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Thu, 31 May 2018 15:23:06 -0700 Subject: [PATCH 20/23] delete blob/file/ local file in case of failed / cancelled transfer --- cmd/cancel.go | 2 +- cmd/copy.go | 10 +++-- cmd/copyUtil.go | 6 +-- cmd/resume.go | 11 +++-- cmd/rpc.go | 2 +- cmd/syncUploadEnumerator.go | 3 ++ common/fe-ste-models.go | 5 ++- ste/init.go | 10 +++-- ste/mgr-JobMgr.go | 3 +- ste/mgr-JobPartTransferMgr.go | 6 +++ ste/xfer-blobToLocal.go | 80 ++++++++++++++++++++--------------- ste/xfer-fileToLocal.go | 70 +++++++++++++++--------------- ste/xfer-localToBlockBlob.go | 49 ++++++++++++++++++++- ste/xfer-localToFile.go | 13 +++++- 14 files changed, 179 insertions(+), 91 deletions(-) diff --git a/cmd/cancel.go b/cmd/cancel.go index 168b1d4c1..5993102a6 100644 --- a/cmd/cancel.go +++ b/cmd/cancel.go @@ -62,7 +62,7 @@ func (cca cookedCancelCmdArgs) process() error { if !cancelJobResponse.CancelledPauseResumed { return fmt.Errorf("job cannot be cancelled because %s", cancelJobResponse.ErrorMsg) } - fmt.Println(fmt.Sprintf("Job %s cancelled successfully", cca.jobID)) + //fmt.Println(fmt.Sprintf("Job %s cancelled successfully", cca.jobID)) return nil } diff --git a/cmd/copy.go b/cmd/copy.go index 85870a0d1..0b7d27f44 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -480,13 +480,15 @@ func (cca cookedCopyCmdArgs) waitUntilJobCompletion(jobID common.JobID, wg *sync for { select { case <-CancelChannel: - fmt.Println("Cancelling Job") - cookedCancelCmdArgs{jobID: jobID}.process() - os.Exit(1) + err := cookedCancelCmdArgs{jobID: jobID}.process() + if err != nil { + fmt.Println(fmt.Sprintf("error occurred while cancelling the job %s. Failed with error %s", jobID, err.Error())) + os.Exit(1) + } default: jobStatus := copyHandlerUtil{}.fetchJobStatus(jobID, &startTime, &bytesTransferredInLastInterval, cca.outputJson) // happy ending to the front end - if jobStatus == common.EJobStatus.Completed() { + if jobStatus == common.EJobStatus.Completed() || jobStatus == common.EJobStatus.Cancelled() { os.Exit(0) } diff --git a/cmd/copyUtil.go b/cmd/copyUtil.go index f731a99b0..f239296bb 100644 --- a/cmd/copyUtil.go +++ b/cmd/copyUtil.go @@ -241,7 +241,7 @@ func (util copyHandlerUtil) blobNameMatchesThePattern(pattern string , blobName // Replace "/" with its url encoded value "%2F" // This is to handle cases like matching "dir* and dir/a.txt" // or matching "dir/* and dir/a/b.txt" - if string(os.PathSeparator) == "/" { + if os.PathSeparator == '/' { pattern = strings.Replace(pattern, "/", "%2F", -1) blobName = strings.Replace(blobName, "/", "%2F", -1) } @@ -411,8 +411,8 @@ func (copyHandlerUtil) fetchJobStatus(jobID common.JobID, startTime *time.Time, *startTime = time.Now() *bytesTransferredInLastInterval = summary.BytesOverWire throughPut := common.Ifffloat64(timeElapsed != 0, bytesInMb / timeElapsed, 0) - message := fmt.Sprintf("%v Complete, throughput : %v MB/s, ( %d transfers: %d successful, %d failed, %d pending. Job ordered completely %v)", - summary.JobProgressPercentage, ste.ToFixed(throughPut, 4), summary.TotalTransfers, summary.TransfersCompleted, summary.TransfersFailed, + message := fmt.Sprintf("%v Complete, JobStatus %s , throughput : %v MB/s, ( %d transfers: %d successful, %d failed, %d pending. Job ordered completely %v)", + summary.JobProgressPercentage, summary.JobStatus, ste.ToFixed(throughPut, 4), summary.TotalTransfers, summary.TransfersCompleted, summary.TransfersFailed, summary.TotalTransfers-(summary.TransfersCompleted+summary.TransfersFailed), summary.CompleteJobOrdered) fmt.Println(message) } diff --git a/cmd/resume.go b/cmd/resume.go index c6cd9b71b..25b7eb755 100644 --- a/cmd/resume.go +++ b/cmd/resume.go @@ -115,14 +115,17 @@ func waitUntilJobCompletion(jobID common.JobID) { for { select { case <-CancelChannel: - fmt.Println("Cancelling Job") - cookedCancelCmdArgs{jobID: jobID}.process() - os.Exit(1) + //fmt.Println("Cancelling Job") + err := cookedCancelCmdArgs{jobID: jobID}.process() + if err != nil { + fmt.Println(fmt.Sprintf("error occurred while cancelling the job %s. Failed with error %s", jobID, err.Error())) + os.Exit(1) + } default: jobStatus := copyHandlerUtil{}.fetchJobStatus(jobID, &startTime, &bytesTransferredInLastInterval,false) // happy ending to the front end - if jobStatus == common.EJobStatus.Completed() { + if jobStatus == common.EJobStatus.Completed() || jobStatus == common.EJobStatus.Cancelled(){ os.Exit(0) } diff --git a/cmd/rpc.go b/cmd/rpc.go index 2cbbde782..f2d695935 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -47,7 +47,7 @@ func inprocSend(rpcCmd common.RpcCmd, requestData interface{}, responseData inte responseData = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Paused()) case common.ERpcCmd.CancelJob(): - *(responseData.(*common.CancelPauseResumeResponse)) = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Cancelled()) + *(responseData.(*common.CancelPauseResumeResponse)) = ste.CancelPauseJobOrder(requestData.(common.JobID), common.EJobStatus.Cancelling()) case common.ERpcCmd.ResumeJob(): *(responseData.(*common.CancelPauseResumeResponse)) = ste.ResumeJobOrder(requestData.(common.ResumeJob)) diff --git a/cmd/syncUploadEnumerator.go b/cmd/syncUploadEnumerator.go index 3344f2e28..92b276ff8 100644 --- a/cmd/syncUploadEnumerator.go +++ b/cmd/syncUploadEnumerator.go @@ -355,6 +355,9 @@ func (e *syncUploadEnumerator) enumerate(src string, isRecursiveOn bool, dst str e.CopyJobRequest.LogLevel = e.LogLevel e.DeleteJobRequest.LogLevel = e.LogLevel + // Set the force flag to true + e.CopyJobRequest.ForceWrite = true + err, isSourceAFile := e.compareLocalAgainstRemote(src, isRecursiveOn, dst, wg, p, waitUntilJobCompletion) if err != nil { return err diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 396e6e5f0..ef2a40bc4 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -160,8 +160,9 @@ func (j *JobStatus) AtomicStore(newJobStatus JobStatus) { func (JobStatus) InProgress() JobStatus { return JobStatus(0) } func (JobStatus) Paused() JobStatus { return JobStatus(1) } -func (JobStatus) Cancelled() JobStatus { return JobStatus(2) } -func (JobStatus) Completed() JobStatus { return JobStatus(3) } +func (JobStatus) Cancelling() JobStatus { return JobStatus(2)} +func (JobStatus) Cancelled() JobStatus { return JobStatus(3) } +func (JobStatus) Completed() JobStatus { return JobStatus(4) } func (js JobStatus) String() string { return EnumHelper{}.StringInteger(js, reflect.TypeOf(js)) } diff --git a/ste/init.go b/ste/init.go index 211100836..bcfb14b66 100644 --- a/ste/init.go +++ b/ste/init.go @@ -101,7 +101,7 @@ func MainSTE(concurrentConnections int, targetRateInMBps int64, azcopyAppPathFol func(writer http.ResponseWriter, request *http.Request) { var payload common.JobID deserialize(request, &payload) - serialize(CancelPauseJobOrder(payload, common.EJobStatus.Cancelled()), writer) + serialize(CancelPauseJobOrder(payload, common.EJobStatus.Cancelling()), writer) }) http.HandleFunc(common.ERpcCmd.PauseJob().Pattern(), func(writer http.ResponseWriter, request *http.Request) { @@ -195,7 +195,7 @@ func CancelPauseJobOrder(jobID common.JobID, desiredJobStatus common.JobStatus) fallthrough case common.EJobStatus.Paused(): // Logically, It's OK to pause an already-paused job fallthrough - case common.EJobStatus.Cancelled(): + case common.EJobStatus.Cancelling(): jpp0.SetJobStatus(desiredJobStatus) msg := fmt.Sprintf("JobID=%v %s", jobID, common.IffString(desiredJobStatus == common.EJobStatus.Paused(), "paused", "canceled")) @@ -380,8 +380,10 @@ func GetJobSummary(jobID common.JobID) common.ListJobSummaryResponse { js.BytesOverWire = uint64(JobsAdmin.BytesOverWire()) // Job is completed if Job order is complete AND ALL transfers are completed/failed // FIX: active or inactive state, then job order is said to be completed if final part of job has been ordered. - if (js.CompleteJobOrdered) && (jp0.Plan().JobStatus() == common.EJobStatus.Completed()) { - js.JobStatus = common.EJobStatus.Completed() + part0PlanStatus := jp0.Plan().JobStatus() + if (js.CompleteJobOrdered) && (part0PlanStatus == common.EJobStatus.Completed() || + part0PlanStatus == common.EJobStatus.Cancelled()) { + js.JobStatus = part0PlanStatus } return js } diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index b85a60501..efcd3d48a 100644 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -141,7 +141,8 @@ func (jm *jobMgr) ReportJobPartDone() uint32 { } switch part0Plan := jobPart0Mgr.Plan(); part0Plan.JobStatus() { - case common.EJobStatus.Cancelled(): + case common.EJobStatus.Cancelling(): + part0Plan.SetJobStatus(common.EJobStatus.Cancelled()) if shouldLog { jm.Log(pipeline.LogInfo, fmt.Sprintf("all parts of Job %v successfully cancelled; cleaning up the Job", jm.jobID)) } diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index 282a405d9..97571b124 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -24,6 +24,7 @@ type IJobPartTransferMgr interface { StartJobXfer() IsForceWriteTrue() bool ReportChunkDone() (lastChunk bool, chunksDone uint32) + TransferStatus()(common.TransferStatus) SetStatus(status common.TransferStatus) SetNumberOfChunks(numChunks uint32) ReportTransferDone() uint32 @@ -146,6 +147,11 @@ func (jptm *jobPartTransferMgr) ReportChunkDone() (lastChunk bool, chunksDone ui return chunksDone == jptm.numChunks, chunksDone } +// +func (jptm *jobPartTransferMgr) TransferStatus()(common.TransferStatus){ + return jptm.jobPartPlanTransfer.TransferStatus() +} + // TransferStatus updates the status of given transfer for given jobId and partNumber func (jptm *jobPartTransferMgr) SetStatus(status common.TransferStatus) { jptm.jobPartPlanTransfer.SetTransferStatus(status, false) diff --git a/ste/xfer-blobToLocal.go b/ste/xfer-blobToLocal.go index 1f9274321..46bbf45dd 100644 --- a/ste/xfer-blobToLocal.go +++ b/ste/xfer-blobToLocal.go @@ -23,7 +23,6 @@ package ste import ( "fmt" "io" - "io/ioutil" "net/url" "os" "strings" @@ -73,21 +72,11 @@ func BlobToLocalPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *p if blobSize == 0 { err := createEmptyFile(info.Destination) if err != nil { - if strings.Contains(err.Error(), "too many open files") { - // dst file could not be created because azcopy process - // reached the open file descriptor limit set for each process. - // Rescheduling the transfer. - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, " rescheduled since process reached open file descriptor limit.") - } - jptm.RescheduleTransfer() - } else { - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) - } - jptm.SetStatus(common.ETransferStatus.Failed()) - jptm.ReportTransferDone() + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) } + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() return } lMTime, plmt := jptm.PreserveLastModifiedTime() @@ -97,7 +86,15 @@ func BlobToLocalPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *p if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" failed while preserving last modified time for destionation %s", info.Destination)) } - return + jptm.SetStatus(common.ETransferStatus.Failed()) + // Since the transfer failed, the file created above should be deleted + err = deleteFile(info.Destination) + if err != nil { + // If there was an error deleting the file, log the error + if jptm.ShouldLog(pipeline.LogError) { + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", info.Destination, err.Error())) + } + } } if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" successfully preserved the last modified time for destinaton %s", info.Destination)) @@ -112,21 +109,11 @@ func BlobToLocalPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *p } else { // 3b: source has content dstFile, err := createFileOfSize(info.Destination, blobSize) if err != nil { - if strings.Contains(err.Error(), "too many open files") { - // dst file could not be created because azcopy process - // reached the open file descriptor limit set for each process. - // Rescheduling the transfer. - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, " rescheduled since process reached open file descriptor limit.") - } - jptm.RescheduleTransfer() - } else { - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) - } - jptm.SetStatus(common.ETransferStatus.Failed()) - jptm.ReportTransferDone() + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) } + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() return } @@ -138,6 +125,14 @@ func BlobToLocalPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *p jptm.Log(pipeline.LogInfo, "transfer failed because dst file did not memory mapped successfully") } jptm.SetStatus(common.ETransferStatus.Failed()) + // Since the transfer failed, the file created above should be deleted + err = deleteFile(info.Destination) + if err != nil { + // If there was an error deleting the file, log the error + if jptm.ShouldLog(pipeline.LogError) { + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", info.Destination, err.Error())) + } + } jptm.ReportTransferDone() return } @@ -158,13 +153,13 @@ func BlobToLocalPrologue(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *p } // schedule the download chunk job - jptm.ScheduleChunks(generateDownloadBlobFunc(jptm, srcBlobURL, blockIdCount, dstMMF, startIndex, adjustedChunkSize, pacer)) + jptm.ScheduleChunks(generateDownloadBlobFunc(jptm, srcBlobURL, blockIdCount, dstMMF, info.Destination, startIndex, adjustedChunkSize, pacer)) blockIdCount++ } } } -func generateDownloadBlobFunc(jptm IJobPartTransferMgr, transferBlobURL azblob.BlobURL, chunkId int32, destinationMMF common.MMF, startIndex int64, adjustedChunkSize int64, p *pacer) chunkFunc { +func generateDownloadBlobFunc(jptm IJobPartTransferMgr, transferBlobURL azblob.BlobURL, chunkId int32, destinationMMF common.MMF, destinationPath string, startIndex int64, adjustedChunkSize int64, p *pacer) chunkFunc { return func(workerId int) { chunkDone := func() { // adding the bytes transferred or skipped of a transfer to determine the progress of transfer. @@ -174,8 +169,20 @@ func generateDownloadBlobFunc(jptm IJobPartTransferMgr, transferBlobURL azblob.B if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" has worker %d which is finalizing cancellation of the Transfer", workerId)) } - jptm.ReportTransferDone() destinationMMF.Unmap() + // If the current transfer status value is less than or equal to 0 + // then transfer either failed or was cancelled + // the file created locally should be deleted + if jptm.TransferStatus() <= 0 { + err := deleteFile(destinationPath) + if err != nil { + // If there was an error deleting the file, log the error + if jptm.ShouldLog(pipeline.LogError) { + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", destinationPath, err.Error())) + } + } + } + jptm.ReportTransferDone() } } if jptm.WasCanceled() { @@ -198,8 +205,6 @@ func generateDownloadBlobFunc(jptm IJobPartTransferMgr, transferBlobURL azblob.B body := get.Body(azblob.RetryReaderOptions{MaxRetryRequests: MaxRetryPerDownloadBody}) body = newResponseBodyPacer(body, p) _, err = io.ReadFull(body, destinationMMF[startIndex:startIndex+adjustedChunkSize]) - io.Copy(ioutil.Discard, body) - body.Close() if err != nil { // cancel entire transfer because this chunk has failed if !jptm.WasCanceled() { @@ -275,6 +280,11 @@ func createEmptyFile(destinationPath string) error { return nil } +// deletes the file +func deleteFile(destinationPath string) error{ + return os.Remove(destinationPath) +} + // create a file, given its path and length func createFileOfSize(destinationPath string, fileSize int64) (*os.File, error) { diff --git a/ste/xfer-fileToLocal.go b/ste/xfer-fileToLocal.go index 6ea9ff6db..1a5e76e15 100644 --- a/ste/xfer-fileToLocal.go +++ b/ste/xfer-fileToLocal.go @@ -26,7 +26,6 @@ import ( "io/ioutil" "net/url" "os" - "strings" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-azcopy/common" @@ -73,21 +72,11 @@ func FileToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { if fileSize == 0 { err := createEmptyFile(info.Destination) if err != nil { - if strings.Contains(err.Error(), "too many open files") { - // dst file could not be created because azcopy process - // reached the open file descriptor limit set for each process. - // Rescheduling the transfer. - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, " rescheduled since process reached open file descriptor limit.") - } - jptm.RescheduleTransfer() - } else { - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) - } - jptm.SetStatus(common.ETransferStatus.Failed()) - jptm.ReportTransferDone() + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) } + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() return } lMTime, plmt := jptm.PreserveLastModifiedTime() @@ -97,6 +86,14 @@ func FileToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" failed while preserving last modified time for destionation %s", info.Destination)) } + //delete the file if transfer failed + err := os.Remove(info.Destination) + if err != nil { + if jptm.ShouldLog(pipeline.LogError){ + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", info.Destination, err.Error())) + } + } + jptm.SetStatus(common.ETransferStatus.Failed()) return } if jptm.ShouldLog(pipeline.LogInfo) { @@ -112,30 +109,26 @@ func FileToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { } else { // 3b: source has content dstFile, err := createFileOfSize(info.Destination, fileSize) if err != nil { - if strings.Contains(err.Error(), "too many open files") { - // dst file could not be created because azcopy process - // reached the open file descriptor limit set for each process. - // Rescheduling the transfer. - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, " rescheduled since process reached open file descriptor limit.") - } - jptm.RescheduleTransfer() - } else { - if jptm.ShouldLog(pipeline.LogInfo) { - jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) - } - jptm.SetStatus(common.ETransferStatus.Failed()) - jptm.ReportTransferDone() + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, "transfer failed because dst file could not be created locally. Failed with error "+err.Error()) } + jptm.SetStatus(common.ETransferStatus.Failed()) + jptm.ReportTransferDone() return } - dstMMF, err := common.NewMMF(dstFile, true, 0, info.SourceSize) if err != nil { dstFile.Close() if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, "transfer failed because dst file did not memory mapped successfully") } + //delete the file if transfer failed + err := os.Remove(info.Destination) + if err != nil { + if jptm.ShouldLog(pipeline.LogError){ + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", info.Destination, err.Error())) + } + } jptm.SetStatus(common.ETransferStatus.Failed()) jptm.ReportTransferDone() return @@ -157,13 +150,13 @@ func FileToLocal(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pacer) { } // schedule the download chunk job - jptm.ScheduleChunks(generateDownloadFileFunc(jptm, srcFileURL, chunkIDCount, dstFile, dstMMF, startIndex, adjustedChunkSize)) + jptm.ScheduleChunks(generateDownloadFileFunc(jptm, srcFileURL, chunkIDCount, info.Destination, dstFile, dstMMF, startIndex, adjustedChunkSize)) chunkIDCount++ } } } -func generateDownloadFileFunc(jptm IJobPartTransferMgr, transferFileURL azfile.FileURL, chunkID int32, destinationFile *os.File, destinationMMF common.MMF, startIndex int64, adjustedChunkSize int64) chunkFunc { +func generateDownloadFileFunc(jptm IJobPartTransferMgr, transferFileURL azfile.FileURL, chunkID int32, destinationPath string, destinationFile *os.File, destinationMMF common.MMF, startIndex int64, adjustedChunkSize int64) chunkFunc { return func(workerId int) { chunkDone := func() { // adding the bytes transferred or skipped of a transfer to determine the progress of transfer. @@ -173,7 +166,6 @@ func generateDownloadFileFunc(jptm IJobPartTransferMgr, transferFileURL azfile.F if jptm.ShouldLog(pipeline.LogInfo) { jptm.Log(pipeline.LogInfo, fmt.Sprintf(" has worker %d which is finalizing cancellation of the Transfer", workerId)) } - jptm.ReportTransferDone() destinationMMF.Unmap() err := destinationFile.Close() if err != nil { @@ -181,6 +173,18 @@ func generateDownloadFileFunc(jptm IJobPartTransferMgr, transferFileURL azfile.F jptm.Log(pipeline.LogInfo, fmt.Sprintf(" has worker %d which failed closing the file %s", workerId, destinationFile.Name())) } } + jptm.ReportTransferDone() + // If the status of transfer is less than or equal to 0 + // then transfer failed or cancelled + // the downloaded file needs to be deleted + if jptm.TransferStatus() <= 0 { + err := os.Remove(destinationPath) + if err != nil { + if jptm.ShouldLog(pipeline.LogError){ + jptm.Log(pipeline.LogError, fmt.Sprintf("error deleting the file %s. Failed with error %s", destinationPath, err.Error())) + } + } + } } } if jptm.WasCanceled() { diff --git a/ste/xfer-localToBlockBlob.go b/ste/xfer-localToBlockBlob.go index 9efad3d5e..64637cdd4 100644 --- a/ste/xfer-localToBlockBlob.go +++ b/ste/xfer-localToBlockBlob.go @@ -32,6 +32,7 @@ import ( "os" "strings" "unsafe" + "net/http" ) type blockBlobUpload struct { @@ -162,6 +163,15 @@ func LocalToBlockBlob(jptm IJobPartTransferMgr, p pipeline.Pipeline, pacer *pace } jptm.Cancel() jptm.SetStatus(common.ETransferStatus.BlobTierFailure()) + // Since transfer failed while setting the page blob tier + // Deleting the created page blob + _, err := pageBlobUrl.Delete(context.TODO(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + // Log the error if deleting the page blob failed. + if jptm.ShouldLog(pipeline.LogInfo) { + jptm.Log(pipeline.LogInfo, fmt.Sprintf("error deleting the page blob %s. Failed with error %s", pageBlobUrl, err.Error())) + } + } jptm.ReportTransferDone() srcMmf.Unmap() return @@ -244,8 +254,23 @@ func (bbu *blockBlobUpload) blockBlobUploadFunc(chunkId int32, startIndex int64, // and the chunkFunc has been changed to the version without param workId // transfer done is internal function which marks the transfer done, unmaps the src file and close the source file. transferDone := func() { - bbu.jptm.ReportTransferDone() bbu.srcMmf.Unmap() + // Get the Status of the transfer + // If the transfer status value < 0, then transfer failed with some failure + // there is a possibility that some uncommitted blocks will be there + // Delete the uncommitted blobs + if bbu.jptm.TransferStatus() <= 0 { + _, err := bbu.blobURL.ToBlockBlobURL().Delete(context.TODO(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if stErr, ok := err.(azblob.StorageError); ok && stErr.Response().StatusCode != http.StatusNotFound{ + // If the delete failed with Status Not Found, then it means there were no uncommitted blocks. + // Other errors report that uncommitted blocks are there + if bbu.jptm.ShouldLog(pipeline.LogInfo) { + bbu.jptm.Log(pipeline.LogInfo, fmt.Sprintf("error occurred while deleting the uncommitted " + + "blocks of blob %s. Failed with error %s", bbu.blobURL.String(), err.Error())) + } + } + } + bbu.jptm.ReportTransferDone() } if bbu.jptm.WasCanceled() { @@ -408,6 +433,14 @@ func PutBlobUploadFunc(jptm IJobPartTransferMgr, srcMmf common.MMF, blockBlobUrl fmt.Sprintf(" failed to set tier %s on blob and failed with error %s", blockBlobTier, string(err.Error()))) } jptm.SetStatus(common.ETransferStatus.BlobTierFailure()) + // since blob tier failed, the transfer failed + // the blob created should be deleted + _, err := blockBlobUrl.Delete(context.TODO(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + if jptm.ShouldLog(pipeline.LogInfo){ + jptm.Log(pipeline.LogInfo, fmt.Sprintf("error deleting the blob %s. Failed with error %s", blockBlobUrl.String(), err.Error())) + } + } } } jptm.SetStatus(common.ETransferStatus.Success()) @@ -439,8 +472,20 @@ func (pbu *pageBlobUpload) pageBlobUploadFunc(startPage int64, calculatedPageSiz fmt.Sprintf("has worker %d which is finalizing transfer", workerId)) } pbu.jptm.SetStatus(common.ETransferStatus.Success()) - pbu.jptm.ReportTransferDone() pbu.srcMmf.Unmap() + // If the value of transfer Status is less than 0 + // transfer failed. Delete the page blob created + if pbu.jptm.TransferStatus() <= 0 { + // Deleting the created page blob + _, err := pbu.blobUrl.ToPageBlobURL().Delete(context.TODO(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + // Log the error if deleting the page blob failed. + if pbu.jptm.ShouldLog(pipeline.LogInfo) { + pbu.jptm.Log(pipeline.LogInfo, fmt.Sprintf("error deleting the page blob %s. Failed with error %s", pbu.blobUrl, err.Error())) + } + } + } + pbu.jptm.ReportTransferDone() } } diff --git a/ste/xfer-localToFile.go b/ste/xfer-localToFile.go index 3fc1f58b8..8a9996eb2 100644 --- a/ste/xfer-localToFile.go +++ b/ste/xfer-localToFile.go @@ -196,7 +196,6 @@ func fileUploadFunc(jptm IJobPartTransferMgr, srcFile *os.File, srcMmf common.MM fmt.Sprintf("has worker %d which is finalizing transfer", workerId)) } jptm.SetStatus(common.ETransferStatus.Success()) - jptm.ReportTransferDone() srcMmf.Unmap() err := srcFile.Close() if err != nil { @@ -205,6 +204,18 @@ func fileUploadFunc(jptm IJobPartTransferMgr, srcFile *os.File, srcMmf common.MM fmt.Sprintf("got an error while closing file %s because of %s", srcFile.Name(), err.Error())) } } + // If the transfer status is less than or equal to 0 + // then transfer was either failed or cancelled + // the file created in share needs to be deleted + if jptm.TransferStatus() <= 0 { + _,err = fileURL.Delete(context.TODO()) + if err != nil { + if jptm.ShouldLog(pipeline.LogError) { + jptm.Log(pipeline.LogInfo, fmt.Sprintf("error deleting the file %s. Failed with error %s", fileURL.String(), err.Error())) + } + } + } + jptm.ReportTransferDone() } } From 12b68a8ba63ffeaf8dc1fb4968e7cdd2eb299573 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Fri, 1 Jun 2018 11:20:23 -0700 Subject: [PATCH 21/23] changed the log file open and initialize code; avoid opening of log files for all job whose part plan file exists --- common/logger.go | 5 +++++ ste/mgr-JobMgr.go | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/common/logger.go b/common/logger.go index 64e34dfbb..00b367cd2 100644 --- a/common/logger.go +++ b/common/logger.go @@ -145,6 +145,11 @@ func (jl *jobLogger) CloseLog() { } func (jl jobLogger) Log(loglevel pipeline.LogLevel, msg string) { + // If the logger for Job is not initialized i.e file is not open + // or logger instance is not initialized, then initialize it + if jl.file == nil || jl.logger == nil { + jl.OpenLog() + } if jl.ShouldLog(loglevel) { jl.logger.Println(msg) } diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index efcd3d48a..0405c0232 100644 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -55,7 +55,9 @@ func newJobMgr(appLogger common.ILogger, jobID common.JobID, appCtx context.Cont } func (jm *jobMgr) reset(appCtx context.Context) IJobMgr { - jm.logger.OpenLog() + // Not opening the log here since it opens the log file for + // all the Jobs existing in the history. + //jm.logger.OpenLog() jm.ctx, jm.cancel = context.WithCancel(appCtx) atomic.StoreUint64(&jm.atomicNumberOfBytesCovered, 0) atomic.StoreUint64(&jm.atomicTotalBytesToXfer, 0) From d8f454ea555bbf76bf026cbc7efcb84833803448 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Fri, 1 Jun 2018 12:37:29 -0700 Subject: [PATCH 22/23] changed the cancellation cases in CancelPauseJobOrder for handling the multiple cancel command from user --- ste/init.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ste/init.go b/ste/init.go index bcfb14b66..d0891b943 100644 --- a/ste/init.go +++ b/ste/init.go @@ -191,11 +191,23 @@ func CancelPauseJobOrder(jobID common.JobID, desiredJobStatus common.JobStatus) CancelledPauseResumed: false, ErrorMsg: fmt.Sprintf("Can't %s JobID=%v because it has already completed", verb, jobID), } + case common.EJobStatus.Cancelled(): + // If the status of Job is cancelled, it means that it has already been cancelled + // No need to cancel further + jr = common.CancelPauseResumeResponse{ + CancelledPauseResumed:false, + ErrorMsg:fmt.Sprintf("cannot cancel the job %s since it is already cancelled", jobID), + } + case common.EJobStatus.Cancelling(): + // If the status of Job is cancelling, it means that it has already been requested for cancellation + // No need to cancel further + jr = common.CancelPauseResumeResponse{ + CancelledPauseResumed:true, + ErrorMsg:fmt.Sprintf("cannot cancel the job %s since it has already been requested for cancellation", jobID), + } case common.EJobStatus.InProgress(): fallthrough case common.EJobStatus.Paused(): // Logically, It's OK to pause an already-paused job - fallthrough - case common.EJobStatus.Cancelling(): jpp0.SetJobStatus(desiredJobStatus) msg := fmt.Sprintf("JobID=%v %s", jobID, common.IffString(desiredJobStatus == common.EJobStatus.Paused(), "paused", "canceled")) From 2af23a04a6103e91aead52dff9b6af764e774bd3 Mon Sep 17 00:00:00 2001 From: Prateek Jain Date: Mon, 4 Jun 2018 16:08:37 -0700 Subject: [PATCH 23/23] add environment variables holding the test suite configuration --- testSuite/scripts/init.py | 53 +++++++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/testSuite/scripts/init.py b/testSuite/scripts/init.py index 66439e84d..2c9f61357 100644 --- a/testSuite/scripts/init.py +++ b/testSuite/scripts/init.py @@ -105,8 +105,7 @@ def execute_user_scenario_file_1() : def execute_user_scenario_2(): test_blob_download_with_special_characters() -def init(): - # initializing config parser to read the testsuite_config file. +def parse_config_file_set_env(): config = configparser.RawConfigParser() files_read = config.read('../test_suite_config.ini') if len(files_read) != 1: @@ -122,26 +121,64 @@ def init(): platform_list.index(osType) except: raise "not able to find the config defined for ostype " + osType + # set all the environment variables + # TEST_DIRECTORY_PATH is the location where test_data folder will be created and test files will be created further. + # set the environment variable TEST_DIRECTORY_PATH + os.environ['TEST_DIRECTORY_PATH'] = config[osType]['TEST_DIRECTORY_PATH'] + + # AZCOPY_EXECUTABLE_PATH is the location of the azcopy executable + # azcopy executable will be copied to test data folder. + # set the environment variables + os.environ['AZCOPY_EXECUTABLE_PATH'] = config[osType]['AZCOPY_EXECUTABLE_PATH'] + + # TEST_SUITE_EXECUTABLE_LOCATION is the location of the test suite executable + # test suite executable will be copied to test data folder. + # set the environment variable TEST_SUITE_EXECUTABLE_LOCATION + os.environ['TEST_SUITE_EXECUTABLE_LOCATION'] = config[osType]['TEST_SUITE_EXECUTABLE_LOCATION'] + + # CONTAINER_SAS_URL is the shared access signature of the container where test data will be uploaded to and downloaded from. + os.environ['CONTAINER_SAS_URL'] = config['CREDENTIALS']['CONTAINER_SAS_URL'] + + # share_sas_url is the URL with SAS of the share where test data will be uploaded to and downloaded from. + os.environ['SHARE_SAS_URL'] = config['CREDENTIALS']['SHARE_SAS_URL'] + + # container sas of the premium storage account. + os.environ['PREMIUM_CONTAINER_SAS_URL'] = config['CREDENTIALS']['PREMIUM_CONTAINER_SAS_URL'] + +def init(): + # Check the environment variables. + # If they are not set, then parse the config file and set + # environment variables. If any of the env variable is not set + # test_config_file is parsed and env variables are reset. + if os.environ.get('TEST_DIRECTORY_PATH', '-1') == '-1' or \ + os.environ.get('AZCOPY_EXECUTABLE_PATH', '-1') == '-1' or \ + os.environ.get('TEST_SUITE_EXECUTABLE_LOCATION', '-1') == '-1' or \ + os.environ.get('CONTAINER_SAS_URL', '-1') == '-1' or \ + os.environ.get('SHARE_SAS_URL', '-1') == '-1' or \ + os.environ.get('PREMIUM_CONTAINER_SAS_URL', '-1') == '-1': + parse_config_file_set_env() + + # Get the environment variables value # test_dir_path is the location where test_data folder will be created and test files will be created further. - test_dir_path = config[osType]['TEST_DIRECTORY_PATH'] + test_dir_path = os.environ.get('TEST_DIRECTORY_PATH') # azcopy_exec_location is the location of the azcopy executable # azcopy executable will be copied to test data folder. - azcopy_exec_location = config[osType]['AZCOPY_EXECUTABLE_PATH'] + azcopy_exec_location = os.environ.get('AZCOPY_EXECUTABLE_PATH') # test_suite_exec_location is the location of the test suite executable # test suite executable will be copied to test data folder. - test_suite_exec_location = config[osType]['TEST_SUITE_EXECUTABLE_LOCATION'] + test_suite_exec_location = os.environ.get('TEST_SUITE_EXECUTABLE_LOCATION') # container_sas is the shared access signature of the container where test data will be uploaded to and downloaded from. - container_sas = config['CREDENTIALS']['CONTAINER_SAS_URL'] + container_sas = os.environ.get('CONTAINER_SAS_URL') # share_sas_url is the URL with SAS of the share where test data will be uploaded to and downloaded from. - share_sas_url = config['CREDENTIALS']['SHARE_SAS_URL'] + share_sas_url = os.environ.get('SHARE_SAS_URL') # container sas of the premium storage account. - premium_container_sas = config['CREDENTIALS']['PREMIUM_CONTAINER_SAS_URL'] + premium_container_sas = os.environ.get('PREMIUM_CONTAINER_SAS_URL') # deleting the log files. for f in glob.glob('*.log'):