From 2aaca9f90f6c8c62c94f07e16a7a6502c7c17a2c Mon Sep 17 00:00:00 2001 From: normesta Date: Thu, 3 Oct 2019 11:38:25 -0700 Subject: [PATCH 1/6] Final set of edits --- cmd/copy.go | 78 +++++----- cmd/env.go | 2 +- cmd/helpMessages.go | 345 +++++++++++++++++++++++++------------------- cmd/jobsResume.go | 4 +- cmd/jobsShow.go | 2 +- cmd/list.go | 6 +- cmd/login.go | 18 +-- cmd/make.go | 2 +- cmd/remove.go | 18 +-- cmd/root.go | 4 +- cmd/sync.go | 22 +-- 11 files changed, 277 insertions(+), 224 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index d187ccc71..286afd9f2 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1280,48 +1280,48 @@ func init() { rootCmd.AddCommand(cpCmd) // filters change which files get transferred - cpCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when uploading from local file system.") - cpCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "only include these files when copying. "+ - "Support use of *. Files should be separated with ';'.") - cpCmd.PersistentFlags().StringVar(&raw.includePath, "include-path", "", "only include these paths when copying. "+ - "Does not support using wildcards. Checks relative path prefix. ex. myFolder;myFolder/subDirName/file.pdf") - cpCmd.PersistentFlags().StringVar(&raw.excludePath, "exclude-path", "", "exclude these paths when copying. "+ - "Does not support using wildcards. Checks relative path prefix. ex. myFolder;myFolder/subDirName/file.pdf") + cpCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "Follow symbolic links when uploading from local file system.") + cpCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "Include only these files when copying. "+ + "This option supports wildcard characters (*). Separate files by using a ';'.") + cpCmd.PersistentFlags().StringVar(&raw.includePath, "include-path", "", "Include only these paths when copying. "+ + "This option does not support wildcard characters (*). Checks relative path prefix (For example: myFolder;myFolder/subDirName/file.pdf).") + cpCmd.PersistentFlags().StringVar(&raw.excludePath, "exclude-path", "", "Exclude these paths when copying. "+ + "This option does not support wildcard characters (*). Checks relative path prefix(For example: myFolder;myFolder/subDirName/file.pdf).") // This flag is implemented only for Storage Explorer. - cpCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "defines the location of json which has the list of only files to be copied") - cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "exclude these files when copying. Support use of *.") - cpCmd.PersistentFlags().StringVar(&raw.forceWrite, "overwrite", "true", "defines whether to overwrite the conflicting files at the destination. Could be set to true, false, or prompt.") - cpCmd.PersistentFlags().BoolVar(&raw.autoDecompress, "decompress", false, "automatically decompress files when downloading, if their content-encoding indicates that they are compressed. The supported content-encoding values are 'gzip' and 'deflate'. File extensions of '.gz'/'.gzip' or '.zz' aren't necessary, but will be removed if present.") - cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when uploading from local file system.") - cpCmd.PersistentFlags().StringVar(&raw.fromTo, "from-to", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") - cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "exclude-blob-type", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+ - "this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';' ") + cpCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "Defines the location of json which has the list of only files to be copied.") + cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "Exclude these files when copying. This option supports wildcard characters (*)") + cpCmd.PersistentFlags().StringVar(&raw.forceWrite, "overwrite", "true", "Overwrite the conflicting files/blobs at the destination if this flag is set to true. (default true).") + cpCmd.PersistentFlags().BoolVar(&raw.autoDecompress, "decompress", false, "Automatically decompress files when downloading, if their content-encoding indicates that they are compressed. The supported content-encoding values are 'gzip' and 'deflate'. File extensions of '.gz'/'.gzip' or '.zz' aren't necessary, but will be removed if present.") + cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Look into sub-directories recursively when uploading from local file system.") + cpCmd.PersistentFlags().StringVar(&raw.fromTo, "from-to", "", "Optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.") + cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "exclude-blob-type", "", "Optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from the container "+ + "or the account. Use of this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';'. ") // options change how the transfers are performed - cpCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "use this block size (specified in MiB) when uploading to/downloading from Azure Storage. Default is automatically calculated based on file size. Decimal fractions are allowed - e.g. 0.25") - cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs).") - cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "defines the type of blob at the destination. This is used in case of upload / account to account copy. Use --blob-type detect for auto-detection of VHD and VHDX files as page blobs when no source blob type is available. (For instance, a VHD from local/Azure Files/S3 is detected as a page blob, but a VHD from blob would be detected as its source type)") + cpCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "Use this block size (specified in MiB) when uploading to Azure Storage, and downloading from Azure Storage. The default value is automatically calculated based on file size. Decimal fractions are allowed (For example: 0.25).") + cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').") + cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "Defines the type of blob at the destination. This is used for uploading blobs and when copying between accounts (default 'None').") cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.") - cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "upload page blob to Azure Storage using this blob tier.") - cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "upload to Azure Storage with these key-value pairs as metadata.") - cpCmd.PersistentFlags().StringVar(&raw.contentType, "content-type", "", "specifies content type of the file. Implies no-guess-mime-type. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "set the content-encoding header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentDisposition, "content-disposition", "", "set the content-disposition header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentLanguage, "content-language", "", "set the content-language header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.cacheControl, "cache-control", "", "set the cache-control header. Returned on download.") - cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "prevents AzCopy from detecting the content-type based on the extension/content of the file.") - cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "only available when destination is file system.") - cpCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob/file. (By default the hash is NOT created.) Only available when uploading.") - cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") - - cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, fail the transfer.") - cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "preserve full properties during service to service copy. "+ - "For S3 and Azure File non-single file source, as list operation doesn't return full properties of objects/files, to preserve full properties AzCopy needs to send one additional request per object/file.") - cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "preserve access tier during service to service copy. "+ - "please refer to https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-storage-tiers to ensure destination storage account supports setting access tier. "+ - "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. ") - cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "check if source has changed after enumerating. "+ + cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "Upload page blob to Azure Storage using this blob tier. (default 'None').") + cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "Upload to Azure Storage with these key-value pairs as metadata.") + cpCmd.PersistentFlags().StringVar(&raw.contentType, "content-type", "", "Specifies the content type of the file. Implies no-guess-mime-type. Returned on download.") + cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "Set the content-encoding header. Returned on download.") + cpCmd.PersistentFlags().StringVar(&raw.contentDisposition, "content-disposition", "", "Set the content-disposition header. Returned on download.") + cpCmd.PersistentFlags().StringVar(&raw.contentLanguage, "content-language", "", "Set the content-language header. Returned on download.") + cpCmd.PersistentFlags().StringVar(&raw.cacheControl, "cache-control", "", "Set the cache-control header. Returned on download.") + cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "Prevents AzCopy from detecting the content-type based on the extension or content of the file.") + cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "Only available when destination is file system.") + cpCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. (By default the hash is NOT created.) Only available when uploading.") + cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent')") + + cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, the transfer is cancelled.") + cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "Preserve full properties during service to service copy. "+ + "For AWS S3 and Azure File non-single file source, the list operation doesn't return full properties of objects amd files. To preserve full properties, AzCopy needs to send one additional request per object or file.") + cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "Preserve access tier during service to service copy. "+ + "Please refer to [Azure Blob storage: hot, cool, and archive access tiers](https://docs.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers) to ensure destination storage account supports setting access tier. "+ + "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. (default true). ") + cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "Check if source has changed after enumerating. "+ "For S2S copy, as source is a remote resource, validating whether source has changed need additional request costs. ") - cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "specifies how invalid metadata keys are handled. AvailabeOptions: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid.") + cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').") // s2sGetPropertiesInBackend is an optional flag for controlling whether S3 object's or Azure file's full properties are get during enumerating in frontend or // right before transferring in ste(backend). @@ -1339,6 +1339,6 @@ func init() { cpCmd.PersistentFlags().MarkHidden("s2s-get-properties-in-backend") // Hide the flush-threshold flag since it is implemented only for CI. - cpCmd.PersistentFlags().Uint32Var(&ste.ADLSFlushThreshold, "flush-threshold", 7500, "Adjust the number of blocks to flush at once on ADLS gen 2") + cpCmd.PersistentFlags().Uint32Var(&ste.ADLSFlushThreshold, "flush-threshold", 7500, "Adjust the number of blocks to flush at once on accounts that have a hierarchical namespace.") cpCmd.PersistentFlags().MarkHidden("flush-threshold") } diff --git a/cmd/env.go b/cmd/env.go index d34e21fc1..ea159950d 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -43,6 +43,6 @@ var envCmd = &cobra.Command{ } func init() { - envCmd.PersistentFlags().BoolVar(&showSensitive, "show-sensitive", false, "Show sensitive/secret environment variables") + envCmd.PersistentFlags().BoolVar(&showSensitive, "show-sensitive", false, "Shows sensitive/secret environment variables.") rootCmd.AddCommand(envCmd) } diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index 1a4267bdf..98382819f 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -3,13 +3,13 @@ package cmd import "github.com/Azure/azure-storage-azcopy/common" // ===================================== ROOT COMMAND ===================================== // -const rootCmdShortDescription = "AzCopy is a command line tool that moves data into/out of Azure Storage." +const rootCmdShortDescription = "AzCopy is a command line tool that moves data into and out of Azure Storage." const rootCmdLongDescription = "AzCopy " + common.AzcopyVersion + ` Project URL: github.com/Azure/azure-storage-azcopy -AzCopy is a command line tool that moves data into/out of Azure Storage. +AzCopy is a command line tool that moves data into and out of Azure Storage. To report issues or to learn more about the tool, go to github.com/Azure/azure-storage-azcopy The general format of the commands is: 'azcopy [command] [arguments] --[flag-name]=[flag-value]'. @@ -30,100 +30,122 @@ Copies source data to a destination location. The supported directions are: Please refer to the examples for more information. Advanced: -Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). -The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: - - /etc/mime.types - - /etc/apache2/mime.types - - /etc/apache/mime.types +AzCopy automatically detects the content type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). + +The built-in lookup table is small, but on Unix, it is augmented by the local system's mime.types file(s) if available under one or more of these names: + +- /etc/mime.types +- /etc/apache2/mime.types +- /etc/apache/mime.types On Windows, MIME types are extracted from the registry. This feature can be turned off with the help of a flag. Please refer to the flag section. ` + environmentVariableNotice -const copyCmdExample = `Upload a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: -- azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" +const copyCmdExample = `Upload a single file by using OAuth authentication. If you have not yet logged into AzCopy, please use azcopy login command before you run the following command. -Same as above, but this time also compute MD5 hash of the file content and save it as the blob's Content-MD5 property. -- azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" --put-md5 + - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" + +Same as above, but this time also compute MD5 hash of the file content and save it as the blob's Content-MD5 property: + + - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" --put-md5 + +Upload a single file by using a SAS token: -Upload a single file with a SAS: - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload a single file with a SAS using piping (block blobs only): +Upload a single file by using a SAS token and piping (block blobs only): + - cat "/path/to/file.txt" | azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Upload an entire directory with a SAS: +Upload an entire directory by using a SAS token: + - azcopy cp "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true or - azcopy cp "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --put-md5 -Upload a set of files with a SAS using wildcards: +Upload a set of files by using a SAS token and wildcard (*) characters: + - azcopy cp "/path/*foo/*bar/*.pdf" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" -Upload files and directories with a SAS using wildcards: +Upload files and directories by using a SAS token and wildcard (*) characters: + - azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Download a single file using OAuth authentication. Please use 'azcopy login' command first if you aren't logged in yet: +Download a single file by using OAuth authentication. If you have not yet logged into AzCopy, please use azcopy login command before you run the following command. + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]" "/path/to/file.txt" -Download a single file with a SAS: +Download a single file by using a SAS token: + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "/path/to/file.txt" -Download a single file with a SAS using piping (block blobs only): +Download a single file by using a SAS token and then piping the output to a file (block blobs only): + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" > "/path/to/file.txt" -Download an entire directory with a SAS: +Download an entire directory by using a SAS token: + - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "/path/to/dir" --recursive=true -A note about wildcards in URLs: +A note about using a wildcard character (*) in URLs: -The only usage of a wildcard in a URL that is supported is the final, trailing /*, and in the container name when not specifying a blob name. -If you happen to use the character * in the name of a blob, please manually encode it to %2A to avoid it being treated as a wildcard character. +There's only two supported ways to use a wildcard character in a URL. +You can use one just after the final forward slash (/) of a URL to obtain all of the files in a directory. +You can also use one in the name of a container as long as the URL refers only to a container and not to a blob. -Download the contents of a folder directly to the destination (rather than under a sub-directory): - - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/folder]/*?[SAS]" "/path/to/dir" +Download the contents of a directory without copying the containing directory itself. + + - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/folder]/*?[SAS]" "/path/to/dir" -Download an entire storage account at once - - azcopy cp "https://[srcaccount].blob.core.windows.net/" "/path/to/dir" --recursive +Download an entire storage account. -Download an entire storage account at once with a wildcarded container name - - azcopy cp "https://[srcaccount].blob.core.windows.net/[container*name]" "/path/to/dir" --recursive + - azcopy cp "https://[srcaccount].blob.core.windows.net/" "/path/to/dir" --recursive + +Download an entire storage account by using a wildcard symbol (*) in the container name. + + - azcopy cp "https://[srcaccount].blob.core.windows.net/[container*name]" "/path/to/dir" --recursive + +Copy a single blob to another blob by using a SAS token. -Copy a single blob with SAS to another blob with SAS: - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Copy a single blob with SAS to another blob with OAuth token. Please use 'azcopy login' command first if you aren't logged in yet. Note that the OAuth token is used to access the destination storage account: +Copy a single blob to another blob by using a SAS token and an OAuth token. If you log into AzCopy by using the azcopy login command, the destination account doesn't need a SAS token. + - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]" -Copy an entire directory from blob virtual directory with SAS to another blob virtual directory with SAS: +Copy one blob virtual directory to another by using a SAS token: + - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Copy an entire account data from blob account with SAS to another blob account with SAS: +Copy all blob containers, directories, and blobs from storage account to another by using a SAS token: + - azcopy cp "https://[srcaccount].blob.core.windows.net?[SAS]" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true -Copy a single object from S3 with access key to blob with SAS: - - Set environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 source. +Copy a single object to Blob Storage from Amazon Web Services (AWS) S3 by using an access key and a SAS token. First, set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. + - azcopy cp "https://s3.amazonaws.com/[bucket]/[object]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Copy an entire directory from S3 with access key to blob virtual directory with SAS: - - Set environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 source. +Copy an entire directory to Blob Storage from AWS S3 by using an access key and a SAS token. First, set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. + - azcopy cp "https://s3.amazonaws.com/[bucket]/[folder]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true - - Please refer to https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html for what [folder] means for S3. + + Please refer to https://docs.aws.amazon.com/AmazonS3/latest/user-guide/using-folders.html to better understand the [folder] placeholder. -Copy all buckets in S3 service with access key to blob account with SAS: - - Set environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 source. +Copy all buckets to Blob Storage from Amazon Web Services (AWS) by using an access key and a SAS token. First, set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. + - azcopy cp "https://s3.amazonaws.com/" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true -Copy all buckets in a S3 region with access key to blob account with SAS: - - Set environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for S3 source. +Copy all buckets to Blob Storage from an Amazon Web Services (AWS) region by using an access key and a SAS token. First, set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. + - azcopy cp "https://s3-[region].amazonaws.com/" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true ` // ===================================== ENV COMMAND ===================================== // -const envCmdShortDescription = "Shows the environment variables that can configure AzCopy's behavior" +const envCmdShortDescription = "Shows the environment variables that you can use to configure the behavior of AzCopy." -const envCmdLongDescription = `Shows the environment variables that can configure AzCopy's behavior. +const envCmdLongDescription = `Shows the environment variables that you can use to configure the behavior of AzCopy. ` + environmentVariableNotice @@ -134,162 +156,177 @@ const jobsCmdLongDescription = "Sub-commands related to managing jobs." const jobsCmdExample = "azcopy jobs show [jobID]" -const listJobsCmdShortDescription = "Display information on all jobs" +const listJobsCmdShortDescription = "Displays information on all jobs" const listJobsCmdLongDescription = ` -Display information on all jobs.` +Displays information on all jobs.` const showJobsCmdShortDescription = "Show detailed information for the given job ID" const showJobsCmdLongDescription = ` -Show detailed information for the given job ID: if only the job ID is supplied without a flag, then the progress summary of the job is returned. -(In the progress information shown by this command, the byte counts and percent complete do not include files currently in progress.) -If the with-status flag is set, then the list of transfers in the job with the given value will be shown.` +If you provide only a job ID, and not a flag, then this command returns the progress summary only. +The byte counts and percent complete that appears when you run this command don't reflect files related to jobs that are currently in progress. +If you set the with-status flag, then only the list of transfers associated with the given status appear.` -const resumeJobsCmdShortDescription = "Resume the existing job with the given job ID" +const resumeJobsCmdShortDescription = "Resume the existing job with the given job ID." const resumeJobsCmdLongDescription = ` Resume the existing job with the given job ID.` -const removeJobsCmdShortDescription = "Remove all files associated with the given job ID" +const removeJobsCmdShortDescription = "Remove all files associated with the given job ID." const removeJobsCmdLongDescription = ` Remove all files associated with the given job ID. -Note that the location of log files and job plan files can be customized. Please refer to the env command.` +Note that you can customize the location where log and plan files are saved. See the env command to learn more.` const removeJobsCmdExample = " azcopy jobs rm e52247de-0323-b14d-4cc8-76e0be2e2d44" -const cleanJobsCmdShortDescription = "Clean up log files and job plan files (used for progress tracking and resuming) for jobs" +const cleanJobsCmdShortDescription = "Remove all log and plan files for all jobs" const cleanJobsCmdLongDescription = ` -Clean up log files and job plan files (used for progress tracking and resuming) for jobs. - -Note that the location of log files and job plan files can be customized. Please refer to the env command.` +Note that you can customize the location where log and plan files are saved. See the env command to learn more.` const cleanJobsCmdExample = " azcopy jobs clean --with-status=completed" // ===================================== LIST COMMAND ===================================== // const listCmdShortDescription = "List the entities in a given resource" -const listCmdLongDescription = `List the entities in a given resource. Only Blob containers are supported at the moment.` +const listCmdLongDescription = `List the entities in a given resource. In the current release, only Blob containers are supported.` const listCmdExample = "azcopy list [containerURL]" // ===================================== LOGIN COMMAND ===================================== // -const loginCmdShortDescription = "Log in to Azure Active Directory to access Azure Storage resources." +const loginCmdShortDescription = "Log in to Azure Active Directory (AD) to access Azure Storage resources." -const loginCmdLongDescription = `Log in to Azure Active Directory to access Azure Storage resources. -Note that, to be authorized to your Azure Storage account, you must assign your user 'Storage Blob Data Contributor' role on the Storage account. +const loginCmdLongDescription = `To be authorized to your Azure Storage account, you must assign the **Storage Blob Data Contributor** role to your user account in the context of either the Storage account, parent resource group or parent subscription. This command will cache encrypted login information for current user using the OS built-in mechanisms. Please refer to the examples for more information. ` + environmentVariableNotice -const environmentVariableNotice = "(NOTICE FOR SETTING ENVIRONMENT VARIABLES: Bear in mind that setting an environment variable from the command line " + - "will be readable in your command line history. " + - "For variables that contain credentials, consider clearing these entries from your history " + - "or using a small script of sorts to prompt for and set these variables.)" +const environmentVariableNotice = "If you set an environment variable by using the command line, that variable will be readable in your command line history. " + + "Consider clearing variables that contain credentials from your command line history. " + + "To keep variables from appearing in your history, you can use a script to prompt the user for their credentials, and to set the environment variable." const loginCmdExample = `Log in interactively with default AAD tenant ID set to common: - azcopy login Log in interactively with a specified tenant ID: -- azcopy login --tenant-id "[TenantID]" -Log in using a VM's system-assigned identity: -- azcopy login --identity + - azcopy login --tenant-id "[TenantID]" + +Log in by using the system-assigned identity of a Virtual Machine (VM): + + - azcopy login --identity + +Log in by using the user-assigned identity of a VM and a Client ID of the service identity: + + - azcopy login --identity --identity-client-id "[ServiceIdentityClientID]" + +Log in by using the user-assigned identity of a VM and an Object ID of the service identity: + + - azcopy login --identity --identity-object-id "[ServiceIdentityObjectID]" -Log in using a VM's user-assigned identity with a Client ID of the service identity: -- azcopy login --identity --identity-client-id "[ServiceIdentityClientID]" +Log in by using the user-assigned identity of a VM and a Resource ID of the service identity: + + - azcopy login --identity --identity-resource-id "/subscriptions//resourcegroups/myRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myID" -Log in using a VM's user-assigned identity with an Object ID of the service identity: -- azcopy login --identity --identity-object-id "[ServiceIdentityObjectID]" +Log in as a service principal by using a client secret: +Set the environment variable AZCOPY_SPA_CLIENT_SECRET to the client secret for secret based service principal auth. -Log in using a VM's user-assigned identity with a Resource ID of the service identity: -- azcopy login --identity --identity-resource-id "/subscriptions//resourcegroups/myRG/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myID" + - azcopy login --service-principal -Log in as a service principal using a client secret: -- Set the environment variable AZCOPY_SPA_CLIENT_SECRET to the client secret for secret based service principal auth -- azcopy login --service-principal +Log in as a service principal by using a certificate and a password: +Set the environment variable AZCOPY_SPA_CERT_PASSWORD to the certificate's password for cert based service principal auth -Log in as a service principal using a certificate & password: -- Set the environment variable AZCOPY_SPA_CERT_PASSWORD to the certificate's password for cert based service principal auth -- azcopy login --service-principal --certificate-path /path/to/my/cert -Please treat /path/to/my/cert as a path to a PEM or PKCS12 file-- AzCopy does not reach into the system cert store to obtain your certificate. ---certificate-path is mandatory when doing cert-based service principal auth. + - azcopy login --service-principal --certificate-path /path/to/my/cert + + Please treat /path/to/my/cert as a path to a PEM or PKCS12 file-- AzCopy does not reach into the system cert store to obtain your certificate. + --certificate-path is mandatory when doing cert-based service principal auth. ` // ===================================== LOGOUT COMMAND ===================================== // const logoutCmdShortDescription = "Log out to terminate access to Azure Storage resources." -const logoutCmdLongDescription = `Log out to terminate access to Azure Storage resources. -This command will remove all the cached login information for the current user.` +const logoutCmdLongDescription = `This command will remove all of the cached login information for the current user.` // ===================================== MAKE COMMAND ===================================== // -const makeCmdShortDescription = "Create a container/share/filesystem" +const makeCmdShortDescription = "Create a container or file share." -const makeCmdLongDescription = `Create a container/share/filesystem represented by the given resource URL.` +const makeCmdLongDescription = `Create a container or file share represented by the given resource URL.` const makeCmdExample = ` - azcopy make "https://[account-name].[blob,file,dfs].core.windows.net/[top-level-resource-name]" ` // ===================================== REMOVE COMMAND ===================================== // -const removeCmdShortDescription = "Delete entities from Azure Storage Blob/File/ADLS Gen2" +const removeCmdShortDescription = "Delete blobs or files from an Azure storage account" -const removeCmdLongDescription = `Delete entities from Azure Storage Blob/File/ADLS Gen2.` +const removeCmdLongDescription = `"Delete blobs or files from an Azure storage account"` const removeCmdExample = ` -Remove a single blob with SAS: - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" +Remove a single blob by using a SAS token: + + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" + +Remove an entire virtual directory by using a SAS token: + + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Remove an entire virtual directory with a SAS: - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true +Remove only the blobs inside of a virtual directory, but don't remove any subdirectories or blobs within those subdirectories: -Remove only the top blobs inside a virtual directory but not its sub-directories: - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false -Remove a subset of blobs in a virtual directory (ex: only jpg and pdf files, or if the blob name is "exactName"): - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --include="*.jpg;*.pdf;exactName" +Remove a subset of blobs in a virtual directory (For example: remove only jpg and pdf files, or if the blob name is "exactName"): -Remove an entire virtual directory but exclude certain blobs from the scope (ex: every blob that starts with foo or ends with bar): - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --exclude="foo*;*bar" + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --include="*.jpg;*.pdf;exactName" + +Remove an entire virtual directory but exclude certain blobs from the scope (For example: every blob that starts with foo or ends with bar): + + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true --exclude="foo*;*bar" Remove specific blobs and virtual directories by putting their relative paths (NOT URL-encoded) in a file: - - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/parent/dir]" --recursive=true --list-of-files=/usr/bar/list.txt - - file content: - dir1/dir2 - blob1 - blob2 -Remove a single file from ADLS Gen2 (include/exclude not supported): - - azcopy rm "https://[account].dfs.core.windows.net/[container]/[path/to/file]?[SAS]" + - azcopy rm "https://[account].blob.core.windows.net/[container]/[path/to/parent/dir]" --recursive=true --list-of-files=/usr/bar/list.txt + - file content: + dir1/dir2 + blob1 + blob2 + +Remove a single file from a Blob Storage account that has a hierarchical namespace (include/exclude not supported): -Remove a single directory from ADLS Gen2 (include/exclude not supported): - - azcopy rm "https://[account].dfs.core.windows.net/[container]/[path/to/directory]?[SAS]" + - azcopy rm "https://[account].dfs.core.windows.net/[container]/[path/to/file]?[SAS]" + +Remove a single directory from a Blob Storage account that has a hierarchical namespace (include/exclude not supported): + + - azcopy rm "https://[account].dfs.core.windows.net/[container]/[path/to/directory]?[SAS]" ` // ===================================== SYNC COMMAND ===================================== // const syncCmdShortDescription = "Replicate source to the destination location" const syncCmdLongDescription = ` -Replicate a source to a destination location. The last modified times are used for comparison, the file is skipped if the last modified time in the destination is more recent. The supported pairs are: +The last modified times are used for comparison. The file is skipped if the last modified time in the destination is more recent. The supported pairs are: + - local <-> Azure Blob (either SAS or OAuth authentication can be used) - Azure Blob <-> Azure Blob (Source must include a SAS or is publicly accessible; either SAS or OAuth authentication can be used for destination) - Azure File <-> Azure File (Source must include a SAS or is publicly accessible; SAS authentication should be used for destination) -Please note that the sync command differs from the copy command in several ways: - 0. The recursive flag is on by default. - 1. The source and destination should not contain patterns(such as * or ?). - 2. The include/exclude flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. - 3. If there are files/blobs at the destination that are not present at the source, the user will be prompted to delete them. This prompt can be silenced by using the corresponding flags to automatically answer the deletion question. - 4. When syncing between virtual directories, add a trailing slash to the path (refer to examples) if there's a blob with the same name as one of the virtual directories +The sync command differs from the copy command in several ways: + + 1. The recursive flag is on by default. + 2. The source and destination should not contain patterns (For example: * or ?). + 3. The include and exclude flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. + 4. If there are files and blobs at the destination that are not present at the source, the user is prompted to delete them. You can use the corresponding flags to silence the prompt to automatically answer the deletion question. + 5. When syncing between virtual directories, add a trailing slash to the path (refer to examples) if there's a blob with the same name as one of the virtual directories. Advanced: -Please note that AzCopy automatically detects the Content Type of the files when uploading from the local disk, based on the file extension or content (if no extension is specified). + +Please note that if you don't specify a file extension, AzCopy automatically detects the content type of the files when uploading from the local disk, based on the file extension or content. The built-in lookup table is small but on Unix it is augmented by the local system's mime.types file(s) if available under one or more of these names: + - /etc/mime.types - /etc/apache2/mime.types - /etc/apache/mime.types @@ -299,38 +336,48 @@ On Windows, MIME types are extracted from the registry. const syncCmdExample = ` Sync a single file: - - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" -Same as above, but this time also compute MD5 hash of the file content and save it as the blob's Content-MD5 property. - - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" --put-md5 + - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" + +Same as above, but also compute an MD5 hash of the file content, and then save that MD5 hash as the blob's Content-MD5 property. + + - azcopy sync "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" --put-md5 -Sync an entire directory including its sub-directories (note that recursive is by default on): - - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" +Sync an entire directory including its subdirectories (note that recursive is by default on): + + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" or - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --put-md5 -Sync only the top files inside a directory but not its sub-directories: - - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false +Sync only the files inside of a directory but not subdirectories or the files inside of subdirectories: + + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=false + +Sync a subset of files in a directory (For example: only jpg and pdf files, or if the file name is "exactName"): -Sync a subset of files in a directory (ex: only jpg and pdf files, or if the file name is "exactName"): - - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --include="*.jpg;*.pdf;exactName" + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --include="*.jpg;*.pdf;exactName" -Sync an entire directory but exclude certain files from the scope (ex: every file that starts with foo or ends with bar): - - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --exclude="foo*;*bar" +Sync an entire directory but exclude certain files from the scope (For example: every file that starts with foo or ends with bar): + + - azcopy sync "/path/to/dir" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --exclude="foo*;*bar" Sync a single blob: - - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" + + - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" Sync a virtual directory: - - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=true -Sync a virtual directory sharing the same name as a blob (add a trailing slash to the path in order to disambiguate): - - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]/?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]/" --recursive=true + - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]" --recursive=true + +Sync a virtual directory that has the same name as a blob (add a trailing slash to the path in order to disambiguate): + + - azcopy sync "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]/?[SAS]" "https://[account].blob.core.windows.net/[container]/[path/to/virtual/dir]/" --recursive=true Sync an Azure File directory (same syntax as Blob): - - azcopy sync "https://[account].file.core.windows.net/[share]/[path/to/dir]?[SAS]" "https://[account].file.core.windows.net/[share]/[path/to/dir]" --recursive=true -Note: if include/exclude flags are used together, only files matching the include patterns would be looked at, but those matching the exclude patterns would be always be ignored. + - azcopy sync "https://[account].file.core.windows.net/[share]/[path/to/dir]?[SAS]" "https://[account].file.core.windows.net/[share]/[path/to/dir]" --recursive=true + +Note: if include and exclude flags are used together, only files matching the include patterns are used, but those matching the exclude patterns are ignored. ` // ===================================== DOC COMMAND ===================================== // @@ -348,34 +395,40 @@ const benchCmdShortDescription = "Performs a performance benchmark" // TODO: document whether we delete the uploaded data const benchCmdLongDescription = ` -Runs a performance benchmark, by uploading uploading test data to a specified destination. -The test data is automatically generated. +Runs a performance benchmark by uploading test data to a specified destination. The test data is automatically generated. The benchmark command runs the same upload process as 'copy', except that: - - there's no source parameter. The command requires only a destination (which must be a blob container, - in the initial release of the benchmark command) - - the payload is described by command line parameters, which control how many files are auto-generated and + + - There's no source parameter. The command requires only a destination URL. In the current release, this destination URL must refer to a blob container. + + - The payload is described by command line parameters, which control how many files are auto-generated and how big they are. The generation process takes place entirely in memory. Disk is not used. - - only a few of 'copy's optional parameters are supported - - additional diagnostics are measured and reported - - by default, the transferred data is deleted at the end of the test run + + - Only a few of the optional parameters that are available to the copy command are supported. + + - Additional diagnostics are measured and reported. + + - By default, the transferred data is deleted at the end of the test run. Benchmark mode will automatically tune itself to the number of parallel TCP connections that gives the maximum throughput. It will display that number at the end. To prevent auto-tuning, set the AZCOPY_CONCURRENCY_VALUE environment variable to a specific number of connections. -All the usual authentication types are supported, however the most convenient approach for benchmarking is typically +All the usual authentication types are supported. However, the most convenient approach for benchmarking is typically to create an empty container with a SAS token and use SAS authentication. ` -const benchCmdExample = `Run a benchmark with default parameters (suitable for benchmarking networks up to 1 Gbps):' -- azcopy bench "https://[account].blob.core.windows.net/[container]?" +const benchCmdExample = `Run a benchmark test with default parameters (suitable for benchmarking networks up to 1 Gbps):' -Run a benchmark uploading 100 files, each 2 GiB in size: (suitable for benchmarking on a fast network, e.g. 10 Gbps):' -- azcopy bench "https://[account].blob.core.windows.net/[container]?" --file-count 100 --size-per-file 2G + - azcopy bench "https://[account].blob.core.windows.net/[container]?" -Same as above, but this time use 50,000 files, each 8 MiB in size and compute their MD5 hashes (the same way that --put-md5 does +Run a benchmark test that uploads 100 files, each 2 GiB in size: (suitable for benchmarking on a fast network, e.g. 10 Gbps):' + + - azcopy bench "https://[account].blob.core.windows.net/[container]?" --file-count 100 --size-per-file 2G + +Same as above, but use 50,000 files, each 8 MiB in size and compute their MD5 hashes (in the same way that the --put-md5 flag does this in the copy command). The purpose of --put-md5 when benchmarking is to test whether MD5 computation affects throughput for the selected file count and size: -- azcopy bench "https://[account].blob.core.windows.net/[container]?" --file-count 50000 --size-per-file 8M --put-md5 + + - azcopy bench "https://[account].blob.core.windows.net/[container]?" --file-count 50000 --size-per-file 8M --put-md5 ` diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index 6968e2361..8b73e7416 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -196,8 +196,8 @@ func init() { resumeCmd.PersistentFlags().StringVar(&resumeCmdArgs.excludeTransfer, "exclude", "", "Filter: exclude these failed transfer(s) when resuming the job. "+ "Files should be separated by ';'.") // oauth options - resumeCmd.PersistentFlags().StringVar(&resumeCmdArgs.SourceSAS, "source-sas", "", "source sas of the source for given JobId") - resumeCmd.PersistentFlags().StringVar(&resumeCmdArgs.DestinationSAS, "destination-sas", "", "destination sas of the destination for given JobId") + resumeCmd.PersistentFlags().StringVar(&resumeCmdArgs.SourceSAS, "source-sas", "", "Source SAS token of the source for a given Job ID.") + resumeCmd.PersistentFlags().StringVar(&resumeCmdArgs.DestinationSAS, "destination-sas", "", "destination SAS token of the destination for a given Job ID.") } type resumeCmdArgs struct { diff --git a/cmd/jobsShow.go b/cmd/jobsShow.go index b9d5655ba..c0a55389d 100644 --- a/cmd/jobsShow.go +++ b/cmd/jobsShow.go @@ -73,7 +73,7 @@ func init() { jobsCmd.AddCommand(shJob) // filters - shJob.PersistentFlags().StringVar(&commandLineInput.OfStatus, "with-status", "", "only list the transfers of job with this status, available values: Started, Success, Failed") + shJob.PersistentFlags().StringVar(&commandLineInput.OfStatus, "with-status", "", "Only list the transfers of job with this status, available values: Started, Success, Failed.") } // handles the list command diff --git a/cmd/list.go b/cmd/list.go index 4fb673ac1..6e1c1774d 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -73,9 +73,9 @@ func init() { }, } - listContainerCmd.PersistentFlags().BoolVar(¶meters.MachineReadable, "machine-readable", false, "Lists file sizes in bytes") - listContainerCmd.PersistentFlags().BoolVar(¶meters.RunningTally, "running-tally", false, "Counts the total number of files & their sizes") - listContainerCmd.PersistentFlags().BoolVar(¶meters.MegaUnits, "mega-units", false, "Displays units in orders of 1000, not 1024") + listContainerCmd.PersistentFlags().BoolVar(¶meters.MachineReadable, "machine-readable", false, "Lists file sizes in bytes.") + listContainerCmd.PersistentFlags().BoolVar(¶meters.RunningTally, "running-tally", false, "Counts the total number of files and their sizes.") + listContainerCmd.PersistentFlags().BoolVar(¶meters.MegaUnits, "mega-units", false, "Displays units in orders of 1000, not 1024.") rootCmd.AddCommand(listContainerCmd) } diff --git a/cmd/login.go b/cmd/login.go index 4b3b98540..2ff980795 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -62,22 +62,22 @@ func init() { rootCmd.AddCommand(lgCmd) - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.tenantID, "tenant-id", "", "the Azure active directory tenant id to use for OAuth device interactive login") - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.aadEndpoint, "aad-endpoint", "", "the Azure active directory endpoint to use for OAuth user interactive login") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.tenantID, "tenant-id", "", "The Azure Active Directory tenant ID to use for OAuth device interactive login.") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.aadEndpoint, "aad-endpoint", "", "The Azure Active Directory endpoint to use for OAuth user interactive login.") // Use identity which aligns to Azure powershell and CLI. - lgCmd.PersistentFlags().BoolVar(&loginCmdArgs.identity, "identity", false, "log in using virtual machine's identity, also known as managed service identity (MSI)") + lgCmd.PersistentFlags().BoolVar(&loginCmdArgs.identity, "identity", false, "Log in using virtual machine's identity, also known as managed service identity (MSI).") // Use SPN certificate to log in. - lgCmd.PersistentFlags().BoolVar(&loginCmdArgs.servicePrincipal, "service-principal", false, "log in via SPN (Service Principal Name) using a certificate or a secret. The client secret or certificate password must be placed in the appropriate environment variable. Type AzCopy env to see names and descriptions of environment variables.") + lgCmd.PersistentFlags().BoolVar(&loginCmdArgs.servicePrincipal, "service-principal", false, "Log in via Service Principal Name (SPN) by using a certificate or a secret. The client secret or certificate password must be placed in the appropriate environment variable. Type AzCopy env to see names and descriptions of environment variables.") // Client ID of user-assigned identity. - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityClientID, "identity-client-id", "", "client ID of user-assigned identity") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityClientID, "identity-client-id", "", "Client ID of user-assigned identity.") // Object ID of user-assigned identity. - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityObjectID, "identity-object-id", "", "object ID of user-assigned identity") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityObjectID, "identity-object-id", "", "Object ID of user-assigned identity.") // Resource ID of user-assigned identity. - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityResourceID, "identity-resource-id", "", "resource ID of user-assigned identity") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.identityResourceID, "identity-resource-id", "", "Resource ID of user-assigned identity.") //login with SPN - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.applicationID, "application-id", "", "application ID of user-assigned identity. Required for service principal auth.") - lgCmd.PersistentFlags().StringVar(&loginCmdArgs.certPath, "certificate-path", "", "path to certificate for SPN authentication. Required for certificate-based service principal auth.") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.applicationID, "application-id", "", "Application ID of user-assigned identity. Required for service principal auth.") + lgCmd.PersistentFlags().StringVar(&loginCmdArgs.certPath, "certificate-path", "", "Path to certificate for SPN authentication. Required for certificate-based service principal auth.") // hide flags // temporaily hide aad-endpoint and support Production environment only. diff --git a/cmd/make.go b/cmd/make.go index dc03623fd..c55fb9fba 100644 --- a/cmd/make.go +++ b/cmd/make.go @@ -214,6 +214,6 @@ func init() { }, } - makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota.") + makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "Specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota.") rootCmd.AddCommand(makeCmd) } diff --git a/cmd/remove.go b/cmd/remove.go index 901cc29b6..f0d15e926 100644 --- a/cmd/remove.go +++ b/cmd/remove.go @@ -77,13 +77,13 @@ func init() { } rootCmd.AddCommand(deleteCmd) - deleteCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when syncing between directories.") - deleteCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs).") - deleteCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - deleteCmd.PersistentFlags().StringVar(&raw.includePath, "include-path", "", "only include these paths when removing. "+ - "Does not support using wildcards. Checks relative path prefix. ex. myFolder;myFolder/subDirName/file.pdf") - deleteCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - deleteCmd.PersistentFlags().StringVar(&raw.excludePath, "exclude-path", "", "exclude these paths when removing. "+ - "Does not support using wildcards. Checks relative path prefix. ex. myFolder;myFolder/subDirName/file.pdf") - deleteCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "defines the location of a file which contains the list of files/directories to be deleted. The relative paths should be delimited by line breaks, and the paths should NOT be URL-encoded.") + deleteCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Look into sub-directories recursively when syncing between directories.") + deleteCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file. Available levels include: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO')") + deleteCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "Include only files where the name matches the pattern list. For example: *.jpg;*.pdf;exactName") + deleteCmd.PersistentFlags().StringVar(&raw.includePath, "include-path", "", "Include only these paths when removing. "+ + "This option does not support wildcard characters (*). Checks relative path prefix. For example: myFolder;myFolder/subDirName/file.pdf") + deleteCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "Exclude files where the name matches the pattern list. For example: *.jpg;*.pdf;exactName") + deleteCmd.PersistentFlags().StringVar(&raw.excludePath, "exclude-path", "", "Exclude these paths when removing. "+ + "This option does not support wildcard characters (*). Checks relative path prefix. For example: myFolder;myFolder/subDirName/file.pdf") + deleteCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "Defines the location of a file which contains the list of files and directories to be deleted. The relative paths should be delimited by line breaks, and the paths should NOT be URL-encoded.") } diff --git a/cmd/root.go b/cmd/root.go index 024f897b7..a7b17e308 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -108,8 +108,8 @@ func init() { // replace the word "global" to avoid confusion (e.g. it doesn't affect all instances of AzCopy) rootCmd.SetUsageTemplate(strings.Replace((&cobra.Command{}).UsageTemplate(), "Global Flags", "Flags Applying to All Commands", -1)) - rootCmd.PersistentFlags().Uint32Var(&cmdLineCapMegaBitsPerSecond, "cap-mbps", 0, "caps the transfer rate, in Mega bits per second. Moment-by-moment throughput may vary slightly from the cap. If zero or omitted, throughput is not capped.") - rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output-type", "text", "format of the command's output, the choices include: text, json.") + rootCmd.PersistentFlags().Uint32Var(&cmdLineCapMegaBitsPerSecond, "cap-mbps", 0, "Caps the transfer rate, in megabits per second. Moment-by-moment throughput might vary slightly from the cap. If this option is set to zero, or it is omitted, the throughput isn't capped.") + rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output-type", "text", "Format of the command's output. The choices include: text, json. The default value is 'text'.") } // always spins up a new goroutine, because sometimes the aka.ms URL can't be reached (e.g. a constrained environment where diff --git a/cmd/sync.go b/cmd/sync.go index 5079f4db4..bedcd777c 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -508,17 +508,17 @@ func init() { } rootCmd.AddCommand(syncCmd) - syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", true, "true by default, look into sub-directories recursively when syncing between directories.") - syncCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "use this block size (specified in MiB) when uploading to/downloading from Azure Storage. Default is automatically calculated based on file size. Decimal fractions are allowed - e.g. 0.25") - syncCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "only include files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "only exclude files whose name matches the pattern list. Example: *.jpg;*.pdf;exactName") - syncCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) only include files whose attributes match the attribute list. Example: A;S;R") - syncCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) exclude files whose attributes match the attribute list. Example: A;S;R") - syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs).") - syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ - "If set to prompt, user will be asked a question before scheduling files/blobs for deletion.") - syncCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob/file. (By default the hash is NOT created.) Only available when uploading.") - syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") + syncCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", true, "True by default, look into sub-directories recursively when syncing between directories. (default true).") + syncCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "Use this block size (specified in MiB) when uploading to Azure Storage or downloading from Azure Storage. Default is automatically calculated based on file size. Decimal fractions are allowed (For example: 0.25).") + syncCmd.PersistentFlags().StringVar(&raw.include, "include-pattern", "", "Include only files where the name matches the pattern list. For example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "Exclude files where the name matches the pattern list. For example: *.jpg;*.pdf;exactName") + syncCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) Include only files whose attributes match the attribute list. For example: A;S;R") + syncCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) Exclude files whose attributes match the attribute list. For example: A;S;R") + syncCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests and responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default INFO).") + syncCmd.PersistentFlags().StringVar(&raw.deleteDestination, "delete-destination", "false", "Defines whether to delete extra files from the destination that are not present at the source. Could be set to true, false, or prompt. "+ + "If set to prompt, the user will be asked a question before scheduling files and blobs for deletion. (default 'false').") + syncCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. (By default the hash is NOT created.) Only available when uploading.") + syncCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. This option is only available when downloading. Available values include: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent').") // TODO follow sym link is not implemented, clarify behavior first //syncCmd.PersistentFlags().BoolVar(&raw.followSymlinks, "follow-symlinks", false, "follow symbolic links when performing sync from local file system.") From cadc4e3c080e42e49a96e74fe5735bd984fcab68 Mon Sep 17 00:00:00 2001 From: normesta Date: Fri, 4 Oct 2019 12:35:58 -0700 Subject: [PATCH 2/6] fixing some conflicts --- cmd/copy.go | 30 +++--------------------------- go.sum | 2 ++ 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index 1f418d3a0..f21ec8c96 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1301,7 +1301,6 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').") cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "Defines the type of blob at the destination. This is used for uploading blobs and when copying between accounts (default 'None').") cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.") -<<<<<<< HEAD cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "Upload page blob to Azure Storage using this blob tier. (default 'None').") cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "Upload to Azure Storage with these key-value pairs as metadata.") cpCmd.PersistentFlags().StringVar(&raw.contentType, "content-type", "", "Specifies the content type of the file. Implies no-guess-mime-type. Returned on download.") @@ -1313,7 +1312,8 @@ func init() { cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "Only available when destination is file system.") cpCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob or file. (By default the hash is NOT created.) Only available when uploading.") cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent')") - + cpCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) Include files whose attributes match the attribute list. For example: A;S;R") + cpCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) Exclude files whose attributes match the attribute list. For example: A;S;R") cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, the transfer is cancelled.") cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "Preserve full properties during service to service copy. "+ "For AWS S3 and Azure File non-single file source, the list operation doesn't return full properties of objects amd files. To preserve full properties, AzCopy needs to send one additional request per object or file.") @@ -1321,31 +1321,7 @@ func init() { "Please refer to [Azure Blob storage: hot, cool, and archive access tiers](https://docs.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers) to ensure destination storage account supports setting access tier. "+ "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. (default true). ") cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "Check if source has changed after enumerating. "+ -======= - cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "upload page blob to Azure Storage using this blob tier.") - cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "upload to Azure Storage with these key-value pairs as metadata.") - cpCmd.PersistentFlags().StringVar(&raw.contentType, "content-type", "", "specifies content type of the file. Implies no-guess-mime-type. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentEncoding, "content-encoding", "", "set the content-encoding header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentDisposition, "content-disposition", "", "set the content-disposition header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.contentLanguage, "content-language", "", "set the content-language header. Returned on download.") - cpCmd.PersistentFlags().StringVar(&raw.cacheControl, "cache-control", "", "set the cache-control header. Returned on download.") - cpCmd.PersistentFlags().BoolVar(&raw.noGuessMimeType, "no-guess-mime-type", false, "prevents AzCopy from detecting the content-type based on the extension/content of the file.") - cpCmd.PersistentFlags().BoolVar(&raw.preserveLastModifiedTime, "preserve-last-modified-time", false, "only available when destination is file system.") - cpCmd.PersistentFlags().BoolVar(&raw.putMd5, "put-md5", false, "create an MD5 hash of each file, and save the hash as the Content-MD5 property of the destination blob/file. (By default the hash is NOT created.) Only available when uploading.") - cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing.") - cpCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) only include files whose attributes match the attribute list. Example: A;S;R") - cpCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) exclude files whose attributes match the attribute list. Example: A;S;R") - - cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, fail the transfer.") - cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "preserve full properties during service to service copy. "+ - "For S3 and Azure File non-single file source, as list operation doesn't return full properties of objects/files, to preserve full properties AzCopy needs to send one additional request per object/file.") - cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "preserve access tier during service to service copy. "+ - "please refer to https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blob-storage-tiers to ensure destination storage account supports setting access tier. "+ - "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. ") - cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "check if source has changed after enumerating. "+ ->>>>>>> 61d6d49327c73094566c973aa7c1a6e25a446d2b - "For S2S copy, as source is a remote resource, validating whether source has changed need additional request costs. ") - cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').") + cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').") // s2sGetPropertiesInBackend is an optional flag for controlling whether S3 object's or Azure file's full properties are get during enumerating in frontend or // right before transferring in ste(backend). diff --git a/go.sum b/go.sum index a7c4187c4..b8b848cfc 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,7 @@ github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda h1:NOo6+gM9NNP github.com/JeffreyRichter/enum v0.0.0-20180725232043-2567042f9cda/go.mod h1:2CaSFTh2ph9ymS6goiOKIBdfhwWUVsX4nQ5QjIYFHHs= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/danieljoos/wincred v1.0.1 h1:fcRTaj17zzROVqni2FiToKUVg3MmJ4NtMSGCySPIr/g= github.com/danieljoos/wincred v1.0.1/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -19,6 +20,7 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/go-ini/ini v1.41.0 h1:526aoxDtxRHFQKMZfcX2OG9oOI8TJ5yPLM0Mkno/uTY= github.com/go-ini/ini v1.41.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jiacfan/keychain v0.0.0-20180920053336-f2c902a3d807 h1:QKbdbbQIbiiWJkCd2zMBiOv7U35YmM1Uq4BOwp2tTCs= github.com/jiacfan/keychain v0.0.0-20180920053336-f2c902a3d807/go.mod h1:IGH0VO3mMxCgF6yPROjtYw4wnCO6EviEgJwiMeNHXdw= From 97ea80f21860fa5074f52d84365ef4a87f546a40 Mon Sep 17 00:00:00 2001 From: normesta Date: Fri, 4 Oct 2019 14:34:26 -0700 Subject: [PATCH 3/6] incorporating some tech review feedback --- cmd/helpMessages.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index 98382819f..fae6b44ea 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -43,7 +43,7 @@ On Windows, MIME types are extracted from the registry. This feature can be turn ` + environmentVariableNotice -const copyCmdExample = `Upload a single file by using OAuth authentication. If you have not yet logged into AzCopy, please use azcopy login command before you run the following command. +const copyCmdExample = `Upload a single file by using OAuth authentication. If you have not yet logged into AzCopy, please run the azcopy login command before you run the following command. - azcopy cp "/path/to/file.txt" "https://[account].blob.core.windows.net/[container]/[path/to/blob]" @@ -73,7 +73,7 @@ Upload files and directories by using a SAS token and wildcard (*) characters: - azcopy cp "/path/*foo/*bar*" "https://[account].blob.core.windows.net/[container]/[path/to/directory]?[SAS]" --recursive=true -Download a single file by using OAuth authentication. If you have not yet logged into AzCopy, please use azcopy login command before you run the following command. +Download a single file by using OAuth authentication. If you have not yet logged into AzCopy, please run the azcopy login command before you run the following command. - azcopy cp "https://[account].blob.core.windows.net/[container]/[path/to/blob]" "/path/to/file.txt" @@ -92,8 +92,8 @@ Download an entire directory by using a SAS token: A note about using a wildcard character (*) in URLs: There's only two supported ways to use a wildcard character in a URL. -You can use one just after the final forward slash (/) of a URL to obtain all of the files in a directory. -You can also use one in the name of a container as long as the URL refers only to a container and not to a blob. +- You can use one just after the final forward slash (/) of a URL. This copies all of the files in a directory directly to the destination without placing them into a subdirectory. +- You can also use one in the name of a container as long as the URL refers only to a container and not to a blob. You can use this approach to obtain files from a subset of containers. Download the contents of a directory without copying the containing directory itself. @@ -103,7 +103,7 @@ Download an entire storage account. - azcopy cp "https://[srcaccount].blob.core.windows.net/" "/path/to/dir" --recursive -Download an entire storage account by using a wildcard symbol (*) in the container name. +Download a subset of containers within a storage account by using a wildcard symbol (*) in the container name. - azcopy cp "https://[srcaccount].blob.core.windows.net/[container*name]" "/path/to/dir" --recursive @@ -111,7 +111,7 @@ Copy a single blob to another blob by using a SAS token. - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" -Copy a single blob to another blob by using a SAS token and an OAuth token. If you log into AzCopy by using the azcopy login command, the destination account doesn't need a SAS token. +Copy a single blob to another blob by using a SAS token and an OAuth token. You have to use a SAS token at the end of the source account URL, but the destination account doesn't need one if you log into AzCopy by using the azcopy login command. - azcopy cp "https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob]?[SAS]" "https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]" @@ -140,6 +140,10 @@ Copy all buckets to Blob Storage from Amazon Web Services (AWS) by using an acce Copy all buckets to Blob Storage from an Amazon Web Services (AWS) region by using an access key and a SAS token. First, set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. - azcopy cp "https://s3-[region].amazonaws.com/" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true + +Copy a subset of buckets by using a wildcard symbol (*) in the bucket name. Like the previous examples, you'll need an access key and a SAS token. Make sure to set the environment variable AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for AWS S3 source. + + - azcopy cp "https://s3.amazonaws.com/[bucket*name]/" "https://[destaccount].blob.core.windows.net?[SAS]" --recursive=true ` // ===================================== ENV COMMAND ===================================== // @@ -237,7 +241,7 @@ Set the environment variable AZCOPY_SPA_CLIENT_SECRET to the client secret for s - azcopy login --service-principal -Log in as a service principal by using a certificate and a password: +Log in as a service principal by using a certificate and it's password: Set the environment variable AZCOPY_SPA_CERT_PASSWORD to the certificate's password for cert based service principal auth - azcopy login --service-principal --certificate-path /path/to/my/cert @@ -315,11 +319,10 @@ The last modified times are used for comparison. The file is skipped if the last The sync command differs from the copy command in several ways: - 1. The recursive flag is on by default. + 1. Sync can copy directories without the recursive flag, but it will not sync subdirectories. 2. The source and destination should not contain patterns (For example: * or ?). - 3. The include and exclude flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. - 4. If there are files and blobs at the destination that are not present at the source, the user is prompted to delete them. You can use the corresponding flags to silence the prompt to automatically answer the deletion question. - 5. When syncing between virtual directories, add a trailing slash to the path (refer to examples) if there's a blob with the same name as one of the virtual directories. + 3. The include-pattern and exclude-pattern flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. + 4. When syncing between virtual directories, add a trailing slash to the path (refer to examples) if there's a blob with the same name as one of the virtual directories. Advanced: From 0c7d699a925b87be7118c929a7f6c243eb39f92f Mon Sep 17 00:00:00 2001 From: normesta Date: Mon, 7 Oct 2019 10:21:08 -0700 Subject: [PATCH 4/6] Fixing an error --- cmd/copy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index f21ec8c96..36605a50b 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1320,8 +1320,8 @@ func init() { cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "Preserve access tier during service to service copy. "+ "Please refer to [Azure Blob storage: hot, cool, and archive access tiers](https://docs.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers) to ensure destination storage account supports setting access tier. "+ "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. (default true). ") - cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "Check if source has changed after enumerating. "+ - cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').") + cpCmd.PersistentFlags().BoolVar(&raw.s2sSourceChangeValidation, "s2s-detect-source-changed", false, "Check if source has changed after enumerating. ") + cpCmd.PersistentFlags().StringVar(&raw.s2sInvalidMetadataHandleOption, "s2s-handle-invalid-metadata", common.DefaultInvalidMetadataHandleOption.String(), "Specifies how invalid metadata keys are handled. Available options: ExcludeIfInvalid, FailIfInvalid, RenameIfInvalid. (default 'ExcludeIfInvalid').") // s2sGetPropertiesInBackend is an optional flag for controlling whether S3 object's or Azure file's full properties are get during enumerating in frontend or // right before transferring in ste(backend). From d853aded23e09693ba8c4108dba0dd8e36e02945 Mon Sep 17 00:00:00 2001 From: normesta Date: Thu, 10 Oct 2019 08:59:42 -0700 Subject: [PATCH 5/6] Addressing some review feedback --- cmd/copy.go | 7 ++++--- cmd/helpMessages.go | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index f196ff66a..061e4fcbc 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1319,7 +1319,8 @@ func init() { // options change how the transfers are performed cpCmd.PersistentFlags().Float64Var(&raw.blockSizeMB, "block-size-mb", 0, "Use this block size (specified in MiB) when uploading to Azure Storage, and downloading from Azure Storage. The default value is automatically calculated based on file size. Decimal fractions are allowed (For example: 0.25).") cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').") - cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "Defines the type of blob at the destination. This is used for uploading blobs and when copying between accounts (default 'None').") + cpCmd.PersistentFlags().StringVar(&raw.blobType, "blob-type", "Detect", "Defines the type of blob at the destination. This is used for uploading blobs and when copying between accounts (default 'Detect'). Valid values include 'Detect', 'BlockBlob', 'PageBlob', and 'AppendBlob'. "+ + "When copying between accounts, a value of 'Detect' causes AzCopy to use the type of source blob to determine the type of the destination blob. When uploading a file, 'Detect' determines if the file is a VHD or a VHDX file based on the file extension. If the file is ether a VHD or VHDX file, AzCopy treats the file as a page blob.") cpCmd.PersistentFlags().StringVar(&raw.blockBlobTier, "block-blob-tier", "None", "upload block blob to Azure Storage using this blob tier.") cpCmd.PersistentFlags().StringVar(&raw.pageBlobTier, "page-blob-tier", "None", "Upload page blob to Azure Storage using this blob tier. (default 'None').") cpCmd.PersistentFlags().StringVar(&raw.metadata, "metadata", "", "Upload to Azure Storage with these key-value pairs as metadata.") @@ -1334,9 +1335,9 @@ func init() { cpCmd.PersistentFlags().StringVar(&raw.md5ValidationOption, "check-md5", common.DefaultHashValidationOption.String(), "Specifies how strictly MD5 hashes should be validated when downloading. Only available when downloading. Available options: NoCheck, LogOnly, FailIfDifferent, FailIfDifferentOrMissing. (default 'FailIfDifferent')") cpCmd.PersistentFlags().StringVar(&raw.includeFileAttributes, "include-attributes", "", "(Windows only) Include files whose attributes match the attribute list. For example: A;S;R") cpCmd.PersistentFlags().StringVar(&raw.excludeFileAttributes, "exclude-attributes", "", "(Windows only) Exclude files whose attributes match the attribute list. For example: A;S;R") - cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, the transfer is cancelled.") + cpCmd.PersistentFlags().BoolVar(&raw.CheckLength, "check-length", true, "Check the length of a file on the destination after the transfer. If there is a mismatch between source and destination, the transfer is marked as failed.") cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveProperties, "s2s-preserve-properties", true, "Preserve full properties during service to service copy. "+ - "For AWS S3 and Azure File non-single file source, the list operation doesn't return full properties of objects amd files. To preserve full properties, AzCopy needs to send one additional request per object or file.") + "For AWS S3 and Azure File non-single file source, the list operation doesn't return full properties of objects and files. To preserve full properties, AzCopy needs to send one additional request per object or file.") cpCmd.PersistentFlags().BoolVar(&raw.s2sPreserveAccessTier, "s2s-preserve-access-tier", true, "Preserve access tier during service to service copy. "+ "Please refer to [Azure Blob storage: hot, cool, and archive access tiers](https://docs.microsoft.com/azure/storage/blobs/storage-blob-storage-tiers) to ensure destination storage account supports setting access tier. "+ "In the cases that setting access tier is not supported, please use s2sPreserveAccessTier=false to bypass copying access tier. (default true). ") diff --git a/cmd/helpMessages.go b/cmd/helpMessages.go index 89df28893..9f0c6fdc2 100644 --- a/cmd/helpMessages.go +++ b/cmd/helpMessages.go @@ -171,7 +171,7 @@ const showJobsCmdShortDescription = "Show detailed information for the given job const showJobsCmdLongDescription = ` If you provide only a job ID, and not a flag, then this command returns the progress summary only. -The byte counts and percent complete that appears when you run this command don't reflect files related to jobs that are currently in progress. +The byte counts and percent complete that appears when you run this command reflect only files that are completed in the job. They don't reflect partially completed files. If you set the with-status flag, then only the list of transfers associated with the given status appear.` const resumeJobsCmdShortDescription = "Resume the existing job with the given job ID." @@ -325,6 +325,7 @@ The sync command differs from the copy command in several ways: 2. The source and destination should not contain patterns (For example: * or ?). 3. The include-pattern and exclude-pattern flags can be a list of patterns matching to the file names. Please refer to the example section for illustration. 4. When syncing between virtual directories, add a trailing slash to the path (refer to examples) if there's a blob with the same name as one of the virtual directories. + 5. If the 'deleteDestination' flag is set to true or prompt, then sync will delete files and blobs at the destination that are not present at the source. Advanced: From 3419340d9da38afe59978f5d04885c91d514abcc Mon Sep 17 00:00:00 2001 From: normesta Date: Thu, 10 Oct 2019 09:08:08 -0700 Subject: [PATCH 6/6] One more item --- cmd/copy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/copy.go b/cmd/copy.go index 061e4fcbc..50a3a2eef 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1310,7 +1310,7 @@ func init() { // This flag is implemented only for Storage Explorer. cpCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "Defines the location of text file which has the list of only files to be copied.") cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude-pattern", "", "Exclude these files when copying. This option supports wildcard characters (*)") - cpCmd.PersistentFlags().StringVar(&raw.forceWrite, "overwrite", "true", "Overwrite the conflicting files/blobs at the destination if this flag is set to true. (default true).") + cpCmd.PersistentFlags().StringVar(&raw.forceWrite, "overwrite", "true", "Overwrite the conflicting files and blobs at the destination if this flag is set to true. (default 'true') Possible values include 'true', 'false', and 'prompt'.") cpCmd.PersistentFlags().BoolVar(&raw.autoDecompress, "decompress", false, "Automatically decompress files when downloading, if their content-encoding indicates that they are compressed. The supported content-encoding values are 'gzip' and 'deflate'. File extensions of '.gz'/'.gzip' or '.zz' aren't necessary, but will be removed if present.") cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "Look into sub-directories recursively when uploading from local file system.") cpCmd.PersistentFlags().StringVar(&raw.fromTo, "from-to", "", "Optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.")