Merge pull request #95 from Azure/dev

10.0.3

Author: zezha-msft

.gitignore

@@ -325,3 +325,6 @@ testSuite/venv1/*
 testSuite/scripts/__pycache__/utility.cpython-36.pyc
 testSuite/scripts/__pycache__/utility.cpython-36.pyc
 venv*
+
+#json create files
+*.json

clean.sh

@@ -7,3 +7,4 @@
 
 # remove logs and memory mapped plan files
 rm -rf ~/.azcopy/*
+rm ./azcopy*

cmd/copy.go

@@ -37,6 +37,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/2018-03-28/azblob"
 	"github.com/Azure/azure-storage-file-go/2017-07-29/azfile"
 	"github.com/spf13/cobra"
+	"io/ioutil"
 )
 
 // upload related
@@ -65,11 +66,12 @@ type rawCopyCmdArgs struct {
 	//blobUrlForRedirection string
 
 	// filters from flags
-	include        string
-	exclude        string
-	recursive      bool
-	followSymlinks bool
-	withSnapshots  bool
+	listOfFilesToCopy string
+	include           string
+	exclude           string
+	recursive         bool
+	followSymlinks    bool
+	withSnapshots     bool
 	// forceWrite flag is used to define the User behavior
 	// to overwrite the existing blobs or not.
 	forceWrite bool
@@ -124,6 +126,46 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) {
 	if err != nil {
 		return cooked, err
 	}
+	// User can provide either listOfFilesToCopy or include since listOFFiles mentions
+	// file names to include explicitly and include file may mention at pattern.
+	// This could conflict enumerating the files to queue up for transfer.
+	if len(raw.listOfFilesToCopy) > 0 && len(raw.include) > 0 {
+		return cooked, fmt.Errorf("user provided argument with both listOfFilesToCopy and include flag. Only one should be provided")
+	}
+
+	// If the user provided the list of files explicitly to be copied, then parse the argument
+	// The user passes the location of json file which will have the list of files to be copied.
+	// The "json file" is chosen as input because there is limit on the number of characters that
+	// can be supplied with the argument, but Storage Explorer folks requirements was not to impose
+	// any limit on the number of files that can be copied.
+	if len(raw.listOfFilesToCopy) > 0 {
+		//files := strings.Split(raw.listOfFilesToCopy, ";")
+		jsonFile, err := os.Open(raw.listOfFilesToCopy)
+		if err != nil {
+			return cooked, fmt.Errorf("cannot open %s file passed with the list-of-file flag", raw.listOfFilesToCopy)
+		}
+		// read our opened xmlFile as a byte array.
+		jsonBytes, err := ioutil.ReadAll(jsonFile)
+		if err != nil {
+			return cooked, fmt.Errorf("error %s read %s file passed with the list-of-file flag", err.Error(), raw.listOfFilesToCopy)
+		}
+		var files common.ListOfFiles
+		err = json.Unmarshal(jsonBytes, &files)
+		if err != nil {
+			return cooked, fmt.Errorf("error %s unmarshalling the contents of %s file passed with the list-of-file flag", err.Error(), raw.listOfFilesToCopy)
+		}
+		for _, file := range files.Files {
+			// If split of the include string leads to an empty string
+			// not include that string
+			if len(file) == 0 {
+				continue
+			}
+			// replace the OS path separator in includePath string with AZCOPY_PATH_SEPARATOR
+			// this replacement is done to handle the windows file paths where path separator "\\"
+			filePath := strings.Replace(file, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1)
+			cooked.listOfFilesToCopy = append(cooked.listOfFilesToCopy, filePath)
+		}
+	}
 
 	// initialize the include map which contains the list of files to be included
 	// parse the string passed in include flag
@@ -260,12 +302,13 @@ type cookedCopyCmdArgs struct {
 	fromTo         common.FromTo
 
 	// filters from flags
-	include        map[string]int
-	exclude        map[string]int
-	recursive      bool
-	followSymlinks bool
-	withSnapshots  bool
-	forceWrite     bool
+	listOfFilesToCopy []string
+	include           map[string]int
+	exclude           map[string]int
+	recursive         bool
+	followSymlinks    bool
+	withSnapshots     bool
+	forceWrite        bool
 
 	// options from flags
 	blockSize uint32
@@ -883,6 +926,8 @@ Copy an entire account with SAS:
 	cpCmd.PersistentFlags().BoolVar(&raw.withSnapshots, "with-snapshots", false, "include the snapshots. Only valid when the source is blobs.")
 	cpCmd.PersistentFlags().StringVar(&raw.include, "include", "", "only include these files when copying. "+
 		"Support use of *. Files should be separated with ';'.")
+	// This flag is implemented only for Storage Explorer.
+	cpCmd.PersistentFlags().StringVar(&raw.listOfFilesToCopy, "list-of-files", "", "defines the location of json which has the list of only files to be copied")
 	cpCmd.PersistentFlags().StringVar(&raw.exclude, "exclude", "", "exclude these files when copying. Support use of *.")
 	cpCmd.PersistentFlags().BoolVar(&raw.forceWrite, "overwrite", true, "overwrite the conflicting files/blobs at the destination if this flag is set to true.")
 	cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when uploading from local file system.")
@@ -909,6 +954,8 @@ Copy an entire account with SAS:
 	cpCmd.PersistentFlags().MarkHidden("acl")
 
 	// permanently hidden
+	// Hide the list-of-files flag since it is implemented only for Storage Explorer.
+	cpCmd.PersistentFlags().MarkHidden("list-of-files")
 	cpCmd.PersistentFlags().MarkHidden("include")
 	cpCmd.PersistentFlags().MarkHidden("output")
 	cpCmd.PersistentFlags().MarkHidden("stdIn-enable")

cmd/copyDownloadBlobEnumerator.go

@@ -112,6 +112,115 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error {
 		}
 	}
 
+	// If the user has provided us with a list of files to be copied explicitly
+	// then there is no need list using the source and then perform pattern matching.
+	if len(cca.listOfFilesToCopy) > 0 {
+		for _, blob := range cca.listOfFilesToCopy {
+			// copy the blobParts in the temporary blobPart since for each blob mentioned in the listOfFilesToCopy flag
+			// blobParts will be modified.
+			tempBlobUrlParts := blobUrlParts
+			if len(parentSourcePath) > 0 && parentSourcePath[len(parentSourcePath)-1] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+				parentSourcePath = parentSourcePath[0 : len(parentSourcePath)-1]
+			}
+			// Create the blobPath using the given source and blobs mentioned with listOfFile flag.
+			// For Example:
+			// 1. source = "https://sdksampleperftest.blob.core.windows.net/bigdata" blob = "file1.txt" blobPath= "file1.txt"
+			// 2. source = "https://sdksampleperftest.blob.core.windows.net/bigdata/dir-1" blob = "file1.txt" blobPath= "dir-1/file1.txt"
+			blobPath := fmt.Sprintf("%s%s%s", parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING, blob)
+			if len(blobPath) > 0 && blobPath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+				blobPath = blobPath[1:]
+			}
+			tempBlobUrlParts.BlobName = blobPath
+			blobURL := azblob.NewBlobURL(tempBlobUrlParts.URL(), p)
+			blobProperties, err := blobURL.GetProperties(ctx, azblob.BlobAccessConditions{})
+			if err == nil {
+				// If the blob represents a folder as per the conditions mentioned in the
+				// api doesBlobRepresentAFolder, then skip the blob.
+				if util.doesBlobRepresentAFolder(blobProperties.NewMetadata()) {
+					continue
+				}
+				//blobRelativePath := util.getRelativePath(parentSourcePath, blobPath)
+				//blobRelativePath := util.getRelativePath(parentSourcePath, blobPath)
+				blobRelativePath := strings.Replace(blobPath, parentSourcePath, "", 1)
+				if len(blobRelativePath) > 0 && blobRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+					blobRelativePath = blobRelativePath[1:]
+				}
+				// check for the special character in blob relative path and get path without special character.
+				blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath)
+
+				e.addTransfer(common.CopyTransfer{
+					Source:           util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobPath)).String(),
+					Destination:      util.generateLocalPath(cca.destination, blobRelativePath),
+					LastModifiedTime: blobProperties.LastModified(),
+					SourceSize:       blobProperties.ContentLength()}, cca)
+				continue
+			}
+			if !cca.recursive {
+				glcm.Info(fmt.Sprintf("error fetching properties of %s. Either it is a directory or getting the blob properties failed. For virtual directories try using the recursive flag", blobPath))
+				continue
+			}
+			// Since the given blob in the listOFFiles flag is not a blob, it can be a virtual directory
+			// If the virtual directory doesn't have a path separator at the end of it, then we should append it.
+			// This is done to avoid listing blobs which shares the common prefix i.e the virtual directory name.
+			// For Example:
+			// 1. source = "https://sdksampleperftest.blob.core.windows.net/bigdata" blob="100k". In this case, it is
+			// a possibility that we have blobs https://sdksampleperftest.blob.core.windows.net/bigdata/100K and
+			// https://sdksampleperftest.blob.core.windows.net/bigdata/100K/f1.txt. So we need to list the blob
+			// https://sdksampleperftest.blob.core.windows.net/bigdata/100K/f1.txt
+			searchPrefix := tempBlobUrlParts.BlobName
+			if len(searchPrefix) > 0 && searchPrefix[len(searchPrefix)-1] != common.AZCOPY_PATH_SEPARATOR_CHAR {
+				searchPrefix += common.AZCOPY_PATH_SEPARATOR_STRING
+			}
+			for marker := (azblob.Marker{}); marker.NotDone(); {
+				// look for all blobs that start with the prefix, so that if a blob is under the virtual directory, it will show up
+				listBlob, err := containerUrl.ListBlobsFlatSegment(ctx, marker,
+					azblob.ListBlobsSegmentOptions{Details: azblob.BlobListingDetails{Metadata: true}, Prefix: searchPrefix})
+				if err != nil {
+					glcm.Info(fmt.Sprintf("cannot list blobs inside directory %s mentioned.", searchPrefix))
+					continue
+				}
+				// If there was no blob listed inside the directory mentioned in the listOfFilesToCopy flag,
+				// report to the user and continue to the next blob mentioned.
+				if !listBlob.NextMarker.NotDone() && len(listBlob.Segment.BlobItems) == 0 {
+					glcm.Info(fmt.Sprintf("cannot list blobs inside directory %s mentioned.", searchPrefix))
+					break
+				}
+				for _, blobInfo := range listBlob.Segment.BlobItems {
+					// If the blob represents a folder as per the conditions mentioned in the
+					// api doesBlobRepresentAFolder, then skip the blob.
+					if util.doesBlobRepresentAFolder(blobInfo.Metadata) {
+						continue
+					}
+					blobRelativePath := strings.Replace(blobInfo.Name, parentSourcePath, "", 1)
+					if len(blobRelativePath) > 0 && blobRelativePath[0] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+						blobRelativePath = blobRelativePath[1:]
+					}
+					//blobRelativePath := util.getRelativePath(parentSourcePath, blobInfo.Name)
+
+					// check for the special character in blob relative path and get path without special character.
+					blobRelativePath = util.blobPathWOSpecialCharacters(blobRelativePath)
+					e.addTransfer(common.CopyTransfer{
+						Source:           util.stripSASFromBlobUrl(util.createBlobUrlFromContainer(blobUrlParts, blobInfo.Name)).String(),
+						Destination:      util.generateLocalPath(cca.destination, blobRelativePath),
+						LastModifiedTime: blobInfo.Properties.LastModified,
+						SourceSize:       *blobInfo.Properties.ContentLength}, cca)
+				}
+				marker = listBlob.NextMarker
+			}
+		}
+		// If there are no transfer to queue up, exit with message
+		if len(e.Transfers) == 0 {
+			glcm.Exit(fmt.Sprintf("no transfer queued for copying data from %s to %s", cca.source, cca.destination), 1)
+			return nil
+		}
+		// dispatch the JobPart as Final Part of the Job
+		err = e.dispatchFinalPart(cca)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
 	// searchPrefix is the used in listing blob inside a container
 	// all the blob listed should have the searchPrefix as the prefix
 	// blobNamePattern represents the regular expression which the blobName should Match
@@ -142,7 +251,7 @@ func (e *copyDownloadBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error {
 		for _, blobInfo := range listBlob.Segment.BlobItems {
 			// If the blob represents a folder as per the conditions mentioned in the
 			// api doesBlobRepresentAFolder, then skip the blob.
-			if util.doesBlobRepresentAFolder(blobInfo) {
+			if util.doesBlobRepresentAFolder(blobInfo.Metadata) {
 				continue
 			}
 			// If the blobName doesn't matches the blob name pattern, then blob is not included

cmd/copyEnumeratorHelper.go

@@ -84,7 +84,7 @@ func enumerateBlobsInContainer(ctx context.Context, containerURL azblob.Containe
 		for _, blobItem := range listContainerResp.Segment.BlobItems {
 			// If the blob represents a folder as per the conditions mentioned in the
 			// api doesBlobRepresentAFolder, then skip the blob.
-			if gCopyUtil.doesBlobRepresentAFolder(blobItem) {
+			if gCopyUtil.doesBlobRepresentAFolder(blobItem.Metadata) {
 				continue
 			}
 

cmd/copyUploadEnumerator.go

@@ -4,13 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/Azure/azure-storage-azcopy/common"
 	"net/url"
 	"os"
 	"path/filepath"
-
 	"strings"
-
-	"github.com/Azure/azure-storage-azcopy/common"
 )
 
 type copyUploadEnumerator common.CopyJobPartOrderRequest
@@ -82,6 +80,84 @@ func (e *copyUploadEnumerator) enumerate(cca *cookedCopyCmdArgs) error {
 	// temporarily save the path of the container
 	cleanContainerPath := destinationURL.Path
 
+	// If the user has provided the listofFiles explicitly to copy, there is no
+	// need to glob the source and match the patterns.
+	// This feature is supported only for Storage Explorer and doesn't follow the symlinks.
+	if len(cca.listOfFilesToCopy) > 0 {
+		for _, file := range cca.listOfFilesToCopy {
+			tempDestinationURl := *destinationURL
+			parentSourcePath, _ := util.sourceRootPathWithoutWildCards(cca.source)
+			if len(parentSourcePath) > 0 && parentSourcePath[len(parentSourcePath)-1] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+				parentSourcePath = parentSourcePath[:len(parentSourcePath)-1]
+			}
+			filePath := fmt.Sprintf("%s%s%s", parentSourcePath, common.AZCOPY_PATH_SEPARATOR_STRING, file)
+			f, err := os.Stat(filePath)
+			if err != nil {
+				glcm.Info(fmt.Sprintf("Error getting the fileInfo for file %s. failed with error %s", filePath, err.Error()))
+				continue
+			}
+			if f.Mode().IsRegular() {
+				// If the file is a regular file, calculate the destination path and queue for transfer.
+				tempDestinationURl.Path = util.generateObjectPath(tempDestinationURl.Path, f.Name())
+				err = e.addTransfer(common.CopyTransfer{
+					Source:           filePath,
+					Destination:      tempDestinationURl.String(),
+					LastModifiedTime: f.ModTime(),
+					SourceSize:       f.Size(),
+				}, cca)
+
+				if err != nil {
+					glcm.Info(fmt.Sprintf("error %s adding source %s and destination %s as a transfer", err.Error(), filePath, destinationURL))
+				}
+				continue
+			}
+			// If the last character of the filePath is a path separator, strip the path separator.
+			if len(filePath) > 0 && filePath[len(filePath)-1] == common.AZCOPY_PATH_SEPARATOR_CHAR {
+				filePath = filePath[:len(filePath)-1]
+			}
+			if f.IsDir() && cca.recursive {
+				// If the file is a directory, walk through all the elements inside the directory and queue the elements for transfer.
+				filepath.Walk(filePath, func(pathToFile string, info os.FileInfo, err error) error {
+					if err != nil {
+						glcm.Info(fmt.Sprintf("Accessing %s failed with error %s", pathToFile, err.Error()))
+						return nil
+					}
+					if info.IsDir() {
+						return nil
+					} else if info.Mode().IsRegular() { // If the resource is file
+						// replace the OS path separator in pathToFile string with AZCOPY_PATH_SEPARATOR
+						// this replacement is done to handle the windows file paths where path separator "\\"
+						pathToFile = strings.Replace(pathToFile, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1)
+
+						// replace the OS path separator in fileOrDirectoryPath string with AZCOPY_PATH_SEPARATOR
+						// this replacement is done to handle the windows file paths where path separator "\\"
+						filePath = strings.Replace(filePath, common.OS_PATH_SEPARATOR, common.AZCOPY_PATH_SEPARATOR_STRING, -1)
+
+						// upload the files
+						// the path in the blob name started at the given fileOrDirectoryPath
+						// example: fileOrDirectoryPath = "/dir1/dir2/dir3" pathToFile = "/dir1/dir2/dir3/file1.txt" result = "dir3/file1.txt"
+						tempDestinationURl.Path = util.generateObjectPath(cleanContainerPath,
+							util.getRelativePath(filePath, pathToFile))
+						err = e.addTransfer(common.CopyTransfer{
+							Source:           pathToFile,
+							Destination:      tempDestinationURl.String(),
+							LastModifiedTime: info.ModTime(),
+							SourceSize:       info.Size(),
+						}, cca)
+						if err != nil {
+							return err
+						}
+					}
+					return nil
+				})
+			}
+		}
+		if e.PartNum == 0 && len(e.Transfers) == 0 {
+			return errors.New("nothing can be uploaded, please use --recursive to upload directories")
+		}
+		return e.dispatchFinalPart(cca)
+	}
+
 	// Get the source path without the wildcards
 	// This is defined since the files mentioned with exclude flag
 	// & include flag are relative to the Source

cmd/copyUtil.go

@@ -757,12 +757,12 @@ func (util copyHandlerUtil) blobPathWOSpecialCharacters(blobPath string) string
 
 // doesBlobRepresentAFolder verifies whether blob is valid or not.
 // Used to handle special scenarios or conditions.
-func (util copyHandlerUtil) doesBlobRepresentAFolder(bInfo azblob.BlobItem) bool {
+func (util copyHandlerUtil) doesBlobRepresentAFolder(metadata azblob.Metadata) bool {
 	// this condition is to handle the WASB V1 directory structure.
 	// HDFS driver creates a blob for the empty directories (let’s call it ‘myfolder’)
 	// and names all the blobs under ‘myfolder’ as such: ‘myfolder/myblob’
 	// The empty directory has meta-data 'hdi_isfolder = true'
-	return bInfo.Metadata["hdi_isfolder"] == "true"
+	return metadata["hdi_isfolder"] == "true"
 }
 
 func startsWith(s string, t string) bool {

cmd/removeBlobEnumerator.go

@@ -88,7 +88,7 @@ func (e *removeBlobEnumerator) enumerate(cca *cookedCopyCmdArgs) error {
 		for _, blobInfo := range listBlob.Segment.BlobItems {
 			// If the blob represents a folder as per the conditions mentioned in the
 			// api doesBlobRepresentAFolder, then skip the blob.
-			if util.doesBlobRepresentAFolder(blobInfo) {
+			if util.doesBlobRepresentAFolder(blobInfo.Metadata) {
 				continue
 			}
 			// If the blobName doesn't matches the blob name pattern, then blob is not included

cmd/root.go

@@ -73,6 +73,7 @@ func Execute(azsAppPathFolder string) {
 		// only help commands reach this point
 		// execute synchronously before exiting
 		detectNewVersion()
+		glcm.Exit("", common.EExitCode.Success())
 	}
 }