Merge pull request #84 from Azure/dev

Dev

Author: prjain-msft

Dockerfile

@@ -3,7 +3,7 @@ FROM golang:1.10
 ENV GOPATH /go
 ENV PATH ${GOPATH}/bin:$PATH
 RUN go get -u github.com/golang/dep/cmd/dep
-RUN go get -u github.com/golang/lint/golint
+RUN go get -u golang.org/x/lint/golint
 
 # Prepare enviroment for OSX cross compilation. 
 # These steps are referenced from https://github.com/karalabe/xgo/blob/master/docker/base/Dockerfile (licensed with MIT)

Gopkg.lock

@@ -8,7 +8,7 @@
 
 [[constraint]]
   name = "github.com/Azure/azure-storage-file-go"
-  version = "0.2.0"
+  branch = "dev"
 
 [[constraint]]
   name = "github.com/Azure/go-autorest"

Gopkg.toml

@@ -29,6 +29,7 @@ build: setup ## build binaries for the project
     # the environment variables need to be passed into the container explicitly
 	GOARCH=amd64 GOOS=linux $(call with_docker,go build -o "azcopy_linux_amd64",-e GOARCH -e GOOS)
 	GOARCH=amd64 GOOS=windows $(call with_docker,go build -o "azcopy_windows_amd64.exe",-e GOARCH -e GOOS)
+	GOARCH=386 GOOS=windows $(call with_docker,go build -o "azcopy_windows_386.exe",-e GOARCH -e GOOS)
 
 build-osx: setup ## build osx binary specially, as it's using CGO
 	CC=o64-clang CXX=o64-clang++ GOOS=darwin GOARCH=amd64 CGO_ENABLED=1 $(call with_docker,go build -o "azcopy_darwin_amd64",-e CC -e CXX -e GOOS -e GOARCH -e CGO_ENABLED)

Makefile

@@ -88,6 +88,8 @@ func init() {
 			if err != nil {
 				glcm.Exit("failed to perform copy command due to error "+err.Error(), common.EExitCode.Error())
 			}
+
+			glcm.Exit("", common.EExitCode.Success())
 		},
 		// hide features not relevant to BFS
 		// TODO remove after preview release.

cmd/cancel.go

@@ -88,6 +88,8 @@ type rawCopyCmdArgs struct {
 	acl                      string
 	logVerbosity             string
 	cancelFromStdin          bool
+	// list of blobTypes to exclude while enumerating the transfer
+	excludeBlobType string
 }
 
 // validates and transform raw input into cooked input
@@ -230,6 +232,21 @@ func (raw rawCopyCmdArgs) cook() (cookedCopyCmdArgs, error) {
 			return cooked, fmt.Errorf("content-type, content-encoding or metadata is set while copying from sevice to service")
 		}
 	}
+
+	// If the user has provided some input with excludeBlobType flag, parse the input.
+	if len(raw.excludeBlobType) > 0 {
+		// Split the string using delimeter ';' and parse the individual blobType
+		blobTypes := strings.Split(raw.excludeBlobType, ";")
+		for _, blobType := range blobTypes {
+			var eBlobType common.BlobType
+			err := eBlobType.Parse(blobType)
+			if err != nil {
+				return cooked, fmt.Errorf("error parsing the excludeBlobType %s provided with excludeBlobTypeFlag ", blobType)
+			}
+			cooked.excludeBlobType = append(cooked.excludeBlobType, eBlobType.ToAzBlobType())
+		}
+	}
+
 	return cooked, nil
 }
 
@@ -251,7 +268,9 @@ type cookedCopyCmdArgs struct {
 	forceWrite     bool
 
 	// options from flags
-	blockSize                uint32
+	blockSize uint32
+	// list of blobTypes to exclude while enumerating the transfer
+	excludeBlobType          []azblob.BlobType
 	blockBlobTier            common.BlockBlobTier
 	pageBlobTier             common.PageBlobTier
 	metadata                 string
@@ -414,13 +433,14 @@ func (cca *cookedCopyCmdArgs) processRedirectionUpload(blobUrl string, blockSize
 func (cca *cookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 	// initialize the fields that are constant across all job part orders
 	jobPartOrder := common.CopyJobPartOrderRequest{
-		JobID:      cca.jobID,
-		FromTo:     cca.fromTo,
-		ForceWrite: cca.forceWrite,
-		Priority:   common.EJobPriority.Normal(),
-		LogLevel:   cca.logVerbosity,
-		Include:    cca.include,
-		Exclude:    cca.exclude,
+		JobID:           cca.jobID,
+		FromTo:          cca.fromTo,
+		ForceWrite:      cca.forceWrite,
+		Priority:        common.EJobPriority.Normal(),
+		LogLevel:        cca.logVerbosity,
+		Include:         cca.include,
+		Exclude:         cca.exclude,
+		ExcludeBlobType: cca.excludeBlobType,
 		BlobAttributes: common.BlobTransferAttributes{
 			BlockSizeInBytes:         cca.blockSize,
 			ContentType:              cca.contentType,
@@ -867,7 +887,8 @@ Copy an entire account with SAS:
 	cpCmd.PersistentFlags().BoolVar(&raw.forceWrite, "overwrite", true, "overwrite the conflicting files/blobs at the destination if this flag is set to true.")
 	cpCmd.PersistentFlags().BoolVar(&raw.recursive, "recursive", false, "look into sub-directories recursively when uploading from local file system.")
 	cpCmd.PersistentFlags().StringVar(&raw.fromTo, "fromTo", "", "optionally specifies the source destination combination. For Example: LocalBlob, BlobLocal, LocalBlobFS.")
-
+	cpCmd.PersistentFlags().StringVar(&raw.excludeBlobType, "excludeBlobType", "", "optionally specifies the type of blob (BlockBlob/ PageBlob/ AppendBlob) to exclude when copying blobs from Container / Account. Use of "+
+		"this flag is not applicable for copying data from non azure-service to service. More than one blob should be separated by ';' ")
 	// options change how the transfers are performed
 	cpCmd.PersistentFlags().StringVar(&raw.output, "output", "text", "format of the command's output, the choices include: text, json.")
 	cpCmd.PersistentFlags().StringVar(&raw.logVerbosity, "log-level", "INFO", "define the log verbosity for the log file, available levels: DEBUG, INFO, WARNING, ERROR, PANIC, and FATAL.")

cmd/copy.go

@@ -171,7 +171,6 @@ func (e *copyBlobToNEnumerator) addTransfersFromContainer(ctx context.Context, s
 		if !gCopyUtil.matchBlobNameAgainstPattern(blobNamePattern, blobItem.Name, cca.recursive) {
 			return false
 		}
-
 		includeExcludeMatchPath := common.IffString(includExcludeContainer,
 			azblob.NewBlobURLParts(srcContainerURL.URL()).ContainerName+"/"+blobItem.Name,
 			blobItem.Name)
@@ -184,7 +183,12 @@ func (e *copyBlobToNEnumerator) addTransfersFromContainer(ctx context.Context, s
 		if gCopyUtil.resourceShouldBeExcluded(parentSourcePath, e.Exclude, includeExcludeMatchPath) {
 			return false
 		}
-
+		// check if blobType of the current blob is present in the list of blob type to exclude.
+		for _, blobType := range e.ExcludeBlobType {
+			if blobItem.Properties.BlobType == blobType {
+				return false
+			}
+		}
 		return true
 	}
 

cmd/copyBlobToNEnumerator.go

@@ -150,6 +150,7 @@ Resume the existing job with the given job ID.`,
 			if err != nil {
 				glcm.Exit(fmt.Sprintf("failed to perform resume command due to error: %s", err.Error()), common.EExitCode.Error())
 			}
+			glcm.Exit("", common.EExitCode.Success())
 		},
 	}
 

cmd/jobsResume.go

@@ -44,7 +44,7 @@ func init() {
 		Aliases: []string{"ls"},
 		Short:   "List the entities in a given resource",
 		Long:    `List the entities in a given resource. Only Blob containers are supported at the moment.`,
-		Example: "azcopy container list [containerURL]",
+		Example: "azcopy list [containerURL]",
 		Args: func(cmd *cobra.Command, args []string) error {
 			// the listContainer command requires necessarily to have an argument
 

cmd/list.go

@@ -49,6 +49,7 @@ func init() {
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 			HandlePauseCommand(commandLineInput)
+			glcm.Exit("", common.EExitCode.Success())
 		},
 		// hide features not relevant to BFS
 		// TODO remove after preview release

cmd/pause.go

@@ -21,16 +21,24 @@
 package cmd
 
 import (
+	"bytes"
+	"context"
 	"github.com/Azure/azure-storage-azcopy/common"
+	"github.com/Azure/azure-storage-blob-go/2018-03-28/azblob"
 	"github.com/spf13/cobra"
+	"net/url"
+	"os"
+	"strings"
+	"time"
 )
 
 var azcopyAppPathFolder string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:   "azcopy",
-	Short: "AzCopy is a command line tool that moves data into/out of Azure Storage.",
+	Version: common.AzcopyVersion, // will enable the user to see the version info in the standard posix way: --version
+	Use:     "azcopy",
+	Short:   "AzCopy is a command line tool that moves data into/out of Azure Storage.",
 	Long: "AzCopy " + common.AzcopyVersion +
 		`
 Project URL: github.com/Azure/azure-storage-azcopy
@@ -40,6 +48,14 @@ To report issues or to learn more about the tool, go to github.com/Azure/azure-s
 
 The general format of the commands is: 'azcopy [command] [arguments] --[flag-name]=[flag-value]'.
 `,
+	PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		// spawn a routine to fetch and compare the local application's version against the latest version available
+		// if there's a newer version that can be used, then write the suggestion to stderr
+		// however if this takes too long the message won't get printed
+		// Note: this function is only triggered for non-help commands
+		go detectNewVersion()
+
+	},
 }
 
 // hold a pointer to the global lifecycle controller so that commands could output messages and exit properly
@@ -52,5 +68,76 @@ func Execute(azsAppPathFolder string) {
 
 	if err := rootCmd.Execute(); err != nil {
 		glcm.Exit(err.Error(), common.EExitCode.Error())
+	} else {
+		// our commands all control their own life explicitly with the lifecycle manager
+		// only help commands reach this point
+		// execute synchronously before exiting
+		detectNewVersion()
+	}
+}
+
+func detectNewVersion() {
+	const versionMetadataUrl = "https://aka.ms/azcopyv10-version-metadata"
+
+	// step 0: check the Stderr before checking version
+	_, err := os.Stderr.Stat()
+	if err != nil {
+		return
+	}
+
+	// step 1: initialize pipeline
+	p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{
+		Retry: azblob.RetryOptions{
+			Policy:        azblob.RetryPolicyExponential,
+			MaxTries:      1,               // try a single time, if network is not available, just fail fast
+			TryTimeout:    time.Second * 3, // don't wait for too long
+			RetryDelay:    downloadRetryDelay,
+			MaxRetryDelay: downloadMaxRetryDelay,
+		},
+		Telemetry: azblob.TelemetryOptions{
+			Value: common.UserAgent,
+		},
+	})
+
+	// step 2: parse source url
+	u, err := url.Parse(versionMetadataUrl)
+	if err != nil {
+		return
+	}
+
+	// step 3: start download
+	blobURL := azblob.NewBlobURL(*u, p)
+	blobStream, err := blobURL.Download(context.TODO(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false)
+	if err != nil {
+		return
+	}
+
+	blobBody := blobStream.Body(azblob.RetryReaderOptions{MaxRetryRequests: downloadMaxTries})
+	defer blobBody.Close()
+
+	// step 4: read newest version str
+	buf := new(bytes.Buffer)
+	n, err := buf.ReadFrom(blobBody)
+	if n == 0 || err != nil {
+		return
+	}
+	// only take the first line, in case the version metadata file is upgraded in the future
+	remoteVersion := strings.Split(buf.String(), "\n")[0]
+
+	// step 5: compare remote version to local version to see if there's a newer AzCopy
+	v1, err := NewVersion(common.AzcopyVersion)
+	if err != nil {
+		return
+	}
+	v2, err := NewVersion(remoteVersion)
+	if err != nil {
+		return
+	}
+
+	if v1.OlderThan(*v2) {
+		executablePathSegments := strings.Split(strings.Replace(os.Args[0], "\\", "/", -1), "/")
+		executableName := executablePathSegments[len(executablePathSegments)-1]
+		// print to stderr instead of stdout, in case the output is in other formats
+		glcm.Error(executableName + ": A newer version " + remoteVersion + " is available to download\n")
 	}
 }

cmd/root.go

@@ -0,0 +1,107 @@
+// Copyright © 2017 Microsoft <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package cmd
+
+import (
+	"errors"
+	"strconv"
+	"strings"
+)
+
+type Version struct {
+	segments []int64
+	preview  bool
+	original string
+}
+
+// To keep the code simple, we assume we only use a simple subset of semantic versions.
+// Namely, the version is either a normal stable version, or a pre-release version with '-preview' attached.
+// Examples: 10.1.0, 11.2.0-preview
+func NewVersion(raw string) (*Version, error) {
+	const standardError = "invalid version string"
+
+	rawSegments := strings.Split(raw, ".")
+	if len(rawSegments) != 3 {
+		return nil, errors.New(standardError)
+	}
+
+	v := &Version{segments: make([]int64, 3), original: raw}
+	for i, str := range rawSegments {
+		if strings.Contains(str, "-") {
+			if i != 2 {
+				return nil, errors.New(standardError)
+			}
+			v.preview = true
+			str = strings.Split(str, "-")[0]
+		}
+
+		val, err := strconv.ParseInt(str, 10, 64)
+		if err != nil {
+			return nil, errors.New("cannot version string")
+		}
+		v.segments[i] = val
+	}
+
+	return v, nil
+}
+
+// compare this version (v) to another version (v2)
+// return -1 if v is smaller/older than v2
+// return 0 if v is equal to v2
+// return 1 if v is bigger/newer than v2
+func (v Version) compare(v2 Version) int {
+	// short-circuit if the two version have the exact same raw string, no need to compare
+	if v.original == v2.original {
+		return 0
+	}
+
+	// compare the major/minor/patch version
+	// if v has a bigger number, it is newer
+	for i, num := range v.segments {
+		if num > v2.segments[i] {
+			return 1
+		} else if num < v2.segments[i] {
+			return -1
+		}
+	}
+
+	// if both or neither versions are previews, then they are equal
+	// usually this shouldn't happen since we already checked whether the two versions have equal raw string
+	// however, it is entirely possible that we have new kinds of pre-release versions that this code is not parsing correctly
+	// in this case we consider both pre-release version equal anyways
+	if (v.preview && v2.preview) || (!v.preview && !v2.preview) {
+		return 0
+	} else if v.preview && !v2.preview {
+		return -1
+	}
+
+	return 1
+}
+
+// detect if version v is older than v2
+func (v Version) OlderThan(v2 Version) bool {
+	return v.compare(v2) == -1
+}
+
+// detect if version v is newer than v2
+func (v Version) NewerThan(v2 Version) bool {
+	return v.compare(v2) == 1
+}