Fix silent failures with leading slash or no name virtual dirs

cmd/copy.go

@@ -390,7 +390,7 @@ func (raw *rawCopyCmdArgs) toOptions() (cooked CookedCopyCmdArgs, err error) {
 			cooked.FromTo)
 	}
 
-	// TODO: Figure out this preservePermissinos stuff
+	// TODO: Figure out this preservePermissions stuff
 	if cooked.preservePermissions.IsTruthy() && cooked.FromTo.From() == common.ELocation.Blob() {
 		// If a user is trying to persist from Blob storage with ACLs, they probably want directories too, because ACLs only exist in HNS.
 		cooked.IncludeDirectoryStubs = true

cmd/copyEnumeratorInit.go

@@ -35,6 +35,7 @@ func (cca *CookedCopyCmdArgs) validateSourceDir(traverser traverser.ResourceTrav
 	var err error
 	// Ensure we're only copying a directory under valid conditions
 	cca.IsSourceDir, err = traverser.IsDirectory(true)
+
 	if cca.IsSourceDir &&
 		!cca.Recursive && // Copies the folder & everything under it
 		!cca.StripTopDir { // Copies only everything under it

cmd/copyUtil.go

@@ -85,6 +85,11 @@ func stripTrailingWildcardOnRemoteSource(source string, location common.Location
 		return
 	}
 
+	if strings.HasSuffix(resourceURL.RawPath, "//*") {
+		if location == common.ELocation.Blob() && strings.HasSuffix(gURLParts.GetObjectName(), "/*") {
+			glcm.Info("Using a leading slash ('/') in blob name is supported in the storage service. \n However, it is not recommended when using AzCopy as it might result in unwanted behavior.")
+		}
+	}
 	// Trim the trailing /*.
 	if strings.HasSuffix(resourceURL.RawPath, "/*") {
 		resourceURL.RawPath = strings.TrimSuffix(resourceURL.RawPath, "/*")

e2etest/zt_newe2e_blob_test.go

@@ -1,6 +1,7 @@
 package e2etest
 
 import (
+	"github.com/Azure/azure-storage-azcopy/v10/cmd"
 	"strconv"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
@@ -217,3 +218,217 @@ func (s *BlobTestSuite) Scenario_DownloadBlobRecursive(svm *ScenarioVariationMan
 		validateObjectContent: true,
 	})
 }
+
+/*
+Scenario_DownloadBlobNoNameDirectory validates we report errors when
+downloading from a blobURL containing an unnamed directory and a wildcard pattern. I.e '//*'
+
+E.g https:/acct.blob/container//* AzCopy and the AzBlob SDK URL parser normalizes the path
+in occurrences - stripTrailingWildCardOnRemoteSources(), Traverser.SplitResourceString()
+
+Which causes the path to not be found. This tests we do not fail silently in that case.
+*/
+func (s *BlobTestSuite) Scenario_DownloadBlobNoNameDirectory(svm *ScenarioVariationManager) {
+	// Test uses a two-step upload, download to replicate the leading-slash blob scenario without tripping the SDK’s empty-name validation
+
+	body := NewRandomObjectContentContainer(SizeFromString("1K"))
+	blobContainer := CreateResource[ContainerResourceManager](svm,
+		GetRootResource(svm, common.ELocation.Blob(), GetResourceOptions{PreferredAccount: pointerTo(PrimaryStandardAcct)}),
+		ResourceDefinitionContainer{},
+	)
+
+	srcLocal := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
+	srcFile := CreateResource[ObjectResourceManager](svm, srcLocal, ResourceDefinitionObject{
+		ObjectName: pointerTo("image.png"),
+		Body:       body,
+		ObjectProperties: ObjectProperties{
+			EntityType: common.EEntityType.File(),
+		},
+	})
+
+	dstLocal := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
+
+	// Upload to https:/acct/container//image.png
+	RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			srcFile,
+			TryApplySpecificAuthType(blobContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "//image.png"}),
+		},
+		Flags: CopyFlags{
+			CopySyncCommonFlags: CopySyncCommonFlags{
+				Recursive: pointerTo(true),
+			},
+		},
+	})
+	ValidateResource[ObjectResourceManager](svm, blobContainer.GetObject(svm, "/image.png", common.EEntityType.File()), ResourceDefinitionObject{
+		Body: body,
+	}, ValidateResourceOptions{validateObjectContent: false})
+
+	// Download using list-of-files from https://container//
+	stdOut, _ := RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			TryApplySpecificAuthType(blobContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "//*"}),
+			dstLocal,
+		},
+		Flags: CopyFlags{
+			ListOfFiles: []string{"image.png"},
+			CopySyncCommonFlags: CopySyncCommonFlags{
+				Recursive: pointerTo(true),
+			},
+		},
+		ShouldFail: true,
+	})
+	ValidateMessageOutput(svm, stdOut, "The specified file was not found.", true)
+}
+
+// Scenario_DownloadBlobNamedDirectory validates list-of-files compatible with downloads from *named* virtual directories
+func (s *BlobTestSuite) Scenario_DownloadBlobNamedDirectory(svm *ScenarioVariationManager) {
+	// Test uses a two-step upload, download to replicate the leading-slash blob scenario without tripping the SDK’s empty-name validation
+	body := NewRandomObjectContentContainer(SizeFromString("1K"))
+	srcCont := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{})
+	namedBlobDirectory := CreateResource[ObjectResourceManager](svm, srcCont, ResourceDefinitionObject{
+		ObjectName: pointerTo("dir"),
+		ObjectProperties: ObjectProperties{
+			EntityType: common.EEntityType.Folder()},
+		ObjectShouldExist: pointerTo(true),
+		Body:              body,
+	})
+
+	srcObjs := make(ObjectResourceMappingFlat)
+	obj := ResourceDefinitionObject{
+		ObjectName:       pointerTo("dir/file.txt"),
+		Body:             body,
+		ObjectProperties: ObjectProperties{EntityType: common.EEntityType.File()}}
+	CreateResource[ObjectResourceManager](svm, srcCont, obj)
+	srcObjs["dir/file.txt"] = obj
+
+	dstFolder := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionObject{
+		ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()},
+	})
+
+	RunAzCopy(
+		svm, AzCopyCommand{
+			Verb: AzCopyVerbCopy,
+			Targets: []ResourceManager{
+				namedBlobDirectory, dstFolder,
+			},
+			Flags: CopyFlags{
+				ListOfFiles: []string{"file.txt"},
+				CopySyncCommonFlags: CopySyncCommonFlags{
+					Recursive: pointerTo(true),
+					GlobalFlags: GlobalFlags{
+						OutputType: pointerTo(cmd.EOutputFormat.Text()),
+					},
+				},
+			},
+			ShouldFail: false,
+		})
+	ValidateResource[ObjectResourceManager](svm, dstFolder,
+		ResourceDefinitionObject{ObjectName: pointerTo("file.txt")},
+		ValidateResourceOptions{validateObjectContent: false})
+
+}
+
+// Scenario_DownloadBlobNoNameWithoutWildcardDirectory tests downloading blobs with leading slash paths (like "/image.png") and list-of-files
+// work without using a wildcard
+func (s *BlobTestSuite) Scenario_DownloadBlobNoNameWithoutWildcardDirectory(svm *ScenarioVariationManager) {
+	// Test uses a two-step upload, download to replicate the leading-slash blob scenario without tripping the SDK’s empty-name validation
+
+	body := NewRandomObjectContentContainer(SizeFromString("1K"))
+	blobContainer := CreateResource[ContainerResourceManager](svm,
+		GetRootResource(svm, common.ELocation.Blob(), GetResourceOptions{PreferredAccount: pointerTo(PrimaryStandardAcct)}),
+		ResourceDefinitionContainer{},
+	)
+
+	srcLocal := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
+	srcFile := CreateResource[ObjectResourceManager](svm, srcLocal, ResourceDefinitionObject{
+		ObjectName: pointerTo("image.png"),
+		Body:       body,
+		ObjectProperties: ObjectProperties{
+			EntityType: common.EEntityType.File(),
+		},
+	})
+
+	dstLocal := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionObject{
+		ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()},
+	})
+
+	// Upload to https:/acct/container//image.png
+	RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			srcFile,
+			TryApplySpecificAuthType(blobContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "//image.png"}),
+		},
+		Flags: CopyFlags{
+			CopySyncCommonFlags: CopySyncCommonFlags{
+				Recursive: pointerTo(true),
+			},
+		},
+	})
+	ValidateResource[ObjectResourceManager](svm, blobContainer.GetObject(svm, "/image.png", common.EEntityType.File()), ResourceDefinitionObject{
+		Body: body,
+	}, ValidateResourceOptions{validateObjectContent: false})
+
+	// Download using list-of-files from https://container//
+	RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			TryApplySpecificAuthType(blobContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "//"}),
+			dstLocal,
+		},
+		Flags: CopyFlags{
+			ListOfFiles: []string{"image.png"},
+			CopySyncCommonFlags: CopySyncCommonFlags{
+				Recursive: pointerTo(true),
+				GlobalFlags: GlobalFlags{
+					OutputType: pointerTo(cmd.EOutputFormat.Text()),
+				},
+			},
+		},
+		ShouldFail: false,
+	})
+	ValidateResource[ObjectResourceManager](svm, dstLocal,
+		ResourceDefinitionObject{Body: body},
+		ValidateResourceOptions{validateObjectContent: false})
+}
+
+func (s *BlobTestSuite) Scenario_DownloadBlobObjNoNameDirectory(svm *ScenarioVariationManager) {
+	// Test uses a two-step upload, download upload then download to replicate the leading-slash blob scenario without tripping the SDK’s empty-name validation
+	blobContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{})
+	localSrc := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionObject{
+		ObjectName:       pointerTo("a.txt"),
+		Body:             NewRandomObjectContentContainer(SizeFromString("1K")),
+		ObjectProperties: ObjectProperties{EntityType: common.EEntityType.File()},
+	})
+	RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			localSrc,
+			TryApplySpecificAuthType(blobContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "//a.txt"}),
+		},
+		Flags: CopyFlags{CopySyncCommonFlags: CopySyncCommonFlags{Recursive: pointerTo(true)}},
+	})
+
+	dstLocal := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{})
+	blobObj := blobContainer.GetObject(svm, "/a.txt", common.EEntityType.File())
+	RunAzCopy(svm, AzCopyCommand{
+		Verb: AzCopyVerbCopy,
+		Targets: []ResourceManager{
+			blobObj,
+			dstLocal,
+		},
+		Flags: CopyFlags{
+			CopySyncCommonFlags: CopySyncCommonFlags{
+				Recursive: pointerTo(true),
+				FromTo:    pointerTo(common.EFromTo.BlobLocal()),
+				GlobalFlags: GlobalFlags{
+					OutputType: pointerTo(cmd.EOutputFormat.Text()),
+				},
+			},
+		},
+		ShouldFail: true,
+	})
+}

traverser/zc_traverser_list.go

@@ -23,7 +23,9 @@ package traverser
 import (
 	"context"
 	"fmt"
+	blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"net/url"
+	"strings"
 
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
@@ -46,6 +48,9 @@ func (l *listTraverser) IsDirectory(bool) (bool, error) {
 // To kill the traverser, close() the channel under it.
 // Behavior demonstrated: https://play.golang.org/p/OYdvLmNWgwO
 func (l *listTraverser) Traverse(preprocessor objectMorpher, processor ObjectProcessor, filters []ObjectFilter) (err error) {
+	itemsProcessed := 0
+	itemsSkipped := 0
+	var lastError error
 	// read a channel until it closes to get a list of objects
 
 	childPath, ok := <-l.listReader
@@ -57,11 +62,26 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor ObjectPro
 		childTraverser, err := l.childTraverserGenerator(childPath)
 		if err != nil {
 			common.GetLifecycleMgr().Info(fmt.Sprintf("Skipping %s due to error %s", childPath, err))
+			itemsSkipped++
+			lastError = err
 			continue
 		}
 		// listTraverser will only ever execute on the source
 
-		isDir, _ := childTraverser.IsDirectory(true)
+		// Handle error to avoid silent failures
+		isDir, isDirErr := childTraverser.IsDirectory(true)
+
+		if isDirErr != nil {
+			if strings.Contains(isDirErr.Error(), common.FILE_NOT_FOUND) {
+				common.GetLifecycleMgr().Info(fmt.Sprintf("Skipping %s: file/directory not found. ", childPath))
+				bURlParts, _ := blobsas.ParseURL(childTraverser.(*BlobTraverser).RawURL)
+				common.GetLifecycleMgr().Info(fmt.Sprintf("'%s' path does not exist. Rename blob name without leading slash", bURlParts.ContainerName+"/"+bURlParts.BlobName))
+				itemsSkipped++
+				lastError = isDirErr
+				continue
+			}
+		}
+
 		if !l.recursive && isDir {
 			continue // skip over directories
 		}
@@ -83,6 +103,16 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor ObjectPro
 		err = childTraverser.Traverse(preProcessorForThisChild, processor, filters)
 		if err != nil {
 			common.GetLifecycleMgr().Info(fmt.Sprintf("Skipping %s as it cannot be scanned due to error: %s", childPath, err))
+			itemsSkipped++
+			lastError = err
+		} else {
+			itemsProcessed++
+		}
+	}
+	// Return err if nothing is processed
+	if itemsProcessed == 0 && itemsSkipped > 0 {
+		if lastError != nil {
+			return fmt.Errorf(": failed to process files. \n Error: %w", lastError)
 		}
 	}