From 7f5ab059c28b09841d009d28b7510d4fcb167dc8 Mon Sep 17 00:00:00 2001 From: wonwuakpa-msft Date: Wed, 11 Dec 2024 12:42:31 -0500 Subject: [PATCH 1/3] Added support for trailing dot transfers to unsafe locations (#2827) * Add AllowToUnsafeDestination * implementation to stop msg sends on a closed channel * RWMutex added on channel closed flag * mutex synchronism mechanism added to channel sends and closures * replaced mutex panic handling implementation with recover * replace console msgs to log msg * modified panic handling * removed unnecessary print stmt * added unsafe destination trailing dot option and tests * testing for trailing dot * added unsafe destination trailing dot option and tests * testing for trailing dot * set AllowToUnsafeDestination on client + tests * test for third trailing dot option * widened the client options type * address Pr comments * Update e2etest/zt_newe2e_basic_functionality_test.go Co-authored-by: Gauri Lamunion <51212198+gapra-msft@users.noreply.github.com> * small fixes * fix tests * cleaned tests * fixed tests * removed list test * trailing dot should be stripped in tests --------- Co-authored-by: Adele Reed Co-authored-by: Gauri Lamunion <51212198+gapra-msft@users.noreply.github.com> --- cmd/copy.go | 14 ++--- cmd/jobsResume.go | 38 +++++++++++--- cmd/removeEnumerator.go | 2 +- cmd/rpc.go | 4 +- cmd/setPropertiesEnumerator.go | 2 +- cmd/zc_enumerator.go | 2 +- cmd/zc_traverser_file.go | 30 ++++++----- cmd/zt_interceptors_for_test.go | 2 +- common/fe-ste-models.go | 12 ++++- common/rpc-models.go | 11 ++-- e2etest/newe2e_resource_managers_local.go | 8 +-- e2etest/newe2e_task_runazcopy_parameters.go | 2 +- e2etest/zt_newe2e_basic_functionality_test.go | 52 +++++++++++++++++++ e2etest/zt_newe2e_file_test.go | 36 +++++++++++++ go.sum | 4 -- jobsAdmin/init.go | 21 ++++---- ste/JobPartPlan.go | 2 +- ste/jobStatusManager.go | 8 +-- ste/mgr-JobMgr.go | 2 +- 19 files changed, 186 insertions(+), 66 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index c7a87d54d..aeabba8e4 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1568,7 +1568,7 @@ func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) { var azureFileSpecificOptions any if cca.FromTo.From() == common.ELocation.File() { azureFileSpecificOptions = &common.FileClientOptions{ - AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), + AllowTrailingDot: cca.trailingDot.IsEnabled(), } } @@ -1590,8 +1590,8 @@ func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) { if cca.FromTo.To() == common.ELocation.File() { azureFileSpecificOptions = &common.FileClientOptions{ - AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable(), - AllowSourceTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable() && cca.FromTo.From() == common.ELocation.File(), + AllowTrailingDot: cca.trailingDot.IsEnabled(), + AllowSourceTrailingDot: cca.trailingDot.IsEnabled() && cca.FromTo.From() == common.ELocation.File(), } } @@ -2134,9 +2134,11 @@ func init() { // so properties can be get in parallel, at same time no additional go routines are created for this specific job. // The usage of this hidden flag is to provide fallback to traditional behavior, when service supports returning full properties during list. cpCmd.PersistentFlags().BoolVar(&raw.s2sGetPropertiesInBackend, "s2s-get-properties-in-backend", true, "True by default. Gets S3 objects' or Azure files' properties in backend, if properties need to be accessed. Properties need to be accessed if s2s-preserve-properties is true, and in certain other cases where we need the properties for modification time checks or MD5 checks.") - cpCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "'Enable' by default to treat file share related operations in a safe manner. Available options: "+strings.Join(common.ValidTrailingDotOptions(), ", ")+". "+ - "Choose 'Disable' to go back to legacy (potentially unsafe) treatment of trailing dot files where the file service will trim any trailing dots in paths. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (ex: mypath and mypath.). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation."+ - "If the destination does not support trailing dot files (Windows or Blob Storage), AzCopy will fail if the trailing dot file is the root of the transfer and skip any trailing dot paths encountered during enumeration.") + cpCmd.PersistentFlags().StringVar(&raw.trailingDot, "trailing-dot", "", "Available options: "+strings.Join(common.ValidTrailingDotOptions(), ", ")+". "+ + "'Enable'(Default) treats trailing dot file operations in a safe manner between systems that support these files. On Windows, the transfers will not occur to stop risk of data corruption. See 'AllowToUnsafeDestination' to bypass this."+ + "'Disable' reverts to the legacy functionality, where trailing dot files are ignored. This can result in potential data corruption if the transfer contains two paths that differ only by a trailing dot (E.g 'path/foo' and 'path/foo.'). If this flag is set to 'Disable' and AzCopy encounters a trailing dot file, it will warn customers in the scanning log but will not attempt to abort the operation."+ + "If the destination does not support trailing dot files (Windows or Blob Storage), AzCopy will fail if the trailing dot file is the root of the transfer and skip any trailing dot paths encountered during enumeration."+ + "'AllowToUnsafeDestination' supports transferring trailing dot files to systems that do not support them e.g Windows. Use with caution acknowledging risk of data corruption, when two files with different contents 'path/bar' and 'path/bar.' (differ only by a trailing dot) are seen as identical.") // Public Documentation: https://docs.microsoft.com/en-us/azure/storage/blobs/encryption-customer-provided-keys // Clients making requests against Azure Blob storage have the option to provide an encryption key on a per-request basis. diff --git a/cmd/jobsResume.go b/cmd/jobsResume.go index 053e6e319..5d8b8acc7 100644 --- a/cmd/jobsResume.go +++ b/cmd/jobsResume.go @@ -295,8 +295,28 @@ func (rca resumeCmdArgs) getSourceAndDestinationServiceClients( } options := createClientOptions(common.AzcopyCurrentJobLogger, nil) + jobID, err := common.ParseJobID(rca.jobID) + if err != nil { + // Error for invalid JobId format + return nil, nil, fmt.Errorf("error parsing the jobId %s. Failed with error %s", rca.jobID, err.Error()) + } + + var getJobDetailsResponse common.GetJobDetailsResponse + // Get job details from the STE + Rpc(common.ERpcCmd.GetJobDetails(), + &common.GetJobDetailsRequest{JobID: jobID}, + &getJobDetailsResponse) + if getJobDetailsResponse.ErrorMsg != "" { + glcm.Error(getJobDetailsResponse.ErrorMsg) + } - srcServiceClient, err := common.GetServiceClientForLocation(fromTo.From(), source, srcCredType, tc, &options, nil) + var fileSrcClientOptions any + if fromTo.From() == common.ELocation.File() { + fileSrcClientOptions = &common.FileClientOptions{ + AllowTrailingDot: getJobDetailsResponse.TrailingDot.IsEnabled(), //Access the trailingDot option of the job + } + } + srcServiceClient, err := common.GetServiceClientForLocation(fromTo.From(), source, srcCredType, tc, &options, fileSrcClientOptions) if err != nil { return nil, nil, err } @@ -306,11 +326,17 @@ func (rca resumeCmdArgs) getSourceAndDestinationServiceClients( srcCred = common.NewScopedCredential(tc, srcCredType) } options = createClientOptions(common.AzcopyCurrentJobLogger, srcCred) - dstServiceClient, err := common.GetServiceClientForLocation(fromTo.To(), destination, dstCredType, tc, &options, nil) + var fileClientOptions any + if fromTo.To() == common.ELocation.File() { + fileClientOptions = &common.FileClientOptions{ + AllowSourceTrailingDot: getJobDetailsResponse.TrailingDot.IsEnabled() && fromTo.From() == common.ELocation.File(), + AllowTrailingDot: getJobDetailsResponse.TrailingDot.IsEnabled(), + } + } + dstServiceClient, err := common.GetServiceClientForLocation(fromTo.To(), destination, dstCredType, tc, &options, fileClientOptions) if err != nil { return nil, nil, err } - return srcServiceClient, dstServiceClient, nil } @@ -357,9 +383,9 @@ func (rca resumeCmdArgs) process() error { } // Get fromTo info, so we can decide what's the proper credential type to use. - var getJobFromToResponse common.GetJobFromToResponse - Rpc(common.ERpcCmd.GetJobFromTo(), - &common.GetJobFromToRequest{JobID: jobID}, + var getJobFromToResponse common.GetJobDetailsResponse + Rpc(common.ERpcCmd.GetJobDetails(), + &common.GetJobDetailsRequest{JobID: jobID}, &getJobFromToResponse) if getJobFromToResponse.ErrorMsg != "" { glcm.Error(getJobFromToResponse.ErrorMsg) diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go index e7ce7213a..f1f1c4339 100755 --- a/cmd/removeEnumerator.go +++ b/cmd/removeEnumerator.go @@ -93,7 +93,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er options := createClientOptions(common.AzcopyCurrentJobLogger, nil) var fileClientOptions any if cca.FromTo.From() == common.ELocation.File() { - fileClientOptions = &common.FileClientOptions{AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable()} + fileClientOptions = &common.FileClientOptions{AllowTrailingDot: cca.trailingDot.IsEnabled()} } targetServiceClient, err := common.GetServiceClientForLocation( cca.FromTo.From(), diff --git a/cmd/rpc.go b/cmd/rpc.go index 49f4967d2..3b1ee9ddf 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -59,8 +59,8 @@ func inprocSend(rpcCmd common.RpcCmd, requestData interface{}, responseData inte case common.ERpcCmd.ResumeJob(): *(responseData.(*common.CancelPauseResumeResponse)) = jobsAdmin.ResumeJobOrder(*requestData.(*common.ResumeJobRequest)) - case common.ERpcCmd.GetJobFromTo(): - *(responseData.(*common.GetJobFromToResponse)) = jobsAdmin.GetJobFromTo(*requestData.(*common.GetJobFromToRequest)) + case common.ERpcCmd.GetJobDetails(): + *(responseData.(*common.GetJobDetailsResponse)) = jobsAdmin.GetJobDetails(*requestData.(*common.GetJobDetailsRequest)) default: panic(fmt.Errorf("Unrecognized RpcCmd: %q", rpcCmd.String())) diff --git a/cmd/setPropertiesEnumerator.go b/cmd/setPropertiesEnumerator.go index 14e9be84e..10c3a5b4a 100755 --- a/cmd/setPropertiesEnumerator.go +++ b/cmd/setPropertiesEnumerator.go @@ -75,7 +75,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator options := createClientOptions(common.AzcopyCurrentJobLogger, nil) var fileClientOptions any if cca.FromTo.From() == common.ELocation.File() { - fileClientOptions = &common.FileClientOptions{AllowTrailingDot: cca.trailingDot == common.ETrailingDotOption.Enable()} + fileClientOptions = &common.FileClientOptions{AllowTrailingDot: cca.trailingDot.IsEnabled()} } targetServiceClient, err := common.GetServiceClientForLocation( diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 3c0a1ec97..66836294b 100644 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -495,7 +495,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat fileURLParts.ShareSnapshot = "" fileURLParts.DirectoryOrFilePath = "" fileOptions := &common.FileClientOptions{ - AllowTrailingDot: trailingDot == common.ETrailingDotOption.Enable(), + AllowTrailingDot: trailingDot.IsEnabled(), } res, err := SplitResourceString(fileURLParts.String(), common.ELocation.File()) diff --git a/cmd/zc_traverser_file.go b/cmd/zc_traverser_file.go index 394def463..9277b741f 100644 --- a/cmd/zc_traverser_file.go +++ b/cmd/zc_traverser_file.go @@ -48,8 +48,8 @@ type fileTraverser struct { // a generic function to notify that a new stored object has been enumerated incrementEnumerationCounter enumerationCounterFunc - trailingDot common.TrailingDotOption - destination *common.Location + trailingDot common.TrailingDotOption + destination *common.Location } func createShareClientFromServiceClient(fileURLParts file.URLParts, client *service.Client) (*share.Client, error) { @@ -123,6 +123,10 @@ func (t *fileTraverser) getPropertiesIfSingleFile() (*file.GetPropertiesResponse func (t *fileTraverser) Traverse(preprocessor objectMorpher, processor objectProcessor, filters []ObjectFilter) (err error) { invalidBlobOrWindowsName := func(path string) bool { if t.destination != nil { + if t.trailingDot == common.ETrailingDotOption.AllowToUnsafeDestination() && (*t.destination != common.ELocation.Blob() || *t.destination != common.ELocation.BlobFS()) { // Allow only Local, Trailing dot files not supported in Blob + return false // Please let me shoot myself in the foot! + } + if (t.destination.IsLocal() && runtime.GOOS == "windows") || *t.destination == common.ELocation.Blob() || *t.destination == common.ELocation.BlobFS() { /* Blob or Windows object name is invalid if it ends with period or one of (virtual) directories in path ends with period. @@ -145,7 +149,7 @@ func (t *fileTraverser) Traverse(preprocessor objectMorpher, processor objectPro WarnStdoutAndScanningLog(fmt.Sprintf(invalidNameErrorMsg, targetURLParts.DirectoryOrFilePath)) return common.EAzError.InvalidBlobOrWindowsName() } - if t.trailingDot != common.ETrailingDotOption.Enable() && strings.HasSuffix(targetURLParts.DirectoryOrFilePath, ".") { + if !t.trailingDot.IsEnabled() && strings.HasSuffix(targetURLParts.DirectoryOrFilePath, ".") { azcopyScanningLogger.Log(common.LogWarning, fmt.Sprintf(trailingDotErrMsg, getObjectNameOnly(targetURLParts.DirectoryOrFilePath))) } // check if the url points to a single file @@ -288,7 +292,7 @@ func (t *fileTraverser) Traverse(preprocessor objectMorpher, processor objectPro WarnStdoutAndScanningLog(fmt.Sprintf(invalidNameErrorMsg, *fileInfo.Name)) continue } else { - if t.trailingDot != common.ETrailingDotOption.Enable() && strings.HasSuffix(*fileInfo.Name, ".") { + if !t.trailingDot.IsEnabled() && strings.HasSuffix(*fileInfo.Name, ".") { azcopyScanningLogger.Log(common.LogWarning, fmt.Sprintf(trailingDotErrMsg, *fileInfo.Name)) } } @@ -301,13 +305,13 @@ func (t *fileTraverser) Traverse(preprocessor objectMorpher, processor objectPro WarnStdoutAndScanningLog(fmt.Sprintf(invalidNameErrorMsg, *dirInfo.Name)) continue } else { - if t.trailingDot != common.ETrailingDotOption.Enable() && strings.HasSuffix(*dirInfo.Name, ".") { + if !t.trailingDot.IsEnabled() && strings.HasSuffix(*dirInfo.Name, ".") { azcopyScanningLogger.Log(common.LogWarning, fmt.Sprintf(trailingDotErrMsg, *dirInfo.Name)) } } enqueueOutput(newAzFileSubdirectoryEntity(currentDirectoryClient, *dirInfo.Name), nil) if t.recursive { - // If recursive is turned on, add sub directories to be processed + // If recursive is turned on, add sub directories to be processed enqueueDir(currentDirectoryClient.NewSubdirectoryClient(*dirInfo.Name)) } @@ -381,14 +385,14 @@ func (t *fileTraverser) Traverse(preprocessor objectMorpher, processor objectPro func newFileTraverser(rawURL string, serviceClient *service.Client, ctx context.Context, recursive, getProperties bool, incrementEnumerationCounter enumerationCounterFunc, trailingDot common.TrailingDotOption, destination *common.Location) (t *fileTraverser) { t = &fileTraverser{ - rawURL: rawURL, - serviceClient: serviceClient, - ctx: ctx, - recursive: recursive, - getProperties: getProperties, + rawURL: rawURL, + serviceClient: serviceClient, + ctx: ctx, + recursive: recursive, + getProperties: getProperties, incrementEnumerationCounter: incrementEnumerationCounter, - trailingDot: trailingDot, - destination: destination, + trailingDot: trailingDot, + destination: destination, } return } diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index 69276815d..68f7cd0b7 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -53,7 +53,7 @@ func (i *interceptor) intercept(cmd common.RpcCmd, request interface{}, response case common.ERpcCmd.PauseJob(): case common.ERpcCmd.CancelJob(): case common.ERpcCmd.ResumeJob(): - case common.ERpcCmd.GetJobFromTo(): + case common.ERpcCmd.GetJobDetails(): fallthrough default: panic("RPC mock not implemented") diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 6eba969a1..09c466b6b 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -153,8 +153,15 @@ var ETrailingDotOption = TrailingDotOption(0) type TrailingDotOption uint8 -func (TrailingDotOption) Enable() TrailingDotOption { return TrailingDotOption(0) } -func (TrailingDotOption) Disable() TrailingDotOption { return TrailingDotOption(1) } +func (TrailingDotOption) Enable() TrailingDotOption { return TrailingDotOption(0) } +func (TrailingDotOption) Disable() TrailingDotOption { return TrailingDotOption(1) } +func (TrailingDotOption) AllowToUnsafeDestination() TrailingDotOption { return TrailingDotOption(2) } + +// Trailing dots are supported in the Enable and AllowToUnsafeDestination options +func (d TrailingDotOption) IsEnabled() bool { + return d == d.Enable() || + d == d.AllowToUnsafeDestination() +} func (d TrailingDotOption) String() string { return enum.StringInt(d, reflect.TypeOf(d)) @@ -178,6 +185,7 @@ func ValidTrailingDotOptions() []string { return []string{ ETrailingDotOption.Enable().String(), ETrailingDotOption.Disable().String(), + ETrailingDotOption.AllowToUnsafeDestination().String(), } } diff --git a/common/rpc-models.go b/common/rpc-models.go index 1e0901a70..3c6f570da 100644 --- a/common/rpc-models.go +++ b/common/rpc-models.go @@ -27,7 +27,7 @@ func (RpcCmd) ListJobTransfers() RpcCmd { return RpcCmd("ListJobTransfers") } func (RpcCmd) CancelJob() RpcCmd { return RpcCmd("Cancel") } func (RpcCmd) PauseJob() RpcCmd { return RpcCmd("PauseJob") } func (RpcCmd) ResumeJob() RpcCmd { return RpcCmd("ResumeJob") } -func (RpcCmd) GetJobFromTo() RpcCmd { return RpcCmd("GetJobFromTo") } +func (RpcCmd) GetJobDetails() RpcCmd { return RpcCmd("GetJobDetails") } func (c RpcCmd) String() string { return enum.String(c, reflect.TypeOf(c)) @@ -375,15 +375,16 @@ type ListJobTransfersResponse struct { Details []TransferDetail } -// GetJobFromToRequest indicates request to get job's FromTo info from job part plan header -type GetJobFromToRequest struct { +// GetJobDetailsRequest indicates request to get job's FromTo and TrailingDot info from job part plan header +type GetJobDetailsRequest struct { JobID JobID } -// GetJobFromToResponse indicates response to get job's FromTo info. -type GetJobFromToResponse struct { +// GetJobDetailsResponse indicates response to get job's FromTo and TrailingDot info. +type GetJobDetailsResponse struct { ErrorMsg string FromTo FromTo Source string Destination string + TrailingDot TrailingDotOption } diff --git a/e2etest/newe2e_resource_managers_local.go b/e2etest/newe2e_resource_managers_local.go index 79b21ad2c..901ca42d6 100644 --- a/e2etest/newe2e_resource_managers_local.go +++ b/e2etest/newe2e_resource_managers_local.go @@ -320,11 +320,11 @@ func (l *LocalObjectResourceManager) ListChildren(a Asserter, recursive bool) ma func (l *LocalObjectResourceManager) GetProperties(a Asserter) ObjectProperties { a.HelperMarker().Helper() stats, err := os.Stat(l.getWorkingPath()) - a.NoError("failed to get stat", err) - var lmt *time.Time - if stats != nil { - lmt = PtrOf(stats.ModTime()) + if err != nil { // Prevent nil dereferences + a.NoError("failed to get stat", err) + return ObjectProperties{} } + lmt := common.Iff(stats == nil, nil, PtrOf(stats.ModTime())) out := ObjectProperties{ LastModifiedTime: lmt, } diff --git a/e2etest/newe2e_task_runazcopy_parameters.go b/e2etest/newe2e_task_runazcopy_parameters.go index 259531eae..633d4fa23 100644 --- a/e2etest/newe2e_task_runazcopy_parameters.go +++ b/e2etest/newe2e_task_runazcopy_parameters.go @@ -191,7 +191,7 @@ type GlobalFlags struct { OutputLevel *common.OutputVerbosity `flag:"output-level,default:DEFAULT"` // TODO: reconsider/reengineer this flag; WI#26475473 - //DebugSkipFiles []string `flag:"debug-skip-files"` + // DebugSkipFiles []string `flag:"debug-skip-files"` // TODO: handle prompting and input; WI#26475441 //CancelFromStdin *bool `flag:"cancel-from-stdin"` diff --git a/e2etest/zt_newe2e_basic_functionality_test.go b/e2etest/zt_newe2e_basic_functionality_test.go index bbd46818f..8b07ee1f7 100644 --- a/e2etest/zt_newe2e_basic_functionality_test.go +++ b/e2etest/zt_newe2e_basic_functionality_test.go @@ -1,6 +1,7 @@ package e2etest import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-storage-azcopy/v10/common" "strconv" @@ -412,6 +413,57 @@ func (s *BasicFunctionalitySuite) Scenario_Copy_EmptySASErrorCodes(svm *Scenario ValidateContainsError(svm, stdout, []string{"https://aka.ms/AzCopyError/NoAuthenticationInformation", "https://aka.ms/AzCopyError/ResourceNotFound"}) } +// Test of Copy to UnsafeDestinations (Windows Local dest) +func (s *BasicFunctionalitySuite) Scenario_CopyUnSafeDest(svm *ScenarioVariationManager) { + azCopyVerb := AzCopyVerbCopy + + fileName := "file." + body := NewRandomObjectContentContainer(SizeFromString("10K")) + + // Scale up from service to object + srcObj := CreateResource[ObjectResourceManager](svm, + GetRootResource(svm, common.ELocation.File()), // Only source is File - Windows Local & Blob doesn't support T.D + ResourceDefinitionObject{ + ObjectName: pointerTo(fileName), + Body: body, + }) + + dstObj := CreateResource[ContainerResourceManager](svm, + GetRootResource(svm, + ResolveVariation(svm, []common.Location{common.ELocation.File(), + common.ELocation.Local()})), + ResourceDefinitionContainer{}).GetObject(svm, "file", common.EEntityType.File()) + + sasOpts := GenericAccountSignatureValues{} + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: azCopyVerb, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcObj, EExplicitCredentialType.SASToken(), svm, + CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + TryApplySpecificAuthType(dstObj, EExplicitCredentialType.SASToken(), svm, + CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + TrailingDot: to.Ptr(common.ETrailingDotOption.AllowToUnsafeDestination()), // Allow download to unsafe Local destination + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, + ResourceDefinitionObject{ + Body: body, + }, false) +} + func (s *BasicFunctionalitySuite) Scenario_TagsPermission(svm *ScenarioVariationManager) { objectType := ResolveVariation(svm, []common.EntityType{common.EEntityType.File(), common.EEntityType.Folder(), common.EEntityType.Symlink()}) srcLoc := ResolveVariation(svm, []common.Location{common.ELocation.Local(), common.ELocation.Blob()}) diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 882124a76..bf8b2d0c9 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -3,6 +3,7 @@ package e2etest import ( "context" "fmt" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-storage-azcopy/v10/common" "math" "strconv" @@ -510,3 +511,38 @@ func (s *FileTestSuite) Scenario_SeveralFileDownloadWildcard(svm *ScenarioVariat Body: body, }, true) } + +// Test copy with AllowToUnsafeDestination option +func (s *FileTestSuite) Scenario_CopyTrailingDotUnsafeDestination(svm *ScenarioVariationManager) { + body := NewRandomObjectContentContainer(0) + + name := "test." + srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.File(), common.ELocation.Local()})), + ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body}) + dstObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Local(), common.ELocation.File()})), + ResourceDefinitionObject{ObjectName: pointerTo("test"), Body: body}) + + if srcObj.Location() == dstObj.Location() { + svm.InvalidateScenario() + return + } + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{TryApplySpecificAuthType(srcObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{}), + TryApplySpecificAuthType(dstObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + TrailingDot: to.Ptr(common.ETrailingDotOption.AllowToUnsafeDestination()), + }, + ListOfFiles: []string{"lof"}, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, false) +} diff --git a/go.sum b/go.sum index 8d183799d..b53d62184 100644 --- a/go.sum +++ b/go.sum @@ -21,14 +21,10 @@ cloud.google.com/go/storage v1.45.0 h1:5av0QcIVj77t+44mV4gffFC/LscFRUhto6UBMB5Si cloud.google.com/go/storage v1.45.0/go.mod h1:wpPblkIuMP5jCB/E48Pz9zIo2S/zD8g+ITmxKkPCITE= cloud.google.com/go/trace v1.11.1 h1:UNqdP+HYYtnm6lb91aNA5JQ0X14GnxkABGlfz2PzPew= cloud.google.com/go/trace v1.11.1/go.mod h1:IQKNQuBzH72EGaXEodKlNJrWykGZxet2zgjtS60OtjA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw= -github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 h1:Bk5uOhSAenHyR5P61D/NzeQCv+4fEVV8mOkJ82NqpWw= github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1/go.mod h1:QZ4pw3or1WPmRBxf0cHd1tknzrT54WPBOQoGutCPvSU= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= diff --git a/jobsAdmin/init.go b/jobsAdmin/init.go index 2c53b99a3..de68fc25b 100755 --- a/jobsAdmin/init.go +++ b/jobsAdmin/init.go @@ -141,11 +141,11 @@ func MainSTE(concurrency ste.ConcurrencySettings, targetRateInMegaBitsPerSec flo serialize(ResumeJobOrder(payload), writer) }) - http.HandleFunc(common.ERpcCmd.GetJobFromTo().Pattern(), + http.HandleFunc(common.ERpcCmd.GetJobDetails().Pattern(), func(writer http.ResponseWriter, request *http.Request) { - var payload common.GetJobFromToRequest + var payload common.GetJobDetailsRequest deserialize(request, &payload) - serialize(GetJobFromTo(payload), writer) + serialize(GetJobDetails(payload), writer) }) // Listen for front-end requests @@ -188,7 +188,7 @@ func ExecuteNewCopyJobPartOrder(order common.CopyJobPartOrderRequest) common.Cop ExistingPlanMMF: nil, SrcClient: order.SrcServiceClient, DstClient: order.DstServiceClient, - SrcIsOAuth: order.S2SSourceCredentialType.IsAzureOAuth(), + SrcIsOAuth: order.S2SSourceCredentialType.IsAzureOAuth(), ScheduleTransfers: true, } jm.AddJobPart2(args) @@ -698,14 +698,14 @@ func ListJobs(givenStatus common.JobStatus) common.ListJobsResponse { return JobsAdmin.ListJobs(givenStatus) } -// GetJobFromTo api returns the job FromTo info. -func GetJobFromTo(r common.GetJobFromToRequest) common.GetJobFromToResponse { +// GetJobDetails api returns the job FromTo info. +func GetJobDetails(r common.GetJobDetailsRequest) common.GetJobDetailsResponse { jm, found := JobsAdmin.JobMgr(r.JobID) if !found { // Job with JobId does not exists. // Search the plan files in Azcopy folder and resurrect the Job. if !JobsAdmin.ResurrectJob(r.JobID, EMPTY_SAS_STRING, EMPTY_SAS_STRING, nil, nil, false) { - return common.GetJobFromToResponse{ + return common.GetJobDetailsResponse{ ErrorMsg: fmt.Sprintf("Job with JobID %v does not exist or is invalid", r.JobID), } } @@ -715,7 +715,7 @@ func GetJobFromTo(r common.GetJobFromToRequest) common.GetJobFromToResponse { // Get zeroth part of the job part plan. jp0, ok := jm.JobPartMgr(0) if !ok { - return common.GetJobFromToResponse{ + return common.GetJobDetailsResponse{ ErrorMsg: fmt.Sprintf("error getting the job's FromTo with JobID %v", r.JobID), } } @@ -723,15 +723,16 @@ func GetJobFromTo(r common.GetJobFromToRequest) common.GetJobFromToResponse { // Use first transfer's source/destination as represent. source, destination, _ := jp0.Plan().TransferSrcDstStrings(0) if source == "" && destination == "" { - return common.GetJobFromToResponse{ + return common.GetJobDetailsResponse{ ErrorMsg: fmt.Sprintf("error getting the source/destination with JobID %v", r.JobID), } } - return common.GetJobFromToResponse{ + return common.GetJobDetailsResponse{ ErrorMsg: "", FromTo: jp0.Plan().FromTo, Source: source, Destination: destination, + TrailingDot: jp0.Plan().DstFileData.TrailingDot, } } diff --git a/ste/JobPartPlan.go b/ste/JobPartPlan.go index f4302d4eb..5b26d9efd 100644 --- a/ste/JobPartPlan.go +++ b/ste/JobPartPlan.go @@ -13,7 +13,7 @@ import ( // dataSchemaVersion defines the data schema version of JobPart order files supported by // current version of azcopy // To be Incremented every time when we release azcopy with changed dataSchema -const DataSchemaVersion common.Version = 18 +const DataSchemaVersion common.Version = 19 const ( CustomHeaderMaxBytes = 256 diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go index 39b42569e..d1daf551a 100755 --- a/ste/jobStatusManager.go +++ b/ste/jobStatusManager.go @@ -85,12 +85,7 @@ func (jm *jobMgr) SendXferDoneMsg(msg xferDoneMsg) { jm.Log(common.LogError, "Cannot send message on channel") } }() - if jm.jstm.xferDone != nil { - select { - case jm.jstm.xferDone <- msg: - case <-jm.jstm.statusMgrDone: // Nobody is listening anymore, let's back off. - } - } + jm.jstm.xferDone <- msg } func (jm *jobMgr) ListJobSummary() common.ListJobSummaryResponse { @@ -187,7 +182,6 @@ func (jm *jobMgr) handleStatusUpdateMessage() { case <-jstm.statusMgrDone: // If we time out, no biggie. This isn't world-ending, nor is it essential info. The other side stopped listening by now. } - // Reset the lists so that they don't keep accumulating and take up excessive memory // There is no need to keep sending the same items over and over again js.FailedTransfers = []common.TransferDetail{} diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index 19fc3f286..ebd800227 100755 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -580,7 +580,7 @@ func (jm *jobMgr) setFinalPartOrdered(partNum PartNumber, isFinalPart bool) { if partNum == 0 { // We can't complain because, when resuming a job, there are actually TWO calls made the ResurrectJob. // The effect is that all the parts are ordered... then all the parts are ordered _again_ (with new JobPartManagers replacing those from the first time) - // (The first resurrect is from GetJobFromTo and the second is from ResumeJobOrder) + // (The first resurrect is from GetJobDetails and the second is from ResumeJobOrder) // So we don't object if the _first_ part clears the flag. The assumption we make, by allowing this special case here, is that // the first part will be scheduled before any higher-numbered part. As long as that assumption is true, this is safe. // TODO: do we really need to to Resurrect the job twice? From faa616653975c4bc3a088f66375cf45796d3f3b3 Mon Sep 17 00:00:00 2001 From: dphulkar-msft <166800991+dphulkar-msft@users.noreply.github.com> Date: Mon, 23 Dec 2024 11:06:02 +0530 Subject: [PATCH 2/3] Updated version of golang.org/x/crypto to resolve CVE-2024-45337 (#2890) --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9d7f62ef4..7bcff4bb4 100644 --- a/go.mod +++ b/go.mod @@ -17,10 +17,10 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/spf13/cobra v1.8.1 github.com/wastore/keyctl v0.3.1 - golang.org/x/crypto v0.28.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/oauth2 v0.23.0 - golang.org/x/sync v0.8.0 - golang.org/x/sys v0.26.0 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 google.golang.org/api v0.202.0 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c ) @@ -85,7 +85,7 @@ require ( go.opentelemetry.io/otel/sdk v1.29.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect - golang.org/x/text v0.19.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.7.0 // indirect google.golang.org/genproto v0.0.0-20241015192408-796eee8c2d53 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241007155032-5fefd90f89a9 // indirect diff --git a/go.sum b/go.sum index b53d62184..ca492b209 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,8 @@ go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt3 go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -229,8 +229,8 @@ golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbht golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -239,14 +239,14 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From bc74a22ad18ce8fc3ba9e04859b6795dce6d3212 Mon Sep 17 00:00:00 2001 From: dphulkar-msft <166800991+dphulkar-msft@users.noreply.github.com> Date: Thu, 2 Jan 2025 11:47:46 +0530 Subject: [PATCH 3/3] fix CVE-2024-45338 (#2899) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7bcff4bb4..4548db24c 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.1 github.com/Azure/go-autorest/autorest/date v0.3.0 github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 - golang.org/x/net v0.30.0 + golang.org/x/net v0.33.0 ) require ( diff --git a/go.sum b/go.sum index ca492b209..5aa2536f0 100644 --- a/go.sum +++ b/go.sum @@ -221,8 +221,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=