From 8388a44374b16eeb85cf6954503aa993d80a33d7 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Fri, 13 Dec 2024 11:53:48 -0500 Subject: [PATCH 1/9] fixed panics, added description for file share full error --- cmd/make.go | 2 +- ste/jobStatusManager.go | 7 +++++-- ste/mgr-JobMgr.go | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cmd/make.go b/cmd/make.go index 13c900c20..ef2603497 100644 --- a/cmd/make.go +++ b/cmd/make.go @@ -208,6 +208,6 @@ func init() { }, } - makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "Specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota.") + makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "Specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota. If `ShareSizeLimitReached` error is hit, increase quota and call resume command.") rootCmd.AddCommand(makeCmd) } diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go index 39b42569e..fc352e664 100755 --- a/ste/jobStatusManager.go +++ b/ste/jobStatusManager.go @@ -44,6 +44,7 @@ type jobStatusManager struct { xferDone chan xferDoneMsg xferDoneDrained chan struct{} // To signal that all xferDone have been processed statusMgrDone chan struct{} // To signal statusManager has closed + isChanClosed bool } func (jm *jobMgr) waitToDrainXferDone() { @@ -139,10 +140,12 @@ func (jm *jobMgr) handleStatusUpdateMessage() { case msg, ok := <-jstm.xferDone: if !ok { // Channel is closed, all transfers have been attended. jstm.xferDone = nil - // close drainXferDone so that other components can know no further updates happen allXferDoneHandled = true - close(jstm.xferDoneDrained) + if jstm.xferDoneDrained != nil { + close(jstm.xferDoneDrained) + jstm.xferDoneDrained = nil + } continue } diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index d4895b73b..d4a5e90cf 100755 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -715,7 +715,10 @@ func (jm *jobMgr) reportJobPartDoneHandler() { (isCancelling && !haveFinalPart) // If we're cancelling, it's OK to try to exit early; the user already accepted this job cannot be resumed. Outgoing requests will fail anyway, so nothing can properly clean up. if shouldComplete { // Inform StatusManager that all parts are done. - close(jm.jstm.xferDone) + if jm.jstm.xferDone != nil { + close(jm.jstm.xferDone) + jm.jstm.xferDone = nil + } // Wait for all XferDone messages to be processed by statusManager. Front end // depends on JobStatus to determine if we've to quit job. Setting it here without From 826b35ffd5af74a2a37920cedd7274ab4e251bfb Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Fri, 13 Dec 2024 12:22:28 -0500 Subject: [PATCH 2/9] linting fix --- ste/jobStatusManager.go | 1 - 1 file changed, 1 deletion(-) diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go index fc352e664..4d4bb0f89 100755 --- a/ste/jobStatusManager.go +++ b/ste/jobStatusManager.go @@ -44,7 +44,6 @@ type jobStatusManager struct { xferDone chan xferDoneMsg xferDoneDrained chan struct{} // To signal that all xferDone have been processed statusMgrDone chan struct{} // To signal statusManager has closed - isChanClosed bool } func (jm *jobMgr) waitToDrainXferDone() { From a8f444bd24650526b7530b5c9384ed225fb717f0 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Tue, 17 Dec 2024 11:41:02 -0500 Subject: [PATCH 3/9] fixed timeout tests --- ste/jobStatusManager.go | 7 +++---- ste/mgr-JobMgr.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go index 545a5b5c3..a2465dd62 100755 --- a/ste/jobStatusManager.go +++ b/ste/jobStatusManager.go @@ -21,6 +21,7 @@ package ste import ( + "sync" "time" "github.com/Azure/azure-storage-azcopy/v10/common" @@ -44,6 +45,7 @@ type jobStatusManager struct { xferDone chan xferDoneMsg xferDoneDrained chan struct{} // To signal that all xferDone have been processed statusMgrDone chan struct{} // To signal statusManager has closed + once sync.Once // Ensure xferDoneDrained is closed once } func (jm *jobMgr) waitToDrainXferDone() { @@ -136,10 +138,7 @@ func (jm *jobMgr) handleStatusUpdateMessage() { jstm.xferDone = nil // close drainXferDone so that other components can know no further updates happen allXferDoneHandled = true - if jstm.xferDoneDrained != nil { - close(jstm.xferDoneDrained) - jstm.xferDoneDrained = nil - } + jstm.once.Do(func() { close(jstm.xferDoneDrained) }) continue } diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index d75715695..daa9e2807 100755 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -717,7 +717,7 @@ func (jm *jobMgr) reportJobPartDoneHandler() { // Inform StatusManager that all parts are done. if jm.jstm.xferDone != nil { close(jm.jstm.xferDone) - jm.jstm.xferDone = nil + // jm.jstm.xferDone = nil } // Wait for all XferDone messages to be processed by statusManager. Front end From 3a778db10fbf21f964451c93df7fb3d90b2d20cf Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Wed, 18 Dec 2024 14:57:29 -0500 Subject: [PATCH 4/9] proper error logging --- cmd/make.go | 2 +- ste/mgr-JobMgr.go | 1 - ste/mgr-JobPartTransferMgr.go | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/make.go b/cmd/make.go index ef2603497..13c900c20 100644 --- a/cmd/make.go +++ b/cmd/make.go @@ -208,6 +208,6 @@ func init() { }, } - makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "Specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota. If `ShareSizeLimitReached` error is hit, increase quota and call resume command.") + makeCmd.PersistentFlags().Uint32Var(&rawArgs.quota, "quota-gb", 0, "Specifies the maximum size of the share in gigabytes (GiB), 0 means you accept the file service's default quota.") rootCmd.AddCommand(makeCmd) } diff --git a/ste/mgr-JobMgr.go b/ste/mgr-JobMgr.go index daa9e2807..ebd800227 100755 --- a/ste/mgr-JobMgr.go +++ b/ste/mgr-JobMgr.go @@ -717,7 +717,6 @@ func (jm *jobMgr) reportJobPartDoneHandler() { // Inform StatusManager that all parts are done. if jm.jstm.xferDone != nil { close(jm.jstm.xferDone) - // jm.jstm.xferDone = nil } // Wait for all XferDone messages to be processed by statusManager. Front end diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index d0bb3e855..ccab7a16c 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -864,6 +864,10 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri common.GetLifecycleMgr().Info(fmt.Sprintf("Authentication failed, it is either not correct, or expired, or does not have the correct permission %s", err.Error())) } + if serviceCode == "ShareSizeLimitReached" { + common.GetLifecycleMgr().Error(fmt.Sprintf("Fix: Increase the file share quota and call Resume command")) + } + // and use the normal cancelling mechanism so that we can exit in a clean and controlled way jptm.jobPartMgr.(*jobPartMgr).jobMgr.CancelPauseJobOrder(common.EJobStatus.Cancelling()) // TODO: this results in the final job output line being: Final Job Status: Cancelled From b97d80a648793c2c0152f29847363ebc886c7014 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Fri, 20 Dec 2024 10:35:35 -0500 Subject: [PATCH 5/9] update logs --- ste/mgr-JobPartTransferMgr.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index ccab7a16c..dfe91adf1 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -865,7 +865,7 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri } if serviceCode == "ShareSizeLimitReached" { - common.GetLifecycleMgr().Error(fmt.Sprintf("Fix: Increase the file share quota and call Resume command")) + common.GetLifecycleMgr().Error("Increase the file share quota and call Resume command.") } // and use the normal cancelling mechanism so that we can exit in a clean and controlled way From 1bd5f652a8a4fa73f9601f7a645c3e5300e6c0e0 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Mon, 6 Jan 2025 12:34:18 -0500 Subject: [PATCH 6/9] added test --- e2etest/zt_newe2e_file_test.go | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index bf8b2d0c9..0bac9ff15 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -546,3 +546,53 @@ func (s *FileTestSuite) Scenario_CopyTrailingDotUnsafeDestination(svm *ScenarioV Body: body, }, false) } + +// Test empty files are not uploaded when File share quota is hit +func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationManager) { + quotaMB := int32(10) // 10 MB quota + shareName := "testsharewithquota" + shareResource := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{ + ContainerName: &shareName, + Properties: ContainerProperties{ + FileContainerProperties: FileContainerProperties{ + Quota: "aMB}, + }, + }) + + fileSizeMB := int64(6) + fileNames := []string{"file_1.txt", "file_2.txt", "file_3.txt"} + + // Create src obj mapping + srcObjs := make(ObjectResourceMappingFlat) + // Creat source files + for _, fileName := range fileNames { + body := NewRandomObjectContentContainer(fileSizeMB * common.MegaByte) + obj := ResourceDefinitionObject{ + ObjectName: &fileName, + Body: body, + } + srcObjs[fileName] = obj + } + + for _, fileName := range fileNames { + srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), srcObjs[fileName]) + dstObj := shareResource.GetObject(svm, fileName, common.EEntityType.File()) + + // Upload all files + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + } + + // Validate uploaded files are not empty + ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} From 9ef35d59f3bfe483958be66794424c8f20bcb728 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Thu, 16 Jan 2025 04:02:54 -0500 Subject: [PATCH 7/9] updated test --- common/fe-ste-models.go | 1 + e2etest/newe2e_assertions.go | 1 + e2etest/newe2e_resource_manager_mock.go | 3 + e2etest/newe2e_resource_managers_file.go | 13 +++ e2etest/newe2e_task_runazcopy.go | 29 ++++-- e2etest/newe2e_task_runazcopy_login_logout.go | 9 ++ e2etest/zt_newe2e_file_test.go | 93 ++++++++++++++----- e2etest/zt_newe2e_jobs_list_test.go | 2 +- go.mod | 11 +++ go.sum | 31 +++++++ 10 files changed, 161 insertions(+), 32 deletions(-) diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 09c466b6b..09050ae7f 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -1119,6 +1119,7 @@ const ( MaxNumberOfBlocksPerBlob = 50000 BlockSizeThreshold = 256 * 1024 * 1024 MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */ + GigaByte = 1024 * 1024 * 1024 MegaByte = 1024 * 1024 KiloByte = 1024 ) diff --git a/e2etest/newe2e_assertions.go b/e2etest/newe2e_assertions.go index bfde4c09c..c7e8d855e 100644 --- a/e2etest/newe2e_assertions.go +++ b/e2etest/newe2e_assertions.go @@ -41,6 +41,7 @@ func (n NoError) MinArgs() int { func (n NoError) Assert(items ...any) bool { for _, v := range items { if v != nil { + fmt.Sprintf("%v in %v is not Nil", v, items) return false } } diff --git a/e2etest/newe2e_resource_manager_mock.go b/e2etest/newe2e_resource_manager_mock.go index 21879ff1c..9138290f7 100644 --- a/e2etest/newe2e_resource_manager_mock.go +++ b/e2etest/newe2e_resource_manager_mock.go @@ -232,6 +232,9 @@ func (m *MockContainerResourceManager) GetProperties(a Asserter) ContainerProper return ContainerProperties{} } +func (m *MockContainerResourceManager) SetProperties(a Asserter, properties *ContainerResourceManager) { +} + func (m *MockContainerResourceManager) Delete(a Asserter) { // No-op it } diff --git a/e2etest/newe2e_resource_managers_file.go b/e2etest/newe2e_resource_managers_file.go index 891caae39..8534e513b 100644 --- a/e2etest/newe2e_resource_managers_file.go +++ b/e2etest/newe2e_resource_managers_file.go @@ -208,6 +208,19 @@ func (s *FileShareResourceManager) GetProperties(a Asserter) ContainerProperties } } +// SetProperties Sets the quota of a file share +func (s *FileShareResourceManager) SetProperties(a Asserter, properties *ContainerProperties) { + a.HelperMarker().Helper() + props := DerefOrZero(properties) + + _, err := s.internalClient.SetProperties(ctx, &share.SetPropertiesOptions{ + Quota: props.FileContainerProperties.Quota}) + + a.NoError("set share properties", err) + + return +} + func (s *FileShareResourceManager) Create(a Asserter, props ContainerProperties) { a.HelperMarker().Helper() s.CreateWithOptions(a, &FileShareCreateOptions{ diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 40e602f45..c4ede72cc 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -53,13 +53,14 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" - AzCopyVerbLogin AzCopyVerb = "login" - AzCopyVerbLogout AzCopyVerb = "logout" - AzCopyVerbJobs AzCopyVerb = "jobs" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbLogin AzCopyVerb = "login" + AzCopyVerbLogout AzCopyVerb = "logout" + AzCopyVerbJobs AzCopyVerb = "jobs" + AzCopyVerbJobsResume AzCopyVerb = "resume" ) type AzCopyTarget struct { @@ -321,6 +322,12 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az out = &AzCopyParsedListStdout{} case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "list": out = &AzCopyParsedJobsListStdout{} + case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "resume": + out = &AzCopyParsedCopySyncRemoveStdout{ // Resume command treated the same as copy/sync/remove + JobPlanFolder: *commandSpec.Environment.JobPlanLocation, + LogFolder: *commandSpec.Environment.LogLocation, + } + default: // We don't know how to parse this. out = &AzCopyRawStdout{} } @@ -346,11 +353,19 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az } err = command.Wait() + a.Assert("wait for finalize", common.Iff[Assertion](commandSpec.ShouldFail, Not{IsNil{}}, IsNil{}), err) a.Assert("expected exit code", common.Iff[Assertion](commandSpec.ShouldFail, Not{Equal{}}, Equal{}), 0, command.ProcessState.ExitCode()) + if err != nil { + a.Log("ShouldFail: Command execution failed with error: %v", err) + } else { + a.Log("ShouldFail: Command executed successfully") + } + a.Log("ShouldFail: Expected exit code: %d, Actual exit code: %d", 0, command.ProcessState.ExitCode()) + // validate log file retention for jobs clean command before the job logs are cleaned up and uploaded if !a.Failed() && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "clean" { ValidateLogFileRetention(a, *commandSpec.Environment.LogLocation, 1) diff --git a/e2etest/newe2e_task_runazcopy_login_logout.go b/e2etest/newe2e_task_runazcopy_login_logout.go index b66f59db6..d0b087fa9 100644 --- a/e2etest/newe2e_task_runazcopy_login_logout.go +++ b/e2etest/newe2e_task_runazcopy_login_logout.go @@ -69,6 +69,15 @@ func RunAzCopyLoginLogout(a Asserter, verb AzCopyVerb) AzCopyStdout { } err = command.Wait() + + err = command.Wait() + if err != nil { + a.Log("Login/ Logout: Command execution failed with error: %v", err) + } else { + a.Log("Login/Logout: Command executed successfully") + } + a.Log("Login/out: Expected exit code: %d, Actual exit code: %d", 0, command.ProcessState.ExitCode()) + a.Assert("wait for finalize", IsNil{}, err) a.Assert("expected exit code", Equal{}, 0, command.ProcessState.ExitCode()) diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 0bac9ff15..9b2fd01e5 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -547,26 +547,30 @@ func (s *FileTestSuite) Scenario_CopyTrailingDotUnsafeDestination(svm *ScenarioV }, false) } -// Test empty files are not uploaded when File share quota is hit +// Test: +// - correct number of non-empty files are uploaded when file share quota is hit +// - transfers complete successfully with resume command after increasing quota func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationManager) { - quotaMB := int32(10) // 10 MB quota - shareName := "testsharewithquota" + quotaGB := int32(1) // 1 GB quota shareResource := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{ - ContainerName: &shareName, + //ContainerName: &shareName, Properties: ContainerProperties{ FileContainerProperties: FileContainerProperties{ - Quota: "aMB}, + Quota: "aGB}, }, }) + svm.Assert("Quota is 1GB", Equal{Deep: true}, + DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), int32(1)) - fileSizeMB := int64(6) - fileNames := []string{"file_1.txt", "file_2.txt", "file_3.txt"} + fileSizeGB := int64(1) // Fits one file + fileNames := []string{"file_1.txt", "file_2.txt"} // Create src obj mapping srcObjs := make(ObjectResourceMappingFlat) - // Creat source files + + // Create source files for _, fileName := range fileNames { - body := NewRandomObjectContentContainer(fileSizeMB * common.MegaByte) + body := NewRandomObjectContentContainer(fileSizeGB * common.GigaByte) obj := ResourceDefinitionObject{ ObjectName: &fileName, Body: body, @@ -574,25 +578,66 @@ func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationMana srcObjs[fileName] = obj } - for _, fileName := range fileNames { - srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), srcObjs[fileName]) - dstObj := shareResource.GetObject(svm, fileName, common.EEntityType.File()) + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), + ResourceDefinitionContainer{Objects: srcObjs}) - // Upload all files - RunAzCopy(svm, AzCopyCommand{ - Verb: AzCopyVerbCopy, - Targets: []ResourceManager{srcObj, dstObj}, - Flags: CopyFlags{ - CopySyncCommonFlags: CopySyncCommonFlags{ - Recursive: pointerTo(true), - BlockSizeMB: pointerTo(4.0), - }, + stdOut, _ := RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, shareResource}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), }, - }) - } + }, + ShouldFail: true, + }) - // Validate uploaded files are not empty + // Error catchers for full file share + ValidateContainsError(svm, stdOut, []string{"Increase the file share quota and call Resume command."}) + ValidateMessageOutput(svm, stdOut, "Increase the file share quota and call Resume command.") + + // Validate single uploaded file is not empty ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{ Objects: srcObjs, }, true) + + fileMap := shareResource.ListObjects(svm, "", true) + svm.Assert("Only one file should upload within the quota", Equal{}, len(fileMap)-1, 1) + + // Increase quota to fit all files + newQuota := int32(2) + if resourceManager, ok := shareResource.(*FileShareResourceManager); ok { + resourceManager.SetProperties(svm, &ContainerProperties{ + FileContainerProperties: FileContainerProperties{ + Quota: &newQuota}}) + } + + // Validate correctly SetProperties updates quota. Prevent nil deref in dry runs + svm.Assert("Quota should be updated", Equal{}, + DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), + newQuota) + + var jobId string + if parsedOut, ok := stdOut.(*AzCopyParsedCopySyncRemoveStdout); ok { + if parsedOut.InitMsg.JobID != "" { + jobId = parsedOut.InitMsg.JobID + } + } else { + // Will enter during dry runs + fmt.Println("failed to cast to AzCopyParsedCopySyncRemoveStdout") + } + + RunAzCopy(svm, AzCopyCommand{ // Resume job with same source and dest targets + Verb: AzCopyVerbJobs, + PositionalArgs: []string{"resume", jobId}, + ShouldFail: false, + }) + + // Validate all files are uploaded + fileMapResume := shareResource.ListObjects(svm, "", true) + svm.Assert("All files should be successfully uploaded after quota increase", + Equal{}, len(fileMapResume), 2) + + // Validate uploaded files are not empty + ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{}, true) } diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index d2fe65e41..be55d565e 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -16,5 +16,5 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { Stdout: &AzCopyParsedJobsListStdout{}, Flags: ListFlags{}, }) - ValidateJobsListOutput(svm, jobsListOutput, 0) + ValidateJobsListOutput(svm, jobsListOutput, -1) } diff --git a/go.mod b/go.mod index 4548db24c..46b0a1618 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,9 @@ require ( github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/google/gops v0.3.28 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect @@ -70,12 +72,20 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect @@ -94,6 +104,7 @@ require ( google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/goversion v1.2.0 // indirect ) go 1.23.1 diff --git a/go.sum b/go.sum index 5aa2536f0..f37e627aa 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -118,8 +120,12 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark= +github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= @@ -146,6 +152,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M= github.com/mattn/go-ieproxy v0.0.12/go.mod h1:Vn+N61199DAnVeTgaF8eoB9PvLO8P3OBnG95ENh7B7c= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= @@ -163,6 +171,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= @@ -171,6 +181,11 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -180,13 +195,23 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/wastore/keyctl v0.3.1 h1:wMkYW9y9jGbQ1ARBLGLwnDdbgrkbuSeuIQeHy+BZOU0= github.com/wastore/keyctl v0.3.1/go.mod h1:1359RfMRDlblBSa2vaPC+kkmIxxt+rgl/FxLG38h9xM= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ= @@ -234,11 +259,15 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -296,3 +325,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w= +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= From 173febd8ad8c26bbc9106f23bbf1545f7cdb9c82 Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Thu, 16 Jan 2025 04:09:32 -0500 Subject: [PATCH 8/9] clean ups --- e2etest/newe2e_assertions.go | 1 - e2etest/newe2e_task_runazcopy.go | 15 +++++++-------- e2etest/zt_newe2e_file_test.go | 1 - e2etest/zt_newe2e_jobs_list_test.go | 2 +- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/e2etest/newe2e_assertions.go b/e2etest/newe2e_assertions.go index c7e8d855e..bfde4c09c 100644 --- a/e2etest/newe2e_assertions.go +++ b/e2etest/newe2e_assertions.go @@ -41,7 +41,6 @@ func (n NoError) MinArgs() int { func (n NoError) Assert(items ...any) bool { for _, v := range items { if v != nil { - fmt.Sprintf("%v in %v is not Nil", v, items) return false } } diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index c4ede72cc..87bfd0f02 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -53,14 +53,13 @@ var _ AzCopyStdout = &AzCopyRawStdout{} type AzCopyVerb string const ( // initially supporting a limited set of verbs - AzCopyVerbCopy AzCopyVerb = "copy" - AzCopyVerbSync AzCopyVerb = "sync" - AzCopyVerbRemove AzCopyVerb = "remove" - AzCopyVerbList AzCopyVerb = "list" - AzCopyVerbLogin AzCopyVerb = "login" - AzCopyVerbLogout AzCopyVerb = "logout" - AzCopyVerbJobs AzCopyVerb = "jobs" - AzCopyVerbJobsResume AzCopyVerb = "resume" + AzCopyVerbCopy AzCopyVerb = "copy" + AzCopyVerbSync AzCopyVerb = "sync" + AzCopyVerbRemove AzCopyVerb = "remove" + AzCopyVerbList AzCopyVerb = "list" + AzCopyVerbLogin AzCopyVerb = "login" + AzCopyVerbLogout AzCopyVerb = "logout" + AzCopyVerbJobs AzCopyVerb = "jobs" ) type AzCopyTarget struct { diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 9b2fd01e5..998ef6d4e 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -594,7 +594,6 @@ func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationMana // Error catchers for full file share ValidateContainsError(svm, stdOut, []string{"Increase the file share quota and call Resume command."}) - ValidateMessageOutput(svm, stdOut, "Increase the file share quota and call Resume command.") // Validate single uploaded file is not empty ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{ diff --git a/e2etest/zt_newe2e_jobs_list_test.go b/e2etest/zt_newe2e_jobs_list_test.go index be55d565e..d2fe65e41 100644 --- a/e2etest/zt_newe2e_jobs_list_test.go +++ b/e2etest/zt_newe2e_jobs_list_test.go @@ -16,5 +16,5 @@ func (s *JobsListSuite) Scenario_JobsListBasic(svm *ScenarioVariationManager) { Stdout: &AzCopyParsedJobsListStdout{}, Flags: ListFlags{}, }) - ValidateJobsListOutput(svm, jobsListOutput, -1) + ValidateJobsListOutput(svm, jobsListOutput, 0) } From 3e611de59092251f2e5533fafce266d7927e7bda Mon Sep 17 00:00:00 2001 From: Wendi Onwuakpa Date: Thu, 16 Jan 2025 21:00:23 -0500 Subject: [PATCH 9/9] removed resuming functionality from test --- e2etest/newe2e_task_runazcopy.go | 7 --- e2etest/newe2e_task_runazcopy_login_logout.go | 8 ---- e2etest/newe2e_task_validation.go | 2 +- e2etest/zt_newe2e_file_test.go | 43 +++---------------- 4 files changed, 8 insertions(+), 52 deletions(-) diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 87bfd0f02..f20b21fb4 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -358,13 +358,6 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az common.Iff[Assertion](commandSpec.ShouldFail, Not{Equal{}}, Equal{}), 0, command.ProcessState.ExitCode()) - if err != nil { - a.Log("ShouldFail: Command execution failed with error: %v", err) - } else { - a.Log("ShouldFail: Command executed successfully") - } - a.Log("ShouldFail: Expected exit code: %d, Actual exit code: %d", 0, command.ProcessState.ExitCode()) - // validate log file retention for jobs clean command before the job logs are cleaned up and uploaded if !a.Failed() && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "clean" { ValidateLogFileRetention(a, *commandSpec.Environment.LogLocation, 1) diff --git a/e2etest/newe2e_task_runazcopy_login_logout.go b/e2etest/newe2e_task_runazcopy_login_logout.go index d0b087fa9..aa319b7ba 100644 --- a/e2etest/newe2e_task_runazcopy_login_logout.go +++ b/e2etest/newe2e_task_runazcopy_login_logout.go @@ -70,14 +70,6 @@ func RunAzCopyLoginLogout(a Asserter, verb AzCopyVerb) AzCopyStdout { err = command.Wait() - err = command.Wait() - if err != nil { - a.Log("Login/ Logout: Command execution failed with error: %v", err) - } else { - a.Log("Login/Logout: Command executed successfully") - } - a.Log("Login/out: Expected exit code: %d, Actual exit code: %d", 0, command.ProcessState.ExitCode()) - a.Assert("wait for finalize", IsNil{}, err) a.Assert("expected exit code", Equal{}, 0, command.ProcessState.ExitCode()) diff --git a/e2etest/newe2e_task_validation.go b/e2etest/newe2e_task_validation.go index 70159c2f4..cd164ece0 100644 --- a/e2etest/newe2e_task_validation.go +++ b/e2etest/newe2e_task_validation.go @@ -209,7 +209,7 @@ func ValidateContainsError(a Asserter, stdout AzCopyStdout, errorMsg []string) { } } fmt.Println(stdout.String()) - a.Error("expected error message not found in azcopy output") + a.Error(fmt.Sprintf("expected error message %v not found in azcopy output", errorMsg)) } func checkMultipleErrors(errorMsg []string, line string) bool { diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 998ef6d4e..5e8db457c 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -549,11 +549,10 @@ func (s *FileTestSuite) Scenario_CopyTrailingDotUnsafeDestination(svm *ScenarioV // Test: // - correct number of non-empty files are uploaded when file share quota is hit -// - transfers complete successfully with resume command after increasing quota +// - TODO 10.29.0 release: transfers complete successfully with resume command func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationManager) { quotaGB := int32(1) // 1 GB quota shareResource := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{ - //ContainerName: &shareName, Properties: ContainerProperties{ FileContainerProperties: FileContainerProperties{ Quota: "aGB}, @@ -562,25 +561,25 @@ func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationMana svm.Assert("Quota is 1GB", Equal{Deep: true}, DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), int32(1)) - fileSizeGB := int64(1) // Fits one file fileNames := []string{"file_1.txt", "file_2.txt"} // Create src obj mapping srcObjs := make(ObjectResourceMappingFlat) // Create source files + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), + ResourceDefinitionContainer{Objects: srcObjs}) for _, fileName := range fileNames { - body := NewRandomObjectContentContainer(fileSizeGB * common.GigaByte) + body := NewRandomObjectContentContainer(int64(1) * common.GigaByte) obj := ResourceDefinitionObject{ ObjectName: &fileName, Body: body, + Size: "1.00 GiB", } srcObjs[fileName] = obj + CreateResource[ObjectResourceManager](svm, srcContainer, obj) } - srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), - ResourceDefinitionContainer{Objects: srcObjs}) - stdOut, _ := RunAzCopy(svm, AzCopyCommand{ Verb: AzCopyVerbCopy, Targets: []ResourceManager{srcContainer, shareResource}, @@ -595,13 +594,8 @@ func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationMana // Error catchers for full file share ValidateContainsError(svm, stdOut, []string{"Increase the file share quota and call Resume command."}) - // Validate single uploaded file is not empty - ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{ - Objects: srcObjs, - }, true) - fileMap := shareResource.ListObjects(svm, "", true) - svm.Assert("Only one file should upload within the quota", Equal{}, len(fileMap)-1, 1) + svm.Assert("One file should be uploaded within the quota", Equal{}, len(fileMap)-1, 1) // -1 to Account for root dir in fileMap // Increase quota to fit all files newQuota := int32(2) @@ -616,27 +610,4 @@ func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationMana DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), newQuota) - var jobId string - if parsedOut, ok := stdOut.(*AzCopyParsedCopySyncRemoveStdout); ok { - if parsedOut.InitMsg.JobID != "" { - jobId = parsedOut.InitMsg.JobID - } - } else { - // Will enter during dry runs - fmt.Println("failed to cast to AzCopyParsedCopySyncRemoveStdout") - } - - RunAzCopy(svm, AzCopyCommand{ // Resume job with same source and dest targets - Verb: AzCopyVerbJobs, - PositionalArgs: []string{"resume", jobId}, - ShouldFail: false, - }) - - // Validate all files are uploaded - fileMapResume := shareResource.ListObjects(svm, "", true) - svm.Assert("All files should be successfully uploaded after quota increase", - Equal{}, len(fileMapResume), 2) - - // Validate uploaded files are not empty - ValidateResource[ContainerResourceManager](svm, shareResource, ResourceDefinitionContainer{}, true) }