diff --git a/common/fe-ste-models.go b/common/fe-ste-models.go index 09c466b6b..09050ae7f 100644 --- a/common/fe-ste-models.go +++ b/common/fe-ste-models.go @@ -1119,6 +1119,7 @@ const ( MaxNumberOfBlocksPerBlob = 50000 BlockSizeThreshold = 256 * 1024 * 1024 MinParallelChunkCountThreshold = 4 /* minimum number of chunks in parallel for AzCopy to be performant. */ + GigaByte = 1024 * 1024 * 1024 MegaByte = 1024 * 1024 KiloByte = 1024 ) diff --git a/e2etest/newe2e_resource_manager_mock.go b/e2etest/newe2e_resource_manager_mock.go index 21879ff1c..9138290f7 100644 --- a/e2etest/newe2e_resource_manager_mock.go +++ b/e2etest/newe2e_resource_manager_mock.go @@ -232,6 +232,9 @@ func (m *MockContainerResourceManager) GetProperties(a Asserter) ContainerProper return ContainerProperties{} } +func (m *MockContainerResourceManager) SetProperties(a Asserter, properties *ContainerResourceManager) { +} + func (m *MockContainerResourceManager) Delete(a Asserter) { // No-op it } diff --git a/e2etest/newe2e_resource_managers_file.go b/e2etest/newe2e_resource_managers_file.go index 891caae39..8534e513b 100644 --- a/e2etest/newe2e_resource_managers_file.go +++ b/e2etest/newe2e_resource_managers_file.go @@ -208,6 +208,19 @@ func (s *FileShareResourceManager) GetProperties(a Asserter) ContainerProperties } } +// SetProperties Sets the quota of a file share +func (s *FileShareResourceManager) SetProperties(a Asserter, properties *ContainerProperties) { + a.HelperMarker().Helper() + props := DerefOrZero(properties) + + _, err := s.internalClient.SetProperties(ctx, &share.SetPropertiesOptions{ + Quota: props.FileContainerProperties.Quota}) + + a.NoError("set share properties", err) + + return +} + func (s *FileShareResourceManager) Create(a Asserter, props ContainerProperties) { a.HelperMarker().Helper() s.CreateWithOptions(a, &FileShareCreateOptions{ diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 40e602f45..f20b21fb4 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -321,6 +321,12 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az out = &AzCopyParsedListStdout{} case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "list": out = &AzCopyParsedJobsListStdout{} + case commandSpec.Verb == AzCopyVerbJobs && len(commandSpec.PositionalArgs) != 0 && commandSpec.PositionalArgs[0] == "resume": + out = &AzCopyParsedCopySyncRemoveStdout{ // Resume command treated the same as copy/sync/remove + JobPlanFolder: *commandSpec.Environment.JobPlanLocation, + LogFolder: *commandSpec.Environment.LogLocation, + } + default: // We don't know how to parse this. out = &AzCopyRawStdout{} } @@ -346,6 +352,7 @@ func RunAzCopy(a ScenarioAsserter, commandSpec AzCopyCommand) (AzCopyStdout, *Az } err = command.Wait() + a.Assert("wait for finalize", common.Iff[Assertion](commandSpec.ShouldFail, Not{IsNil{}}, IsNil{}), err) a.Assert("expected exit code", common.Iff[Assertion](commandSpec.ShouldFail, Not{Equal{}}, Equal{}), diff --git a/e2etest/newe2e_task_runazcopy_login_logout.go b/e2etest/newe2e_task_runazcopy_login_logout.go index b66f59db6..aa319b7ba 100644 --- a/e2etest/newe2e_task_runazcopy_login_logout.go +++ b/e2etest/newe2e_task_runazcopy_login_logout.go @@ -69,6 +69,7 @@ func RunAzCopyLoginLogout(a Asserter, verb AzCopyVerb) AzCopyStdout { } err = command.Wait() + a.Assert("wait for finalize", IsNil{}, err) a.Assert("expected exit code", Equal{}, 0, command.ProcessState.ExitCode()) diff --git a/e2etest/newe2e_task_validation.go b/e2etest/newe2e_task_validation.go index 18f88d1fd..a19b6f2e9 100644 --- a/e2etest/newe2e_task_validation.go +++ b/e2etest/newe2e_task_validation.go @@ -214,7 +214,7 @@ func ValidateContainsError(a Asserter, stdout AzCopyStdout, errorMsg []string) { } } fmt.Println(stdout.String()) - a.Error("expected error message not found in azcopy output") + a.Error(fmt.Sprintf("expected error message %v not found in azcopy output", errorMsg)) } func checkMultipleErrors(errorMsg []string, line string) bool { diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index bf8b2d0c9..5e8db457c 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -546,3 +546,68 @@ func (s *FileTestSuite) Scenario_CopyTrailingDotUnsafeDestination(svm *ScenarioV Body: body, }, false) } + +// Test: +// - correct number of non-empty files are uploaded when file share quota is hit +// - TODO 10.29.0 release: transfers complete successfully with resume command +func (s *FileTestSuite) Scenario_UploadFilesWithQuota(svm *ScenarioVariationManager) { + quotaGB := int32(1) // 1 GB quota + shareResource := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{ + Properties: ContainerProperties{ + FileContainerProperties: FileContainerProperties{ + Quota: "aGB}, + }, + }) + svm.Assert("Quota is 1GB", Equal{Deep: true}, + DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), int32(1)) + + fileNames := []string{"file_1.txt", "file_2.txt"} + + // Create src obj mapping + srcObjs := make(ObjectResourceMappingFlat) + + // Create source files + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), + ResourceDefinitionContainer{Objects: srcObjs}) + for _, fileName := range fileNames { + body := NewRandomObjectContentContainer(int64(1) * common.GigaByte) + obj := ResourceDefinitionObject{ + ObjectName: &fileName, + Body: body, + Size: "1.00 GiB", + } + srcObjs[fileName] = obj + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + } + + stdOut, _ := RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcContainer, shareResource}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + ShouldFail: true, + }) + + // Error catchers for full file share + ValidateContainsError(svm, stdOut, []string{"Increase the file share quota and call Resume command."}) + + fileMap := shareResource.ListObjects(svm, "", true) + svm.Assert("One file should be uploaded within the quota", Equal{}, len(fileMap)-1, 1) // -1 to Account for root dir in fileMap + + // Increase quota to fit all files + newQuota := int32(2) + if resourceManager, ok := shareResource.(*FileShareResourceManager); ok { + resourceManager.SetProperties(svm, &ContainerProperties{ + FileContainerProperties: FileContainerProperties{ + Quota: &newQuota}}) + } + + // Validate correctly SetProperties updates quota. Prevent nil deref in dry runs + svm.Assert("Quota should be updated", Equal{}, + DerefOrZero(shareResource.GetProperties(svm).FileContainerProperties.Quota), + newQuota) + +} diff --git a/go.mod b/go.mod index ddccceb14..3c77af23d 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,9 @@ require ( github.com/go-ini/ini v1.67.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/google/gops v0.3.28 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect @@ -70,12 +72,20 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/xlab/treeprint v1.2.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect @@ -94,6 +104,7 @@ require ( google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect google.golang.org/protobuf v1.35.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + rsc.io/goversion v1.2.0 // indirect ) go 1.22.7 diff --git a/go.sum b/go.sum index 5aa2536f0..f37e627aa 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -118,8 +120,12 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark= +github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= @@ -146,6 +152,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M= github.com/mattn/go-ieproxy v0.0.12/go.mod h1:Vn+N61199DAnVeTgaF8eoB9PvLO8P3OBnG95ENh7B7c= github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= @@ -163,6 +171,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgm github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= @@ -171,6 +181,11 @@ github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= +github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -180,13 +195,23 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/wastore/keyctl v0.3.1 h1:wMkYW9y9jGbQ1ARBLGLwnDdbgrkbuSeuIQeHy+BZOU0= github.com/wastore/keyctl v0.3.1/go.mod h1:1359RfMRDlblBSa2vaPC+kkmIxxt+rgl/FxLG38h9xM= +github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ= +github.com/xlab/treeprint v1.2.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/detectors/gcp v1.29.0 h1:TiaiXB4DpGD3sdzNlYQxruQngn5Apwzi1X0DRhuGvDQ= @@ -234,11 +259,15 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -296,3 +325,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +rsc.io/goversion v1.2.0 h1:SPn+NLTiAG7w30IRK/DKp1BjvpWabYgxlLp/+kx5J8w= +rsc.io/goversion v1.2.0/go.mod h1:Eih9y/uIBS3ulggl7KNJ09xGSLcuNaLgmvvqa07sgfo= diff --git a/ste/jobStatusManager.go b/ste/jobStatusManager.go index d1daf551a..a2465dd62 100755 --- a/ste/jobStatusManager.go +++ b/ste/jobStatusManager.go @@ -21,6 +21,7 @@ package ste import ( + "sync" "time" "github.com/Azure/azure-storage-azcopy/v10/common" @@ -44,6 +45,7 @@ type jobStatusManager struct { xferDone chan xferDoneMsg xferDoneDrained chan struct{} // To signal that all xferDone have been processed statusMgrDone chan struct{} // To signal statusManager has closed + once sync.Once // Ensure xferDoneDrained is closed once } func (jm *jobMgr) waitToDrainXferDone() { @@ -134,10 +136,9 @@ func (jm *jobMgr) handleStatusUpdateMessage() { case msg, ok := <-jstm.xferDone: if !ok { // Channel is closed, all transfers have been attended. jstm.xferDone = nil - // close drainXferDone so that other components can know no further updates happen allXferDoneHandled = true - close(jstm.xferDoneDrained) + jstm.once.Do(func() { close(jstm.xferDoneDrained) }) continue } diff --git a/ste/mgr-JobPartTransferMgr.go b/ste/mgr-JobPartTransferMgr.go index d0bb3e855..dfe91adf1 100644 --- a/ste/mgr-JobPartTransferMgr.go +++ b/ste/mgr-JobPartTransferMgr.go @@ -864,6 +864,10 @@ func (jptm *jobPartTransferMgr) failActiveTransfer(typ transferErrorCode, descri common.GetLifecycleMgr().Info(fmt.Sprintf("Authentication failed, it is either not correct, or expired, or does not have the correct permission %s", err.Error())) } + if serviceCode == "ShareSizeLimitReached" { + common.GetLifecycleMgr().Error("Increase the file share quota and call Resume command.") + } + // and use the normal cancelling mechanism so that we can exit in a clean and controlled way jptm.jobPartMgr.(*jobPartMgr).jobMgr.CancelPauseJobOrder(common.EJobStatus.Cancelling()) // TODO: this results in the final job output line being: Final Job Status: Cancelled