From 0254a7e6be86129779ce4d6ed67bc067046fcafb Mon Sep 17 00:00:00 2001 From: Gauri Lamunion <51212198+gapra-msft@users.noreply.github.com> Date: Thu, 25 Jul 2024 03:06:10 -0700 Subject: [PATCH] Migrated some blob tests to new e2e test framework (#2736) * Migrated smoke tests for auto detect blob type --- e2etest/newe2e_account_registry.go | 7 + e2etest/newe2e_config.go | 4 + e2etest/newe2e_resource_manager_azstorage.go | 15 + e2etest/newe2e_resource_manager_interface.go | 5 +- e2etest/newe2e_resource_managers_blob.go | 6 + e2etest/newe2e_resource_managers_blobfs.go | 3 + e2etest/newe2e_resource_managers_file.go | 7 + e2etest/newe2e_resource_managers_local.go | 18 +- e2etest/newe2e_task_runazcopy.go | 9 +- e2etest/newe2e_task_runazcopy_parameters.go | 1 + ...sks_test.go => zt_aamanaged_disks_test.go} | 0 e2etest/zt_newe2e_blob_test.go | 206 +++++++++++++ e2etest/zt_newe2e_file_test.go | 260 ++++++++++++++++ e2etest/zt_newe2e_remove_test.go | 35 +++ e2etest/zt_newe2e_s2s_test.go | 285 ++++++++++++++++++ ste/sender-azureFile.go | 5 +- testSuite/scripts/run.py | 6 - testSuite/scripts/test_azcopy_operations.py | 53 ---- testSuite/scripts/test_blob_download.py | 151 ---------- testSuite/scripts/test_file_download.py | 151 ---------- testSuite/scripts/test_file_upload.py | 23 -- .../scripts/test_service_to_service_copy.py | 135 +-------- testSuite/scripts/test_upload_block_blob.py | 73 ----- testSuite/scripts/test_upload_page_blob.py | 149 --------- 24 files changed, 864 insertions(+), 743 deletions(-) rename e2etest/{zt_amanaged_disks_test.go => zt_aamanaged_disks_test.go} (100%) create mode 100644 e2etest/zt_newe2e_blob_test.go delete mode 100644 testSuite/scripts/test_azcopy_operations.py delete mode 100644 testSuite/scripts/test_file_download.py delete mode 100644 testSuite/scripts/test_file_upload.py diff --git a/e2etest/newe2e_account_registry.go b/e2etest/newe2e_account_registry.go index 5f1a11e9f..1ba4a8f9d 100644 --- a/e2etest/newe2e_account_registry.go +++ b/e2etest/newe2e_account_registry.go @@ -124,6 +124,7 @@ func DeleteAccount(a Asserter, arm AccountResourceManager) { const ( PrimaryStandardAcct string = "PrimaryStandard" PrimaryHNSAcct string = "PrimaryHNS" + PremiumPageBlobAcct string = "PremiumPageBlob" ) func AccountRegistryInitHook(a Asserter) { @@ -140,10 +141,16 @@ func AccountRegistryInitHook(a Asserter) { accountKey: acctInfo.HNS.AccountKey, accountType: EAccountType.HierarchicalNamespaceEnabled(), } + AccountRegistry[PremiumPageBlobAcct] = &AzureAccountResourceManager{ + accountName: acctInfo.PremiumPage.AccountName, + accountKey: acctInfo.PremiumPage.AccountKey, + accountType: EAccountType.PremiumPageBlobs(), + } } else { // Create standard accounts AccountRegistry[PrimaryStandardAcct] = CreateAccount(a, EAccountType.Standard(), nil) AccountRegistry[PrimaryHNSAcct] = CreateAccount(a, EAccountType.HierarchicalNamespaceEnabled(), nil) + AccountRegistry[PremiumPageBlobAcct] = CreateAccount(a, EAccountType.PremiumPageBlobs(), nil) } } diff --git a/e2etest/newe2e_config.go b/e2etest/newe2e_config.go index c59ee6333..f1a0615cb 100644 --- a/e2etest/newe2e_config.go +++ b/e2etest/newe2e_config.go @@ -65,6 +65,10 @@ type NewE2EConfig struct { AccountName string `env:"NEW_E2E_HNS_ACCOUNT_NAME,required"` AccountKey string `env:"NEW_E2E_HNS_ACCOUNT_KEY,required"` } `env:",required"` + PremiumPage struct { + AccountName string `env:"NEW_E2E_PREMIUM_PAGE_ACCOUNT_NAME,required"` + AccountKey string `env:"NEW_E2E_PREMIUM_PAGE_ACCOUNT_KEY,required"` + } `env:",required"` } `env:",required,minimum_required=1"` } `env:",required,mutually_exclusive"` diff --git a/e2etest/newe2e_resource_manager_azstorage.go b/e2etest/newe2e_resource_manager_azstorage.go index 57b46690e..539d2f7c1 100644 --- a/e2etest/newe2e_resource_manager_azstorage.go +++ b/e2etest/newe2e_resource_manager_azstorage.go @@ -11,8 +11,23 @@ import ( filesas "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/sas" fileservice "github.com/Azure/azure-sdk-for-go/sdk/storage/azfile/service" "github.com/Azure/azure-storage-azcopy/v10/common" + "strings" ) +func addWildCard(uri string, optList ...GetURIOptions) string { + wildcard := FirstOrZero(optList).Wildcard + if wildcard == "" { + return uri + } + if strings.Contains(uri, "?") { + uri = strings.Replace(uri, "?", wildcard+"?", 1) + } else { + uri += wildcard + } + + return uri +} + type AzureAccountResourceManager struct { accountName string accountKey string diff --git a/e2etest/newe2e_resource_manager_interface.go b/e2etest/newe2e_resource_manager_interface.go index e0fa9bf12..7b54159b7 100644 --- a/e2etest/newe2e_resource_manager_interface.go +++ b/e2etest/newe2e_resource_manager_interface.go @@ -15,6 +15,8 @@ import ( type GetURIOptions struct { RemoteOpts RemoteURIOpts AzureOpts AzureURIOpts + // The wildcard string to append to the end of a resource URI. + Wildcard string } type RemoteURIOpts struct { @@ -76,7 +78,7 @@ func TryApplySpecificAuthType(rm ResourceManager, cred ExplicitCredentialTypes, return rrm.WithSpecificAuthType(cred, a, opts...) } - return CreateAzCopyTarget(rm, EExplicitCredentialType.None(), a) + return CreateAzCopyTarget(rm, EExplicitCredentialType.None(), a, opts...) } // ExplicitCredentialTypes defines a more explicit enum for credential types as AzCopy's internal definition is very loose (e.g. Anonymous can be public or SAS); accepts the URI as-is. @@ -301,6 +303,7 @@ type FileProperties struct { FileCreationTime *time.Time FileLastWriteTime *time.Time FilePermissions *string + LastModifiedTime *time.Time } func (f FileProperties) hasCustomTimes() bool { diff --git a/e2etest/newe2e_resource_managers_blob.go b/e2etest/newe2e_resource_managers_blob.go index 769bcc5aa..9380a7785 100644 --- a/e2etest/newe2e_resource_managers_blob.go +++ b/e2etest/newe2e_resource_managers_blob.go @@ -139,6 +139,7 @@ func (b *BlobServiceResourceManager) ListContainers(a Asserter) []string { func (b *BlobServiceResourceManager) URI(opts ...GetURIOptions) string { base := blobStripSAS(b.internalClient.URL()) base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -348,6 +349,7 @@ func (b *BlobContainerResourceManager) Level() cmd.LocationLevel { func (b *BlobContainerResourceManager) URI(opts ...GetURIOptions) string { base := blobStripSAS(b.internalClient.URL()) base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -658,6 +660,9 @@ func (b *BlobObjectResourceManager) GetPropertiesWithOptions(a Asserter, options Type: resp.BlobType, Tags: func() map[string]string { out := make(map[string]string) + if b.internalAccount.AccountType() == EAccountType.PremiumPageBlobs() { + return out + } resp, err := b.internalClient.GetTags(ctx, nil) a.NoError("Get tags", err) for _, tag := range resp.BlobTagSet { @@ -721,6 +726,7 @@ func (b *BlobObjectResourceManager) Level() cmd.LocationLevel { func (b *BlobObjectResourceManager) URI(opts ...GetURIOptions) string { base := blobStripSAS(b.internalClient.URL()) base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } diff --git a/e2etest/newe2e_resource_managers_blobfs.go b/e2etest/newe2e_resource_managers_blobfs.go index 666457b1c..f96da6a1e 100644 --- a/e2etest/newe2e_resource_managers_blobfs.go +++ b/e2etest/newe2e_resource_managers_blobfs.go @@ -84,6 +84,7 @@ func (b *BlobFSServiceResourceManager) Level() cmd.LocationLevel { func (b *BlobFSServiceResourceManager) URI(opts ...GetURIOptions) string { base := dfsStripSAS(b.internalClient.DFSURL()) base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -178,6 +179,7 @@ func (b *BlobFSFileSystemResourceManager) Level() cmd.LocationLevel { func (b *BlobFSFileSystemResourceManager) URI(opts ...GetURIOptions) string { base := dfsStripSAS(b.internalClient.DFSURL()) base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -312,6 +314,7 @@ func (b *BlobFSPathResourceProvider) Level() cmd.LocationLevel { func (b *BlobFSPathResourceProvider) URI(opts ...GetURIOptions) string { base := dfsStripSAS(b.getFileClient().DFSURL()) // obj type doesn't matter here, URL is the same under the hood base = b.internalAccount.ApplySAS(base, b.Location(), opts...) + base = addWildCard(base, opts...) return base } diff --git a/e2etest/newe2e_resource_managers_file.go b/e2etest/newe2e_resource_managers_file.go index fb2996a8a..755ab7b20 100644 --- a/e2etest/newe2e_resource_managers_file.go +++ b/e2etest/newe2e_resource_managers_file.go @@ -85,6 +85,7 @@ func (s *FileServiceResourceManager) Level() cmd.LocationLevel { func (s *FileServiceResourceManager) URI(opts ...GetURIOptions) string { base := fileStripSAS(s.internalClient.URL()) base = s.internalAccount.ApplySAS(base, s.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -181,6 +182,7 @@ func (s *FileShareResourceManager) Level() cmd.LocationLevel { func (s *FileShareResourceManager) URI(opts ...GetURIOptions) string { base := fileStripSAS(s.internalClient.URL()) base = s.internalAccount.ApplySAS(base, s.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -288,6 +290,7 @@ func (s *FileShareResourceManager) ListObjects(a Asserter, targetDir string, rec FileCreationTime: v.Properties.CreationTime, FileLastWriteTime: v.Properties.LastWriteTime, FilePermissions: permissions, + LastModifiedTime: v.Properties.LastModified, }, } } @@ -324,6 +327,7 @@ func (s *FileShareResourceManager) ListObjects(a Asserter, targetDir string, rec FileCreationTime: v.Properties.CreationTime, FileLastWriteTime: v.Properties.LastWriteTime, FilePermissions: permissions, + LastModifiedTime: v.Properties.LastModified, }, } } @@ -398,6 +402,7 @@ func (f *FileObjectResourceManager) Level() cmd.LocationLevel { func (f *FileObjectResourceManager) URI(opts ...GetURIOptions) string { base := fileStripSAS(f.getFileClient().URL()) // restype doesn't matter here, same URL under the hood base = f.internalAccount.ApplySAS(base, f.Location(), opts...) + base = addWildCard(base, opts...) return base } @@ -546,6 +551,7 @@ func (f *FileObjectResourceManager) GetProperties(a Asserter) (out ObjectPropert FileCreationTime: resp.FileCreationTime, FileLastWriteTime: resp.FileLastWriteTime, FilePermissions: permissions, + LastModifiedTime: resp.LastModified, }, } case common.EEntityType.File(): @@ -577,6 +583,7 @@ func (f *FileObjectResourceManager) GetProperties(a Asserter) (out ObjectPropert FileCreationTime: resp.FileCreationTime, FileLastWriteTime: resp.FileLastWriteTime, FilePermissions: permissions, + LastModifiedTime: resp.LastModified, }, } default: diff --git a/e2etest/newe2e_resource_managers_local.go b/e2etest/newe2e_resource_managers_local.go index 19e9631d1..6f0552ae3 100644 --- a/e2etest/newe2e_resource_managers_local.go +++ b/e2etest/newe2e_resource_managers_local.go @@ -55,7 +55,9 @@ func (l *LocalContainerResourceManager) Level() cmd.LocationLevel { } func (l *LocalContainerResourceManager) URI(opts ...GetURIOptions) string { - return l.RootPath + base := l.RootPath + base = addWildCard(base, opts...) + return base } func (l *LocalContainerResourceManager) Parent() ResourceManager { @@ -217,7 +219,9 @@ func (l *LocalObjectResourceManager) Level() cmd.LocationLevel { } func (l *LocalObjectResourceManager) URI(opts ...GetURIOptions) string { - return filepath.Join(l.container.RootPath, l.objectPath) + base := filepath.Join(l.container.RootPath, l.objectPath) + base = addWildCard(base, opts...) + return base } func (l *LocalObjectResourceManager) Parent() ResourceManager { @@ -311,12 +315,10 @@ func (l *LocalObjectResourceManager) GetProperties(a Asserter) ObjectProperties perms := smb.GetSDDL(a) - out.FileProperties = FileProperties{ - FileAttributes: PtrOf(attr.String()), - FileCreationTime: PtrOf(props.FileCreationTime()), - FileLastWriteTime: PtrOf(props.FileLastWriteTime()), - FilePermissions: common.Iff(perms == "", nil, &perms), - } + out.FileProperties.FileAttributes = PtrOf(attr.String()) + out.FileProperties.FileCreationTime = PtrOf(props.FileCreationTime()) + out.FileProperties.FileLastWriteTime = PtrOf(props.FileLastWriteTime()) + out.FileProperties.FilePermissions = common.Iff(perms == "", nil, &perms) } return out diff --git a/e2etest/newe2e_task_runazcopy.go b/e2etest/newe2e_task_runazcopy.go index 1781ce966..1145c8aeb 100644 --- a/e2etest/newe2e_task_runazcopy.go +++ b/e2etest/newe2e_task_runazcopy.go @@ -68,6 +68,8 @@ type CreateAzCopyTargetOptions struct { // SASTokenOptions expects a GenericSignatureValues, which can contain account signatures, or a service signature. SASTokenOptions GenericSignatureValues Scheme string + // The wildcard string to append to the end of a resource URI. + Wildcard string } func CreateAzCopyTarget(rm ResourceManager, authType ExplicitCredentialTypes, a Asserter, opts ...CreateAzCopyTargetOptions) AzCopyTarget { @@ -161,6 +163,7 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri opts.AzureOpts.SASValues = tgt.Opts.SASTokenOptions opts.RemoteOpts.Scheme = tgt.Opts.Scheme + opts.Wildcard = tgt.Opts.Wildcard } else if target.Location() == common.ELocation.S3() { intendedAuthType = EExplicitCredentialType.S3() } else if target.Location() == common.ELocation.GCP() { @@ -169,7 +172,7 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri switch intendedAuthType { case EExplicitCredentialType.PublicAuth(), EExplicitCredentialType.None(): - return target.URI() // no SAS, no nothing. + return target.URI(opts) // no SAS, no nothing. case EExplicitCredentialType.SASToken(): opts.AzureOpts.WithSAS = true return target.URI(opts) @@ -224,10 +227,10 @@ func (c *AzCopyCommand) applyTargetAuth(a Asserter, target ResourceManager) stri } } - return target.URI() // Generate like public + return target.URI(opts) // Generate like public default: a.Error("unsupported credential type") - return target.URI() + return target.URI(opts) } } diff --git a/e2etest/newe2e_task_runazcopy_parameters.go b/e2etest/newe2e_task_runazcopy_parameters.go index 6abf2f3f8..57239f721 100644 --- a/e2etest/newe2e_task_runazcopy_parameters.go +++ b/e2etest/newe2e_task_runazcopy_parameters.go @@ -267,6 +267,7 @@ type CopySyncCommonFlags struct { TrailingDot *common.TrailingDotOption `flag:"trailing-dot"` CPKByName *string `flag:"cpk-by-name"` CPKByValue *bool `flag:"cpk-by-value"` + IncludePattern *string `flag:"include-pattern"` } // CopyFlags is a more exclusive struct including flags exclusi diff --git a/e2etest/zt_amanaged_disks_test.go b/e2etest/zt_aamanaged_disks_test.go similarity index 100% rename from e2etest/zt_amanaged_disks_test.go rename to e2etest/zt_aamanaged_disks_test.go diff --git a/e2etest/zt_newe2e_blob_test.go b/e2etest/zt_newe2e_blob_test.go new file mode 100644 index 000000000..b1f4b6b60 --- /dev/null +++ b/e2etest/zt_newe2e_blob_test.go @@ -0,0 +1,206 @@ +package e2etest + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" +) + +func init() { + suiteManager.RegisterSuite(&BlobTestSuite{}) +} + +type BlobTestSuite struct{} + +func (s *BlobTestSuite) Scenario_UploadBlockBlobs(svm *ScenarioVariationManager) { + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_10_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 10 { + name := "dir_10_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObject, dstContainer.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} + +func (s *BlobTestSuite) Scenario_UploadPageBlob(svm *ScenarioVariationManager) { + fileName := "test_page_blob_1mb.vHd" + body := NewRandomObjectContentContainer(svm, common.MegaByte) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj.(RemoteResourceManager).WithSpecificAuthType(ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.SASToken(), EExplicitCredentialType.OAuth()}), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + BlockSizeMB: pointerTo(4.0), + }, + BlobType: pointerTo(common.EBlobType.PageBlob()), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + }, + }, + }, true) +} + +func (s *BlobTestSuite) Scenario_SetPageBlobTier(svm *ScenarioVariationManager) { + fileName := "test_page_blob.vHd" + body := NewRandomObjectContentContainer(svm, common.KiloByte) + tier := ResolveVariation(svm, []common.PageBlobTier{common.EPageBlobTier.P10(), common.EPageBlobTier.P20(), common.EPageBlobTier.P30(), common.EPageBlobTier.P4(), common.EPageBlobTier.P40(), common.EPageBlobTier.P50()}) + + srcObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}). + GetObject(svm, fileName, common.EEntityType.File()) + + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob(), GetResourceOptions{PreferredAccount: pointerTo(PremiumPageBlobAcct)}), ResourceDefinitionContainer{}) + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + BlobType: pointerTo(common.EBlobType.PageBlob()), + PageBlobTier: pointerTo(tier), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + ObjectProperties: ObjectProperties{ + BlobProperties: BlobProperties{ + Type: pointerTo(blob.BlobTypePageBlob), + BlockBlobAccessTier: pointerTo(tier.ToAccessTierType()), + }, + }, + }, true) +} + +func (s *BlobTestSuite) Scenario_UploadBlob(svm *ScenarioVariationManager) { + // Scale up from service to object + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}).GetObject(svm, "test", common.EEntityType.File()) + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + // Scale up from service to object + srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionObject{ + ObjectName: pointerTo("test"), + Body: body, + }) + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + srcObj, + TryApplySpecificAuthType(dstObj, ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.OAuth(), EExplicitCredentialType.SASToken()}), svm, CreateAzCopyTargetOptions{}), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobTestSuite) Scenario_DownloadBlob(svm *ScenarioVariationManager) { + // Scale up from service to object + dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}).GetObject(svm, "test", common.EEntityType.File()) + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + // Scale up from service to object + srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionObject{ + ObjectName: pointerTo("test"), + Body: body, + }) + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcObj, ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.OAuth(), EExplicitCredentialType.SASToken()}), svm, CreateAzCopyTargetOptions{}), + dstObj, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *BlobTestSuite) Scenario_DownloadBlobRecursive(svm *ScenarioVariationManager) { + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_5_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 5 { + name := "dir_5_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + srcObjs[name] = obj + } + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcObject, ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.OAuth(), EExplicitCredentialType.SASToken()}), svm, CreateAzCopyTargetOptions{}), + dstContainer, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} diff --git a/e2etest/zt_newe2e_file_test.go b/e2etest/zt_newe2e_file_test.go index 5f629b736..48d425297 100644 --- a/e2etest/zt_newe2e_file_test.go +++ b/e2etest/zt_newe2e_file_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" "math" ) @@ -250,3 +251,262 @@ func (s *FileTestSuite) Scenario_Download63MBFile(svm *ScenarioVariationManager) Body: body, }, false) } + +func (s *FileTestSuite) Scenario_UploadDirectory(svm *ScenarioVariationManager) { + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + + dirsToCreate := []string{"dir_file_copy_test", "dir_file_copy_test/sub_dir_copy_test"} + + // Create destination directories + srcObjs := make(ObjectResourceMappingFlat) + for _, dir := range dirsToCreate { + obj := ResourceDefinitionObject{ObjectName: pointerTo(dir), ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()}} + if dstContainer.Location() != common.ELocation.Blob() { + srcObjs[dir] = obj + } + for i := range 3 { + name := dir + "/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + srcObjs[name] = obj + } + } + + for _, obj := range srcObjs { + if obj.EntityType != common.EEntityType.Folder() { + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + } + } + + srcDir := srcContainer.GetObject(svm, dirsToCreate[0], common.EEntityType.Folder()) + dstDir := dstContainer.GetObject(svm, dirsToCreate[0], common.EEntityType.Folder()) + + sasOpts := GenericAccountSignatureValues{} + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcDir, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + TryApplySpecificAuthType(dstDir, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + AsSubdir: pointerTo(false), + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} + +func (s *FileTestSuite) Scenario_DownloadDirectory(svm *ScenarioVariationManager) { + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + + dirsToCreate := []string{"dir_file_copy_test", "dir_file_copy_test/sub_dir_copy_test"} + + // Create destination directories + srcObjs := make(ObjectResourceMappingFlat) + for _, dir := range dirsToCreate { + obj := ResourceDefinitionObject{ObjectName: pointerTo(dir), ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()}} + if dstContainer.Location() != common.ELocation.Blob() { + srcObjs[dir] = obj + } + for i := range 3 { + name := dir + "/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + srcObjs[name] = obj + } + } + + for _, obj := range srcObjs { + if obj.EntityType != common.EEntityType.Folder() { + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + } + } + + srcDir := srcContainer.GetObject(svm, dirsToCreate[0], common.EEntityType.Folder()) + dstDir := dstContainer.GetObject(svm, dirsToCreate[0], common.EEntityType.Folder()) + + sasOpts := GenericAccountSignatureValues{} + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcDir, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + TryApplySpecificAuthType(dstDir, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + AsSubdir: pointerTo(false), + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} + +func (s *FileTestSuite) Scenario_SingleFileUploadWildcard(svm *ScenarioVariationManager) { + size := common.MegaByte + fileName := fmt.Sprintf("test_file_upload_%dB_fullname.txt", size) + body := NewRandomObjectContentContainer(svm, int64(size)) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "/*"}), dstContainer}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *FileTestSuite) Scenario_AllFileUploadWildcard(svm *ScenarioVariationManager) { + size := common.KiloByte + fileName := fmt.Sprintf("test_file_upload_%dB_fullname", size) + body := NewRandomObjectContentContainer(svm, int64(size)) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "/*"}), + dstContainer, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *FileTestSuite) Scenario_AllFileDownloadWildcard(svm *ScenarioVariationManager) { + size := common.KiloByte + fileName := fmt.Sprintf("test_file_upload_%dB_fullname", size) + body := NewRandomObjectContentContainer(svm, int64(size)) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "/*"}), + dstContainer, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *FileTestSuite) Scenario_SeveralFileUploadWildcard(svm *ScenarioVariationManager) { + size := common.KiloByte + fileName := fmt.Sprintf("test_file_upload_%dB_fullname", size) + body := NewRandomObjectContentContainer(svm, int64(size)) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{Wildcard: "/test_file*"}), + dstContainer, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} + +func (s *FileTestSuite) Scenario_SeveralFileDownloadWildcard(svm *ScenarioVariationManager) { + size := common.KiloByte + fileName := fmt.Sprintf("test_file_upload_%dB_fullname", size) + body := NewRandomObjectContentContainer(svm, int64(size)) + + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.File()), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Local()), ResourceDefinitionContainer{}) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm), + dstContainer, + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + BlockSizeMB: pointerTo(4.0), + IncludePattern: pointerTo("test_file*"), + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstContainer.GetObject(svm, srcContainer.ContainerName()+"/"+fileName, common.EEntityType.File()), ResourceDefinitionObject{ + Body: body, + }, true) +} diff --git a/e2etest/zt_newe2e_remove_test.go b/e2etest/zt_newe2e_remove_test.go index 957f4c09b..3d5408a3e 100644 --- a/e2etest/zt_newe2e_remove_test.go +++ b/e2etest/zt_newe2e_remove_test.go @@ -4,6 +4,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { @@ -71,3 +72,37 @@ func (s *RemoveSuite) Scenario_EmptySASErrorCodes(svm *ScenarioVariationManager) // Validate that the stdout contains these error URLs ValidateErrorOutput(svm, stdout, "https://aka.ms/AzCopyError/NoAuthenticationInformation") } + +func (s *RemoveSuite) Scenario_RemoveVirtualDirectory(svm *ScenarioVariationManager) { + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + srcObject := srcContainer.GetObject(svm, "dir_5_files", common.EEntityType.Folder()) + + srcObjs := make(ObjectResourceMappingFlat) + for i := range 5 { + name := "dir_5_files/test" + strconv.Itoa(i) + ".txt" + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: NewRandomObjectContentContainer(svm, SizeFromString("1K"))} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + obj.Body = nil + obj.ObjectShouldExist = to.Ptr(false) + srcObjs[name] = obj + } + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbRemove, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcObject, ResolveVariation(svm, []ExplicitCredentialTypes{EExplicitCredentialType.OAuth(), EExplicitCredentialType.SASToken()}), svm, CreateAzCopyTargetOptions{}), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, srcContainer, ResourceDefinitionContainer{ + Objects: srcObjs, + }, true) +} diff --git a/e2etest/zt_newe2e_s2s_test.go b/e2etest/zt_newe2e_s2s_test.go index 28138357d..b286196b4 100644 --- a/e2etest/zt_newe2e_s2s_test.go +++ b/e2etest/zt_newe2e_s2s_test.go @@ -3,6 +3,7 @@ package e2etest import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-storage-azcopy/v10/common" + "strconv" ) func init() { @@ -41,6 +42,36 @@ func (s *S2STestSuite) Scenario_BlobDestinationSizes(svm *ScenarioVariationManag }, true) } +func (s *S2STestSuite) Scenario_BlobFile1KB(svm *ScenarioVariationManager) { + src := common.ELocation.Blob() + dst := common.ELocation.File() + size := common.KiloByte + fileName := "test_copy.txt" + body := NewRandomObjectContentContainer(svm, int64(size)) + + // TODO : Add S3 to source + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, src), ResourceDefinitionContainer{}) + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcObj.Create(svm, body, ObjectProperties{}) + + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, dst), ResourceDefinitionContainer{}) + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + RunAzCopy(svm, AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{srcObj, dstObj.(RemoteResourceManager).WithSpecificAuthType(EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{})}, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: body, + }, true) +} + func (s *S2STestSuite) Scenario_SingleFileCopyBlobTypeVariations(svm *ScenarioVariationManager) { srcBlobType := ResolveVariation(svm, []blob.BlobType{blob.BlobTypeBlockBlob, blob.BlobTypePageBlob, blob.BlobTypeAppendBlob}) destBlobType := ResolveVariation(svm, []blob.BlobType{blob.BlobTypeBlockBlob, blob.BlobTypePageBlob, blob.BlobTypeAppendBlob}) @@ -334,3 +365,257 @@ func (s *S2STestSuite) Scenario_BlobBlobOAuth(svm *ScenarioVariationManager) { Body: body, }, true) } + +func (s *S2STestSuite) Scenario_S2SContainerSingleFilePropertyAndMetadata(svm *ScenarioVariationManager) { + preserveProperty := ResolveVariation(svm, []bool{true, false}) + fileName := "s2scontainer_propertyandmetadata.txt" + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(), common.ELocation.File()})), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcBody := NewRandomObjectContentContainer(svm, 0) + srcProps := ObjectProperties{ + Metadata: common.Metadata{"Author": pointerTo("gapra"), "Viewport": pointerTo("width"), "Description": pointerTo("test file")}, + HTTPHeaders: contentHeaders{ + contentType: pointerTo("testctype"), + contentEncoding: pointerTo("testcenc"), + contentDisposition: pointerTo("testcdis"), + contentLanguage: pointerTo("testclang"), + cacheControl: pointerTo("testcctrl"), + }, + } + srcObj.Create(svm, srcBody, srcProps) + + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + sasOpts := GenericAccountSignatureValues{} + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + TryApplySpecificAuthType(dstContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + S2SPreserveProperties: pointerTo(preserveProperty), + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: srcBody, + ObjectProperties: common.Iff(preserveProperty, srcProps, ObjectProperties{}), + }, false) +} + +func (s *S2STestSuite) Scenario_S2SContainerSingleFileStripTopDir(svm *ScenarioVariationManager) { + recursive := ResolveVariation(svm, []bool{true, false}) + fileName := "copy_strip_top_dir_file.txt" + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(), common.ELocation.File()})), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + srcObj := srcContainer.GetObject(svm, fileName, common.EEntityType.File()) + srcBody := NewRandomObjectContentContainer(svm, 0) + srcObj.Create(svm, srcBody, ObjectProperties{}) + + dstObj := dstContainer.GetObject(svm, fileName, common.EEntityType.File()) + + sasOpts := GenericAccountSignatureValues{} + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + Wildcard: "/*", + }), + TryApplySpecificAuthType(dstContainer, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(recursive), + }, + }, + }) + + ValidateResource[ObjectResourceManager](svm, dstObj, ResourceDefinitionObject{ + Body: srcBody, + }, false) +} + +func (s *S2STestSuite) Scenario_S2SDirectoryMultipleFiles(svm *ScenarioVariationManager) { + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(), common.ELocation.File()})), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + dirsToCreate := []string{"dir_file_copy_test", "dir_file_copy_test/sub_dir_copy_test"} + + // Create destination directories + dstObjs := make(ObjectResourceMappingFlat) + for _, dir := range dirsToCreate { + if dstContainer.Location() != common.ELocation.Blob() { + dstObj := ResourceDefinitionObject{ObjectName: pointerTo("dir_file_copy_test/" + dir), ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()}} + dstObjs["dir_file_copy_test/"+dir] = dstObj + } + for i := range 10 { + name := dir + "/test" + strconv.Itoa(i) + ".txt" + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + dstObj := ResourceDefinitionObject{ObjectName: pointerTo("dir_file_copy_test/" + name), Body: body} + dstObjs["dir_file_copy_test/"+name] = dstObj // src directory will be created in dest directory + } + } + + sasOpts := GenericAccountSignatureValues{} + + srcDirObj := srcContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + dstDirObj := dstContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + TryApplySpecificAuthType(dstDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: dstObjs, + }, true) +} + +func (s *S2STestSuite) Scenario_S2SDirectoryMultipleFilesStripTopDirRecursive(svm *ScenarioVariationManager) { + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(), common.ELocation.File()})), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + dirsToCreate := []string{"dir_file_copy_test", "dir_file_copy_test/sub_dir_copy_test"} + + // Create destination directories + dstObjs := make(ObjectResourceMappingFlat) + for _, dir := range dirsToCreate { + if dstContainer.Location() != common.ELocation.Blob() { + dirName := dir + dstObj := ResourceDefinitionObject{ObjectName: pointerTo(dirName), ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()}} + dstObjs[dirName] = dstObj + } + for i := range 10 { + name := dir + "/test" + strconv.Itoa(i) + ".txt" + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + dstObj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body} + dstObjs[name] = dstObj + } + } + + sasOpts := GenericAccountSignatureValues{} + + srcDirObj := srcContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + dstDirObj := dstContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + Wildcard: "/*", + }), + TryApplySpecificAuthType(dstDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{ + CopySyncCommonFlags: CopySyncCommonFlags{ + Recursive: pointerTo(true), + }, + }, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: dstObjs, + }, true) +} + +func (s *S2STestSuite) Scenario_S2SDirectoryMultipleFilesStripTopDirNonRecursive(svm *ScenarioVariationManager) { + // Scale up from service to object + srcContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Blob(), common.ELocation.File()})), ResourceDefinitionContainer{}) + dstContainer := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, common.ELocation.Blob()), ResourceDefinitionContainer{}) + + dirsToCreate := []string{"dir_file_copy_test", "dir_file_copy_test/sub_dir_copy_test"} + + // Create destination directories + dstObjs := make(ObjectResourceMappingFlat) + for j, dir := range dirsToCreate { + if dstContainer.Location() != common.ELocation.Blob() { + dirName := dir + if j == 0 { + dstObj := ResourceDefinitionObject{ObjectName: pointerTo(dirName), ObjectProperties: ObjectProperties{EntityType: common.EEntityType.Folder()}} + dstObjs[dirName] = dstObj + } + } + for i := range 10 { + name := dir + "/test" + strconv.Itoa(i) + ".txt" + body := NewRandomObjectContentContainer(svm, SizeFromString("1K")) + obj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body} + CreateResource[ObjectResourceManager](svm, srcContainer, obj) + if j == 0 { + dstObj := ResourceDefinitionObject{ObjectName: pointerTo(name), Body: body} + dstObjs[name] = dstObj + } + } + } + + sasOpts := GenericAccountSignatureValues{} + + srcDirObj := srcContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + dstDirObj := dstContainer.GetObject(svm, "dir_file_copy_test", common.EEntityType.Folder()) + + RunAzCopy( + svm, + AzCopyCommand{ + Verb: AzCopyVerbCopy, + Targets: []ResourceManager{ + TryApplySpecificAuthType(srcDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + Wildcard: "/*", + }), + TryApplySpecificAuthType(dstDirObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{ + SASTokenOptions: sasOpts, + }), + }, + Flags: CopyFlags{}, + }) + + ValidateResource[ContainerResourceManager](svm, dstContainer, ResourceDefinitionContainer{ + Objects: dstObjs, + }, true) +} diff --git a/ste/sender-azureFile.go b/ste/sender-azureFile.go index 97f554d25..2c2322d3c 100644 --- a/ste/sender-azureFile.go +++ b/ste/sender-azureFile.go @@ -43,7 +43,6 @@ type FileClientStub interface { URL() string } - // azureFileSenderBase implements both IFolderSender and (most of) IFileSender. // Why implement both interfaces in the one type, even though they are largely unrelated? Because it // makes functions like newAzureFilesUploader easier to reason about, since they always return the same type. @@ -112,7 +111,7 @@ func newAzureFileSenderBase(jptm IJobPartTransferMgr, destination string, pacer sURL, _ := file.ParseURL(serviceClient.URL()) addFileRequestIntent := (sURL.SAS.Signature() == "") // We are using oAuth - + shareClient := serviceClient.NewShareClient(shareName) if shareSnapshot != "" { shareClient, err = shareClient.WithSnapshot(shareSnapshot) @@ -210,7 +209,7 @@ func (u *azureFileSenderBase) Prologue(state common.PrologueState) (destinationM if fileerror.HasCode(err, fileerror.ParentNotFound) { // Create the parent directories of the file. Note share must be existed, as the files are listed from share or directory. - jptm.Log(common.LogError, fmt.Sprintf("%s: %s \n AzCopy going to create parent directories of the Azure files", fileerror.ParentNotFound, err.Error())) + jptm.Log(common.LogError, fmt.Sprintf("%s: %s \n AzCopy is going to create parent directories of the Azure files", fileerror.ParentNotFound, err.Error())) err = AzureFileParentDirCreator{}.CreateParentDirToRoot(u.ctx, u.getFileClient(), u.shareClient, u.jptm.GetFolderCreationTracker()) if err != nil { u.jptm.FailActiveUpload("Creating parent directory", err) diff --git a/testSuite/scripts/run.py b/testSuite/scripts/run.py index c986f0939..5fdd4e779 100644 --- a/testSuite/scripts/run.py +++ b/testSuite/scripts/run.py @@ -1,9 +1,6 @@ from test_blob_download import * from test_upload_block_blob import * from test_upload_page_blob import * -from test_file_download import * -from test_file_upload import * -from test_azcopy_operations import * from test_blobfs_upload_sharedkey import * from test_blobfs_download_sharedkey import * from test_blob_piping import * @@ -208,9 +205,6 @@ def main(): Block_Upload_User_Scenarios, Blob_Download_User_Scenario, PageBlob_Upload_User_Scenarios, - Azcopy_Operation_User_Scenario, - FileShare_Download_User_Scenario, - FileShare_Upload_User_Scenario, BlobFs_Upload_ShareKey_User_Scenarios, BlobFs_Download_SharedKey_User_Scenarios, Service_2_Service_Copy_User_Scenario, diff --git a/testSuite/scripts/test_azcopy_operations.py b/testSuite/scripts/test_azcopy_operations.py deleted file mode 100644 index e78e0c655..000000000 --- a/testSuite/scripts/test_azcopy_operations.py +++ /dev/null @@ -1,53 +0,0 @@ -import utility as util -import json -import unittest -from collections import namedtuple -import sys -import os - -class Azcopy_Operation_User_Scenario(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - - - # test_remove_virtual_directory creates a virtual directory, removes the virtual directory created - # and then verifies the contents of virtual directory. - def test_remove_virtual_directory(self): - # create dir dir_10_files and 1 kb files inside the dir. - dir_name = "dir_" + str(10) + "_files_rm" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - destination = util.get_resource_sas(dir_name) - result = util.Command("rm").add_arguments(destination).add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - result = util.Command("list").add_arguments(destination).add_flags("resource-num", "0").execute_azcopy_verify() - self.assertTrue(result) - - def test_remove_virtual_directory_oauth(self): - # create dir dir_10_files and 1 kb files inside the dir. - dir_name = "dir_" + str(10) + "_files_rm_oauth" - dir_n_files_path = util.create_test_n_files(1024, 10, dir_name) - - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(util.test_oauth_container_url). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - destination = util.get_object_without_sas(util.test_oauth_container_url, dir_name) - result = util.Command("rm").add_arguments(destination).add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - destination_with_sas = util.get_object_sas(util.test_oauth_container_validate_sas_url, dir_name) - result = util.Command("list").add_arguments(destination_with_sas).add_flags("resource-num", "0").execute_azcopy_verify() - self.assertTrue(result) diff --git a/testSuite/scripts/test_blob_download.py b/testSuite/scripts/test_blob_download.py index d33d2f514..251d36b95 100644 --- a/testSuite/scripts/test_blob_download.py +++ b/testSuite/scripts/test_blob_download.py @@ -9,14 +9,6 @@ import unittest class Blob_Download_User_Scenario(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - # test_download_1kb_blob_to_null verifies that a 1kb blob can be downloaded to null and the md5 can be checked successfully def test_download_1kb_blob_to_null(self): # create file of size 1kb @@ -64,37 +56,6 @@ def test_download_1kb_blob_to_root(self): dst = "/" result = util.Command("copy").add_arguments(src).add_arguments(dst).add_flags("log-level", "info") - # test_download_1kb_blob verifies the download of 1Kb blob using azcopy. - def test_download_1kb_blob(self): - # create file of size 1KB. - filename = "test_1kb_blob_upload.txt" - file_path = util.create_test_file(filename, 1024) - - # Upload 1KB file using azcopy. - src = file_path - dest = util.test_container_url - result = util.Command("copy").add_arguments(src).add_arguments(dest). \ - add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the uploaded blob. - # the resource local path should be the first argument for the azcopy validator. - # the resource sas should be the second argument for azcopy validator. - resource_url = util.get_resource_sas(filename) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify() - self.assertTrue(result) - - # downloading the uploaded file - src = util.get_resource_sas(filename) - dest = util.test_directory_path + "/test_1kb_blob_download.txt" - result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level", - "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the downloaded blob - result = util.Command("testBlob").add_arguments(dest).add_arguments(src).execute_azcopy_verify() - self.assertTrue(result) - # test_download_preserve_last_modified_time verifies the azcopy downloaded file # and its modified time preserved locally on disk def test_blob_download_preserve_last_modified_time(self): @@ -153,38 +114,6 @@ def test_blob_download_63mb_in_4mb(self): destination_sas).execute_azcopy_verify() self.assertTrue(result) - # test_recursive_download_blob downloads a directory recursively from container through azcopy - def recursive_download_blob(self): - # create directory and 5 files of 1KB inside that directory. - dir_name = "dir_" + str(10) + "_files" - dir1_path = util.create_test_n_files(1024, 5, dir_name) - - # upload the directory to container through azcopy with recursive set to true. - result = util.Command("copy").add_arguments(dir1_path).add_arguments(util.test_container_url).\ - add_flags("log-level","info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # verify the uploaded file. - destination_sas = util.get_resource_sas(dir_name) - result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(destination_sas).\ - add_flags("is-object-dir","true").execute_azcopy_verify() - self.assertTrue(result) - try: - shutil.rmtree(dir1_path) - except OSError as e: - self.fail('error removing the file ' + dir1_path) - - # downloading the directory created from container through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(destination_sas).add_arguments(util.test_directory_path).add_flags( - "log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # verify downloaded blob. - result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(destination_sas).\ - add_flags("is-object-dir","true").execute_azcopy_verify() - self.assertTrue(result) - - def test_blob_download_with_special_characters(self): filename_special_characters = "abc|>rd*" # encode filename beforehand to avoid erroring out @@ -211,86 +140,6 @@ def test_blob_download_with_special_characters(self): result = util.Command("testBlob").add_arguments(filepath).add_arguments(resource_url).execute_azcopy_verify() self.assertTrue(result) - # test_download_1kb_blob verifies the download of 1Kb blob using azcopy. - def test_download_1kb_blob_with_oauth(self): - self.util_test_download_1kb_blob_with_oauth() - - def util_test_download_1kb_blob_with_oauth(self): - # create file of size 1KB. - filename = "test_1kb_blob_upload.txt" - file_path = util.create_test_file(filename, 1024) - - # Upload 1KB file using azcopy. - src = file_path - dest = util.test_oauth_container_url - cmd = util.Command("copy").add_arguments(src).add_arguments(dest). \ - add_flags("log-level", "info").add_flags("recursive", "true") - util.process_oauth_command(cmd, "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the uploaded blob. - # the resource local path should be the first argument for the azcopy validator. - # the resource sas should be the second argument for azcopy validator. - dest_validate = util.get_resource_from_oauth_container_validate(filename) - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest_validate).execute_azcopy_verify() - self.assertTrue(result) - - # downloading the uploaded file - src = util.get_resource_from_oauth_container(filename) - src_validate = util.get_resource_from_oauth_container_validate(filename) - dest = util.test_directory_path + "/test_1kb_blob_download.txt" - cmd = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("log-level", "info") - util.process_oauth_command(cmd, "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the downloaded blob - result = util.Command("testBlob").add_arguments(dest).add_arguments(src_validate).execute_azcopy_verify() - self.assertTrue(result) - - # test_recursive_download_blob downloads a directory recursively from container through azcopy - def test_recursive_download_blob_with_oauth(self): - self.util_test_recursive_download_blob_with_oauth() - - def util_test_recursive_download_blob_with_oauth( - self): - # create directory and 5 files of 1KB inside that directory. - dir_name = "util_test_recursive_download_blob_with_oauth_dir_" + str(10) + "_files" - dir1_path = util.create_test_n_files(1024, 5, dir_name) - - dest = util.test_oauth_container_url - # upload the directory to container through azcopy with recursive set to true. - cmd = util.Command("copy").add_arguments(dir1_path).add_arguments(dest).add_flags("log-level", "info") \ - .add_flags("recursive", "true") - util.process_oauth_command(cmd, "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # verify the uploaded file. - dest_validate = util.get_resource_from_oauth_container_validate(dir_name) - result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(dest_validate).add_flags("is-object-dir", - "true").execute_azcopy_verify() - self.assertTrue(result) - - try: - shutil.rmtree(dir1_path) - except OSError as e: - self.fail("error removing the upload files. " + e) - - src_download = util.get_resource_from_oauth_container(dir_name) - # downloading the directory created from container through azcopy with recursive flag to true. - cmd = util.Command("copy").add_arguments(src_download).add_arguments(util.test_directory_path).add_flags( - "log-level", "info").add_flags("recursive", "true") - util.process_oauth_command(cmd, "") - result = cmd.execute_azcopy_copy_command() - self.assertTrue(result) - - # verify downloaded blob. - result = util.Command("testBlob").add_arguments(dir1_path).add_arguments(dest_validate).add_flags("is-object-dir", - "true").execute_azcopy_verify() - self.assertTrue(result) - def test_blob_download_wildcard_recursive_false_1(self): #This test verifies the azcopy behavior when wildcards are # provided in the source and recursive flag is set to false diff --git a/testSuite/scripts/test_file_download.py b/testSuite/scripts/test_file_download.py deleted file mode 100644 index 60c0b2311..000000000 --- a/testSuite/scripts/test_file_download.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -import shutil -import time -import unittest -import utility as util - -class FileShare_Download_User_Scenario(unittest.TestCase): - - # test_upload_download_1kb_file_wildcard_all_files verifies the upload/download of 1Kb file with wildcard using azcopy. - def test_upload_download_1kb_file_wildcard_all_files(self): - # create file of size 1KB. - filename = "test_upload_download_1kb_file_wildcard_all_files.txt" - file_path = util.create_test_file(filename, 1024) - - wildcard_path = file_path.replace(filename, "*") - - # Upload 1KB file using azcopy. - result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the uploaded file. - # the resource local path should be the first argument for the azcopy validator. - # the resource sas should be the second argument for azcopy validator. - resource_url = util.get_resource_sas_from_share(filename) - result = util.Command("testFile").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify() - self.assertTrue(result) - - # downloading the uploaded file - src = util.get_resource_sas_from_share(filename) - src_wildcard = util.get_resource_sas_from_share("*") - dest = util.test_directory_path + "/test_upload_download_1kb_file_wildcard_all_files_dir" - try: - if os.path.exists(dest) and os.path.isdir(dest): - shutil.rmtree(dest) - except: - self.fail('error removing directory ' + dest) - finally: - os.makedirs(dest) - - result = util.Command("copy").add_arguments(src_wildcard).add_arguments(dest). \ - add_flags("log-level", "info").add_flags("include-pattern", filename.replace("wildcard", "*")). \ - execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the downloaded file - result = util.Command("testFile").add_arguments(os.path.join(dest, filename)).add_arguments( - src).execute_azcopy_verify() - self.assertTrue(result) - - # test_upload_download_1kb_file_fullname verifies the upload/download of 1Kb file with wildcard using azcopy. - def test_upload_download_1kb_file_wildcard_several_files(self): - # create file of size 1KB. - filename = "test_upload_download_1kb_file_wildcard_several_files.txt" - prefix = "test_upload_download_1kb_file_wildcard_several*" - file_path = util.create_test_file(filename, 1024) - - wildcardSrc = file_path.replace(filename, prefix) - # Upload 1KB file using azcopy. - result = util.Command("copy").add_arguments(wildcardSrc).add_arguments(util.test_share_url). \ - add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the uploaded file. - # the resource local path should be the first argument for the azcopy validator. - # the resource sas should be the second argument for azcopy validator. - resource_url = util.get_resource_sas_from_share(filename) - result = util.Command("testFile").add_arguments(file_path).add_arguments(resource_url).execute_azcopy_verify() - self.assertTrue(result) - - # downloading the uploaded file - src = util.get_resource_sas_from_share(filename) - wildcardSrc = util.get_resource_sas_from_share(prefix) - dest = util.test_directory_path + "/test_upload_download_1kb_file_wildcard_several_files" - try: - if os.path.exists(dest) and os.path.isdir(dest): - shutil.rmtree(dest) - except: - self.fail('error removing directory ' + dest) - finally: - os.makedirs(dest) - - result = util.Command("copy").add_arguments(src).add_arguments(dest).add_flags("include-pattern", prefix). \ - add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the downloaded file - result = util.Command("testFile").add_arguments(os.path.join(dest, filename)).add_arguments( - src).execute_azcopy_verify() - self.assertTrue(result) - - # util_test_n_1kb_file_in_dir_upload_download_azure_directory verifies the upload of n 1kb file to the share. - def util_test_n_1kb_file_in_dir_upload_download_azure_directory(self, number_of_files, recursive): - # create dir dir_n_files and 1 kb files inside the dir. - dir_name = "util_test_n_1kb_file_in_dir_upload_download_azure_directory_" + recursive + "_" + str( - number_of_files) + "_files" - sub_dir_name = "dir_subdir_" + str(number_of_files) + "_files" - - # create n test files in dir - src_dir = util.create_test_n_files(1024, number_of_files, dir_name) - - # create n test files in subdir, subdir is contained in dir - util.create_test_n_files(1024, number_of_files, os.path.join(dir_name, sub_dir_name)) - - # prepare destination directory. - # TODO: note azcopy v2 currently only support existing directory and share. - dest_azure_dir_name = "dest azure_dir_name" - dest_azure_dir = util.get_resource_sas_from_share(dest_azure_dir_name) - - result = util.Command("create").add_arguments(dest_azure_dir).add_flags("serviceType", "File"). \ - add_flags("resourceType", "Bucket").execute_azcopy_create() - self.assertTrue(result) - - # execute azcopy command - result = util.Command("copy").add_arguments(src_dir).add_arguments(dest_azure_dir). \ - add_flags("recursive", recursive).add_flags("log-level", "debug").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - dest_azure_dir_to_compare = util.get_resource_sas_from_share(dest_azure_dir_name + "/" + dir_name) - result = util.Command("testFile").add_arguments(src_dir).add_arguments(dest_azure_dir_to_compare). \ - add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify() - self.assertTrue(result) - - download_azure_src_dir = dest_azure_dir_to_compare - download_local_dest_dir = src_dir + "_download" - - try: - if os.path.exists(download_local_dest_dir) and os.path.isdir(download_local_dest_dir): - shutil.rmtree(download_local_dest_dir) - except: - print("catch error for removing " + download_local_dest_dir) - finally: - os.makedirs(download_local_dest_dir) - - # downloading the directory created from azure file directory through azcopy with recursive flag to true. - result = util.Command("copy").add_arguments(download_azure_src_dir).add_arguments( - download_local_dest_dir).add_flags("log-level", "debug"). \ - add_flags("recursive", recursive).execute_azcopy_copy_command() - self.assertTrue(result) - - # verify downloaded file. - # todo: ensure the comparing here - result = util.Command("testFile").add_arguments(os.path.join(download_local_dest_dir, dir_name)).add_arguments( - download_azure_src_dir). \ - add_flags("is-object-dir", "true").add_flags("is-recursive", recursive).execute_azcopy_verify() - self.assertTrue(result) - - def test_3_1kb_file_in_dir_upload_download_azure_directory_recursive(self): - self.util_test_n_1kb_file_in_dir_upload_download_azure_directory(3, "true") - diff --git a/testSuite/scripts/test_file_upload.py b/testSuite/scripts/test_file_upload.py deleted file mode 100644 index 86336fb03..000000000 --- a/testSuite/scripts/test_file_upload.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import shutil -import time -import utility as util -import unittest - -class FileShare_Upload_User_Scenario(unittest.TestCase): - - def test_file_upload_1mb_wildcard(self): - # create the test file. - file_name = "test_file_upload_1mb_wildcard" - file_path = util.create_test_file(file_name, 1024 * 1024) - - # execute azcopy upload. - destination = util.get_resource_sas_from_share(file_name) - wildcard_path = file_path.replace(file_name, "test_file_upload_1mb_wildcard*") - result = util.Command("copy").add_arguments(wildcard_path).add_arguments(util.test_share_url).add_flags("log-level", "info"). \ - add_flags("block-size-mb", "4").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute validator. - result = util.Command("testFile").add_arguments(file_path).add_arguments(destination).execute_azcopy_verify() - self.assertTrue(result) \ No newline at end of file diff --git a/testSuite/scripts/test_service_to_service_copy.py b/testSuite/scripts/test_service_to_service_copy.py index 16440113b..657835bda 100644 --- a/testSuite/scripts/test_service_to_service_copy.py +++ b/testSuite/scripts/test_service_to_service_copy.py @@ -12,8 +12,6 @@ class Service_2_Service_Copy_User_Scenario(unittest.TestCase): def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() # init bucket_name common_prefix = 's2scopybucket' @@ -24,10 +22,6 @@ def setUp(self): self.bucket_name_s3_blob = util.get_resource_name(common_prefix + 's3blob') self.bucket_name_block_append_page = util.get_resource_name(common_prefix + 'blockappendpage') - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - ################################## # Test from blob to blob copy. ################################## @@ -36,138 +30,35 @@ def test_copy_single_1kb_file_from_blob_to_blob_with_auth_env_var(self): dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) self.util_test_copy_single_file_from_x_to_x(src_container_url, "Blob", dst_container_url, "Blob", 1, oAuth=True, credTypeOverride="OAuthToken") - - def test_copy_10_files_from_blob_container_to_blob_container(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_n_files_from_x_bucket_to_x_bucket(src_container_url, "Blob", dst_container_url, "Blob") - - def test_copy_file_from_blob_container_to_blob_container_strip_top_dir_recursive(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_file_from_x_bucket_to_x_bucket_strip_top_dir(src_container_url, "Blob", dst_container_url, "Blob", True) - - def test_copy_file_from_blob_container_to_blob_container_strip_top_dir_non_recursive(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_file_from_x_bucket_to_x_bucket_strip_top_dir(src_container_url, "Blob", dst_container_url, "Blob", False) - - def test_copy_n_files_from_blob_dir_to_blob_dir(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_n_files_from_x_dir_to_x_dir(src_container_url, "Blob", dst_container_url, "Blob") - - def test_copy_n_files_from_blob_dir_to_blob_dir_strip_top_dir_recursive(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_n_files_from_x_dir_to_x_dir_strip_top_dir(src_container_url, "Blob", dst_container_url, "Blob", True) - - def test_copy_n_files_from_blob_dir_to_blob_dir_strip_top_dir_non_recursive(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_n_files_from_x_dir_to_x_dir_strip_top_dir(src_container_url, "Blob", dst_container_url, "Blob", False) + # Test oauth support for service to service copy, where source is authenticated with SAS + # and destination is authenticated with OAuth token. + @unittest.skip("covered by blob to blob") + def test_copy_single_17mb_file_from_file_to_blob_oauth(self): + src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) + dst_container_url = util.get_object_without_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) + self.util_test_copy_single_file_from_x_to_x(src_share_url, "File", dst_container_url, "Blob", 17 * 1024 * 1024, True) def test_copy_files_from_blob_account_to_blob_account(self): self.util_test_copy_files_from_x_account_to_x_account( - util.test_s2s_src_blob_account_url, - "Blob", - util.test_s2s_dst_blob_account_url, + util.test_s2s_src_blob_account_url, + "Blob", + util.test_s2s_dst_blob_account_url, "Blob", self.bucket_name) - def test_copy_file_from_blob_container_to_blob_container_propertyandmetadata(self): - src_container_url = util.get_object_sas(util.test_s2s_src_blob_account_url, self.bucket_name) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name) - self.util_test_copy_file_from_x_bucket_to_x_bucket_propertyandmetadata( - src_container_url, - "Blob", - dst_container_url, - "Blob") - ################################## # Test from blob to file copy # Note: tests go from dst blob to src file to avoid the extra config-- Ze's suggestion ################################## - def test_copy_single_1kb_file_from_blob_to_file(self): - src_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_blob_file) - dst_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_blob_file) - self.util_test_copy_single_file_from_x_to_x(src_container_url, "Blob", dst_share_url, "File", 1) - - def test_copy_10_files_from_blob_container_to_file_share(self): - src_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_blob_file) - dst_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_blob_file) - self.util_test_copy_n_files_from_x_bucket_to_x_bucket(src_container_url, "Blob", dst_share_url, "File", 10, 1) - - # not testing implicit container creation (w/out a container name in the dst) as that's tested by the FE tests - - ################################## - # Test from file to blob copy. - ################################## - def test_copy_10_files_from_file_share_to_blob_container(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_n_files_from_x_bucket_to_x_bucket(src_share_url, "File", dst_container_url, "Blob") - - def test_copy_file_from_file_share_to_blob_container_strip_top_dir_recursive(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_file_from_x_bucket_to_x_bucket_strip_top_dir(src_share_url, "File", dst_container_url, "Blob", True) - - def test_copy_file_from_file_share_to_blob_container_strip_top_dir_non_recursive(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_file_from_x_bucket_to_x_bucket_strip_top_dir(src_share_url, "File", dst_container_url, "Blob", False) - - def test_copy_n_files_from_file_dir_to_blob_dir(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_n_files_from_x_dir_to_x_dir(src_share_url, "File", dst_container_url, "Blob") - - def test_copy_n_files_from_file_dir_to_blob_dir_strip_top_dir_recursive(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_n_files_from_x_dir_to_x_dir_strip_top_dir(src_share_url, "File", dst_container_url, "Blob", True) - - def test_copy_n_files_from_file_dir_to_blob_dir_strip_top_dir_non_recursive(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_n_files_from_x_dir_to_x_dir_strip_top_dir(src_share_url, "File", dst_container_url, "Blob", False) def test_copy_files_from_file_account_to_blob_account(self): self.util_test_copy_files_from_x_account_to_x_account( - util.test_s2s_src_file_account_url, - "File", - util.test_s2s_dst_blob_account_url, + util.test_s2s_src_file_account_url, + "File", + util.test_s2s_dst_blob_account_url, "Blob", self.bucket_name_file_blob) - def test_copy_file_from_file_share_to_blob_container_propertyandmetadata(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_file_from_x_bucket_to_x_bucket_propertyandmetadata( - src_share_url, - "File", - dst_container_url, - "Blob") - - def test_copy_file_from_file_share_to_blob_container_no_preserve_propertyandmetadata(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_file_from_x_bucket_to_x_bucket_propertyandmetadata( - src_share_url, - "File", - dst_container_url, - "Blob", - False) - - # Test oauth support for service to service copy, where source is authenticated with SAS - # and destination is authenticated with OAuth token. - @unittest.skip("covered by blob to blob") - def test_copy_single_17mb_file_from_file_to_blob_oauth(self): - src_share_url = util.get_object_sas(util.test_s2s_src_file_account_url, self.bucket_name_file_blob) - dst_container_url = util.get_object_without_sas(util.test_s2s_dst_blob_account_url, self.bucket_name_file_blob) - self.util_test_copy_single_file_from_x_to_x(src_share_url, "File", dst_container_url, "Blob", 17 * 1024 * 1024, True) - ################################## # Test from S3 to blob copy. ################################## diff --git a/testSuite/scripts/test_upload_block_blob.py b/testSuite/scripts/test_upload_block_blob.py index c1dedadd3..f719b14b0 100644 --- a/testSuite/scripts/test_upload_block_blob.py +++ b/testSuite/scripts/test_upload_block_blob.py @@ -8,47 +8,6 @@ import utility as util class Block_Upload_User_Scenarios(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - - def util_test_1kb_blob_upload(self, use_oauth_session=False): - # Creating a single File Of size 1 KB - filename = "test1KB.txt" - file_path = util.create_test_file(filename, 1024) - - # executing the azcopy command to upload the 1KB file. - src = file_path - if not use_oauth_session: - dest = util.get_resource_sas(filename) - dest_validate = dest - else: - dest = util.get_resource_from_oauth_container(filename) - dest_validate = util.get_resource_from_oauth_container_validate(filename) - - result = util.Command("copy").add_arguments(src).add_arguments(dest). \ - add_flags("log-level", "info").add_flags("recursive", "true").execute_azcopy_copy_command() - self.assertTrue(result) - - # Verifying the uploaded blob. - # the resource local path should be the first argument for the azcopy validator. - # the resource sas should be the second argument for azcopy validator. - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest_validate).execute_azcopy_verify() - self.assertTrue(result) - - # test_1kb_blob_upload verifies the 1KB blob upload by azcopy. - def test_1kb_blob_upload_with_sas(self): - #Test the case with SAS - self.util_test_1kb_blob_upload() - - def test_1kb_blob_upload_with_oauth(self): - #Test the case with OAuth - self.util_test_1kb_blob_upload(True) - # test_63mb_blob_upload verifies the azcopy upload of 63mb blob upload. def test_63mb_blob_upload(self): # creating file of 63mb size. @@ -67,38 +26,6 @@ def test_63mb_blob_upload(self): result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest).execute_azcopy_verify() self.assertTrue(result) - # test_n_1kb_blob_upload verifies the upload of n 1kb blob to the container. - def util_test_n_1kb_blob_upload(self, number_of_files, use_oauth_session=False): - # create dir dir_n_files and 1 kb files inside the dir. - dir_name = "dir_" + str(number_of_files) + "_files" - dir_n_files_path = util.create_test_n_files(1024, number_of_files, dir_name) - - if not use_oauth_session: - dest = util.test_container_url - dest_validate = util.get_resource_sas(dir_name) - else: - dest = util.test_oauth_container_url - dest_validate = util.get_resource_from_oauth_container_validate(dir_name) - - # execute azcopy command - result = util.Command("copy").add_arguments(dir_n_files_path).add_arguments(dest). \ - add_flags("recursive", "true").add_flags("log-level", "info").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute the validator. - result = util.Command("testBlob").add_arguments(dir_n_files_path).add_arguments(dest_validate). \ - add_flags("is-object-dir", "true").execute_azcopy_verify() - self.assertTrue(result) - - - # test_10_1kb_blob_upload_with_sas verifies the upload of 10 1kb blob to the container with SAS. - def test_10_1kb_blob_upload_with_sas(self): - self.util_test_n_1kb_blob_upload(10) - - # test_10_1kb_blob_upload_with_sas verifies the upload of 10 1kb blob to the container with OAuth. - def test_10_1kb_blob_upload_with_oauth(self): - self.util_test_n_1kb_blob_upload(10, True) - # test_metaData_content_encoding_content_type verifies the meta data, content type, # content encoding of 2kb upload to container through azcopy. def test_blob_metadata_content_encoding_content_type(self): diff --git a/testSuite/scripts/test_upload_page_blob.py b/testSuite/scripts/test_upload_page_blob.py index b1efd47e4..18e26491a 100644 --- a/testSuite/scripts/test_upload_page_blob.py +++ b/testSuite/scripts/test_upload_page_blob.py @@ -4,48 +4,6 @@ class PageBlob_Upload_User_Scenarios(unittest.TestCase): - def setUp(self): - cmd = util.Command("login").add_arguments("--service-principal").add_flags("application-id", os.environ['ACTIVE_DIRECTORY_APPLICATION_ID']).add_flags("tenant-id", os.environ['OAUTH_TENANT_ID']) - cmd.execute_azcopy_copy_command() - - def tearDown(self): - cmd = util.Command("logout") - cmd.execute_azcopy_copy_command() - - # util_test_page_blob_upload_1mb verifies the azcopy upload of 1mb file - # as a page blob. - def util_test_page_blob_upload_1mb(self, use_oauth=False): - # create the test gile. - file_name = "test_page_blob_1mb.vhd" - file_path = util.create_test_file(file_name, 1024 * 1024) - - # execute azcopy upload. - if not use_oauth: - dest = util.get_resource_sas(file_name) - dest_validate = dest - else: - dest = util.get_resource_from_oauth_container(file_name) - dest_validate = util.get_resource_from_oauth_container_validate(file_name) - - result = util.Command("copy").add_arguments(file_path).add_arguments(dest).add_flags("log-level", "info"). \ - add_flags("block-size-mb", "4").add_flags("blob-type", "PageBlob").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute validator. - result = util.Command("testBlob").add_arguments(file_path).add_arguments(dest_validate). \ - add_flags("blob-type", "PageBlob").execute_azcopy_verify() - self.assertTrue(result) - - # test_page_blob_upload_1mb_with_sas verifies the azcopy upload of 1mb file - # as a page blob with sas. - def test_page_blob_upload_1mb_with_sas(self): - self.util_test_page_blob_upload_1mb(False) - - # test_page_blob_upload_1mb_with_oauth verifies the azcopy upload of 1mb file - # as a page blob with oauth. - def test_page_blob_upload_1mb_with_oauth(self): - self.util_test_page_blob_upload_1mb(True) - # test_page_range_for_complete_sparse_file verifies the number of Page ranges for # complete empty file i.e each character is Null character. def test_page_range_for_complete_sparse_file(self): @@ -139,110 +97,3 @@ def test_page_blob_upload_partial_sparse_file(self): result = util.Command("testBlob").add_arguments(download_dest).add_arguments(copy_destination_sas)\ .add_flags("blob-type", "PageBlob").execute_azcopy_verify() self.assertTrue(result) - - def test_set_page_blob_tier(self): - # test for P10 Page Blob Access Tier - filename = "test_page_P10_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P10").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P10" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P10").execute_azcopy_verify() - self.assertTrue(result) - - # test for P20 Page Blob Access Tier - filename = "test_page_P20_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P20").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P20" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P20").execute_azcopy_verify() - self.assertTrue(result) - - # test for P30 Page Blob Access Tier - filename = "test_page_P30_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P30").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P30" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P30").execute_azcopy_verify() - self.assertTrue(result) - - # test for P4 Page Blob Access Tier - filename = "test_page_P4_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P4").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P4" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P4").execute_azcopy_verify() - self.assertTrue(result) - - # test for P40 Page Blob Access Tier - filename = "test_page_P40_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P40").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P40" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P40").execute_azcopy_verify() - self.assertTrue(result) - - # test for P50 Page Blob Access Tier - filename = "test_page_P50_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P50").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P50" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P50").execute_azcopy_verify() - self.assertTrue(result) - - # test for P6 Page Blob Access Tier - filename = "test_page_P6_blob_tier.vhd" - file_path = util.create_test_file(filename, 100 * 1024) - destination_sas = util.get_resource_sas_from_premium_container_sas(filename) - result = util.Command("copy").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("log-level", "info").add_flags("blob-type", "PageBlob").add_flags("page-blob-tier", - "P6").execute_azcopy_copy_command() - self.assertTrue(result) - - # execute azcopy validate order. - # added the expected blob-tier "P50" - result = util.Command("testBlob").add_arguments(file_path).add_arguments(destination_sas). \ - add_flags("blob-type", "PageBlob").add_flags("blob-tier", "P6").execute_azcopy_verify() - self.assertTrue(result)