diff --git a/cmd/copy.go b/cmd/copy.go index aeabba8e4..ffe19ce90 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1349,7 +1349,7 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) // Use the concurrency environment value - concurrencyEnvVar := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) + concurrencyEnvVar := common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) pipingUploadParallelism := pipingUploadParallelism if concurrencyEnvVar != "" { @@ -1485,7 +1485,7 @@ func (cca *CookedCopyCmdArgs) getSrcCredential(ctx context.Context, jpo *common. func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) // Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value - if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) && glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" { + if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) && common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" { jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto() } @@ -1971,7 +1971,7 @@ func getPerfDisplayText(perfDiagnosticStrings []string, constraint common.PerfCo } func shouldDisplayPerfStates() bool { - return glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != "" + return common.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != "" } func isStdinPipeIn() (bool, error) { diff --git a/cmd/credentialUtil.go b/cmd/credentialUtil.go index 2137a2dd2..e3bda9d3c 100644 --- a/cmd/credentialUtil.go +++ b/cmd/credentialUtil.go @@ -89,17 +89,17 @@ func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) { var err error autoOAuth.Do(func() { var lca loginCmdArgs - autoLoginType := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType())) + autoLoginType := strings.ToLower(common.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType())) if autoLoginType == "" { glcm.Info("Autologin not specified.") return } - if tenantID := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.TenantID()); tenantID != "" { + if tenantID := common.GetEnvironmentVariable(common.EEnvironmentVariable.TenantID()); tenantID != "" { lca.tenantID = tenantID } - if endpoint := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AADEndpoint()); endpoint != "" { + if endpoint := common.GetEnvironmentVariable(common.EEnvironmentVariable.AADEndpoint()); endpoint != "" { lca.aadEndpoint = endpoint } @@ -107,14 +107,14 @@ func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) { lca.loginType = autoLoginType switch autoLoginType { case common.EAutoLoginType.SPN().String(): - lca.applicationID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ApplicationID()) - lca.certPath = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePath()) - lca.certPass = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword()) - lca.clientSecret = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret()) + lca.applicationID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ApplicationID()) + lca.certPath = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePath()) + lca.certPass = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword()) + lca.clientSecret = common.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret()) case common.EAutoLoginType.MSI().String(): - lca.identityClientID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityClientID()) - lca.identityObjectID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityObjectID()) - lca.identityResourceID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityResourceString()) + lca.identityClientID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityClientID()) + lca.identityObjectID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityObjectID()) + lca.identityResourceID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityResourceString()) case common.EAutoLoginType.Device().String(): case common.EAutoLoginType.AzCLI().String(): case common.EAutoLoginType.PsCred().String(): @@ -174,7 +174,7 @@ var stashedEnvCredType = "" func GetCredTypeFromEnvVar() common.CredentialType { rawVal := stashedEnvCredType if stashedEnvCredType == "" { - rawVal = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CredentialType()) + rawVal = common.GetEnvironmentVariable(common.EEnvironmentVariable.CredentialType()) if rawVal == "" { return common.ECredentialType.Unknown() } @@ -183,7 +183,7 @@ func GetCredTypeFromEnvVar() common.CredentialType { // Remove the env var after successfully fetching once, // in case of env var is further spreading into child processes unexpectedly. - glcm.ClearEnvironmentVariable(common.EEnvironmentVariable.CredentialType()) + common.ClearEnvironmentVariable(common.EEnvironmentVariable.CredentialType()) // Try to get the value set. var credType common.CredentialType @@ -364,7 +364,7 @@ func isPublic(ctx context.Context, blobResourceURL string, cpkOptions common.Cpk RetryDelay: ste.UploadRetryDelay, MaxRetryDelay: ste.UploadMaxRetryDelay, }, policy.TelemetryOptions{ - ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent), + ApplicationID: common.AddUserAgentPrefix(common.UserAgent), }, nil, ste.LogOptions{}, nil) blobClient, _ := blob.NewClientWithNoCredential(bURLParts.String(), &blob.ClientOptions{ClientOptions: clientOptions}) @@ -397,7 +397,7 @@ func mdAccountNeedsOAuth(ctx context.Context, blobResourceURL string, cpkOptions RetryDelay: ste.UploadRetryDelay, MaxRetryDelay: ste.UploadMaxRetryDelay, }, policy.TelemetryOptions{ - ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent), + ApplicationID: common.AddUserAgentPrefix(common.UserAgent), }, nil, ste.LogOptions{}, nil) blobClient, _ := blob.NewClientWithNoCredential(blobResourceURL, &blob.ClientOptions{ClientOptions: clientOptions}) @@ -454,8 +454,8 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio } if location == common.ELocation.S3() { - accessKeyID := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID()) - secretAccessKey := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey()) + accessKeyID := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID()) + secretAccessKey := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey()) if accessKeyID == "" || secretAccessKey == "" { credType = common.ECredentialType.S3PublicBucket() public = true @@ -467,7 +467,7 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio } if location == common.ELocation.GCP() { - googleAppCredentials := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials()) + googleAppCredentials := common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials()) if googleAppCredentials == "" { return common.ECredentialType.Unknown(), false, errors.New("GOOGLE_APPLICATION_CREDENTIALS environment variable must be set before using GCP transfer feature") } @@ -508,8 +508,8 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio // BlobFS currently supports Shared key. Remove this piece of code, once // we deprecate that. if location == common.ELocation.BlobFS() { - name := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AccountName()) - key := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AccountKey()) + name := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountName()) + key := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountKey()) if name != "" && key != "" { // TODO: To remove, use for internal testing, SharedKey should not be supported from commandline credType = common.ECredentialType.SharedKey() warnIfSharedKeyAuthForDatalake() @@ -591,7 +591,7 @@ func createClientOptions(logger common.ILoggerResetable, srcCred *common.ScopedC RetryDelay: ste.UploadRetryDelay, MaxRetryDelay: ste.UploadMaxRetryDelay, }, policy.TelemetryOptions{ - ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent), + ApplicationID: common.AddUserAgentPrefix(common.UserAgent), }, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), logOptions, srcCred) } diff --git a/cmd/env.go b/cmd/env.go index 87784040b..fbac937d8 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -29,7 +29,7 @@ var envCmd = &cobra.Command{ Long: envCmdLongDescription, Run: func(cmd *cobra.Command, args []string) { for _, env := range common.VisibleEnvironmentVariables { - val := glcm.GetEnvironmentVariable(env) + val := common.GetEnvironmentVariable(env) if env.Hidden && !showSensitive { val = "REDACTED" } diff --git a/cmd/login.go b/cmd/login.go index 2d27c87e9..c40cd9379 100644 --- a/cmd/login.go +++ b/cmd/login.go @@ -40,8 +40,8 @@ var lgCmd = &cobra.Command{ return nil }, RunE: func(cmd *cobra.Command, args []string) error { - loginCmdArg.certPass = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword()) - loginCmdArg.clientSecret = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret()) + loginCmdArg.certPass = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword()) + loginCmdArg.clientSecret = common.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret()) loginCmdArg.persistToken = true if loginCmdArg.certPass != "" || loginCmdArg.clientSecret != "" { diff --git a/cmd/root.go b/cmd/root.go index 2b53212c1..f6ddf790a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -81,8 +81,9 @@ var rootCmd = &cobra.Command{ Short: rootCmdShortDescription, Long: rootCmdLongDescription, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if glcm.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout()) != "" { - timeout, err := time.ParseDuration(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout()) + "m") + requestTryTimeout := common.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout()) + if requestTryTimeout != "" { + timeout, err := time.ParseDuration(requestTryTimeout + "m") if err == nil { ste.UploadTryTimeout = timeout } diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 4cfeb629e..9a1fd9689 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -606,7 +606,7 @@ func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context. isDFS: isDFS, } - disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning())) + disableHierarchicalScanning := strings.ToLower(common.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning())) // disableHierarchicalScanning should be true for permanent delete if (disableHierarchicalScanning == "false" || disableHierarchicalScanning == "") && includeDeleted && (includeSnapshot || includeVersion) { diff --git a/cmd/zc_traverser_gcp_service.go b/cmd/zc_traverser_gcp_service.go index 500595c80..82d43fa94 100644 --- a/cmd/zc_traverser_gcp_service.go +++ b/cmd/zc_traverser_gcp_service.go @@ -93,7 +93,7 @@ func (t *gcpServiceTraverser) Traverse(preprocessor objectMorpher, processor obj } func newGCPServiceTraverser(rawURL *url.URL, ctx context.Context, getProperties bool, incrementEnumerationCounter enumerationCounterFunc) (*gcpServiceTraverser, error) { - projectID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleCloudProject()) + projectID = common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleCloudProject()) t := &gcpServiceTraverser{ ctx: ctx, incrementEnumerationCounter: incrementEnumerationCounter, diff --git a/cmd/zt_interceptors_for_test.go b/cmd/zt_interceptors_for_test.go index 68f7cd0b7..e15627484 100644 --- a/cmd/zt_interceptors_for_test.go +++ b/cmd/zt_interceptors_for_test.go @@ -22,8 +22,6 @@ package cmd import ( "fmt" - "os" - "github.com/Azure/azure-storage-azcopy/v10/common" ) @@ -83,10 +81,6 @@ type mockedLifecycleManager struct { outputFormat common.OutputFormat } -func (m *mockedLifecycleManager) DownloadToTempPath() bool { - return false -} - func (m *mockedLifecycleManager) ReportAllJobPartsDone() { } @@ -143,24 +137,11 @@ func (*mockedLifecycleManager) SurrenderControl() func (*mockedLifecycleManager) RegisterCloseFunc(func()) {} func (mockedLifecycleManager) AllowReinitiateProgressReporting() {} func (*mockedLifecycleManager) InitiateProgressReporting(common.WorkController) {} -func (*mockedLifecycleManager) ClearEnvironmentVariable(env common.EnvironmentVariable) { - _ = os.Setenv(env.Name, "") -} -func (*mockedLifecycleManager) GetEnvironmentVariable(env common.EnvironmentVariable) string { - value := os.Getenv(env.Name) - if value == "" { - return env.DefaultValue - } - return value -} func (m *mockedLifecycleManager) SetOutputFormat(format common.OutputFormat) { m.outputFormat = format } func (*mockedLifecycleManager) EnableInputWatcher() {} func (*mockedLifecycleManager) EnableCancelFromStdIn() {} -func (*mockedLifecycleManager) AddUserAgentPrefix(userAgent string) string { - return userAgent -} func (*mockedLifecycleManager) SetForceLogging() {} diff --git a/common/ProxyLookupCache.go b/common/ProxyLookupCache.go index 258685bfd..52e7c772a 100644 --- a/common/ProxyLookupCache.go +++ b/common/ProxyLookupCache.go @@ -43,7 +43,7 @@ func init() { lookupMethod: GetProxyFunc(), } - ev := GetLifecycleMgr().GetEnvironmentVariable(EEnvironmentVariable.CacheProxyLookup()) + ev := GetEnvironmentVariable(EEnvironmentVariable.CacheProxyLookup()) if strings.ToLower(ev) == "true" { GlobalProxyLookup = c.getProxy } else { diff --git a/common/clientFactory.go b/common/clientFactory.go index b45a57ae7..b32c651ac 100644 --- a/common/clientFactory.go +++ b/common/clientFactory.go @@ -27,8 +27,8 @@ import ( ) func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error) { - name := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountName()) - key := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountKey()) + name := GetEnvironmentVariable(EEnvironmentVariable.AccountName()) + key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey()) // If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables if name == "" || key == "" { return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential") @@ -37,8 +37,8 @@ func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error) { } func GetBlobSharedKeyCredential() (*blob.SharedKeyCredential, error) { - name := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountName()) - key := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountKey()) + name := GetEnvironmentVariable(EEnvironmentVariable.AccountName()) + key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey()) // If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables if name == "" || key == "" { return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential") diff --git a/common/credentialFactory.go b/common/credentialFactory.go index f657bd86f..5831d30dd 100644 --- a/common/credentialFactory.go +++ b/common/credentialFactory.go @@ -63,14 +63,13 @@ func (o CredentialOpOptions) panicError(err error) { // CreateS3Credential creates AWS S3 credential according to credential info. func CreateS3Credential(ctx context.Context, credInfo CredentialInfo, options CredentialOpOptions) (*credentials.Credentials, error) { - glcm := GetLifecycleMgr() switch credInfo.CredentialType { case ECredentialType.S3PublicBucket(): return credentials.NewStatic("", "", "", credentials.SignatureAnonymous), nil case ECredentialType.S3AccessKey(): - accessKeyID := glcm.GetEnvironmentVariable(EEnvironmentVariable.AWSAccessKeyID()) - secretAccessKey := glcm.GetEnvironmentVariable(EEnvironmentVariable.AWSSecretAccessKey()) - sessionToken := glcm.GetEnvironmentVariable(EEnvironmentVariable.AwsSessionToken()) + accessKeyID := GetEnvironmentVariable(EEnvironmentVariable.AWSAccessKeyID()) + secretAccessKey := GetEnvironmentVariable(EEnvironmentVariable.AWSSecretAccessKey()) + sessionToken := GetEnvironmentVariable(EEnvironmentVariable.AwsSessionToken()) // create and return s3 credential return credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), nil // S3 uses V4 signature @@ -185,11 +184,11 @@ func GetCpkInfo(cpkInfo bool) *blob.CPKInfo { } // fetch EncryptionKey and EncryptionKeySHA256 from the environment variables - glcm := GetLifecycleMgr() - encryptionKey := glcm.GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKey()) - encryptionKeySHA256 := glcm.GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKeySHA256()) + encryptionKey := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKey()) + encryptionKeySHA256 := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKeySHA256()) encryptionAlgorithmAES256 := blob.EncryptionAlgorithmTypeAES256 + glcm := GetLifecycleMgr() if encryptionKey == "" || encryptionKeySHA256 == "" { glcm.Error("fatal: failed to fetch cpk encryption key (" + EEnvironmentVariable.CPKEncryptionKey().Name + ") or hash (" + EEnvironmentVariable.CPKEncryptionKeySHA256().Name + ") from environment variables") diff --git a/common/environment.go b/common/environment.go index 5a4b7f3d0..088088890 100644 --- a/common/environment.go +++ b/common/environment.go @@ -23,6 +23,7 @@ package common import ( "encoding/json" "fmt" + "os" "reflect" "runtime" "strings" @@ -37,6 +38,20 @@ type EnvironmentVariable struct { Hidden bool } +// GetEnvironmentVariable gets the environment variable or its default value +func GetEnvironmentVariable(env EnvironmentVariable) string { + value := os.Getenv(env.Name) + if value == "" { + return env.DefaultValue + } + return value +} + +// ClearEnvironmentVariable clears the environment variable +func ClearEnvironmentVariable(variable EnvironmentVariable) { + _ = os.Setenv(variable.Name, "") +} + // This array needs to be updated when a new public environment variable is added // Things are here, rather than in command line parameters for one of two reasons: // 1. They are optional and obscure (e.g. performance tuning parameters) or diff --git a/common/lifecyleMgr.go b/common/lifecyleMgr.go index 3537ac789..4aa6c419a 100644 --- a/common/lifecyleMgr.go +++ b/common/lifecyleMgr.go @@ -61,19 +61,15 @@ type LifecycleMgr interface { SurrenderControl() // give up control, this should never return InitiateProgressReporting(WorkController) // start writing progress with another routine AllowReinitiateProgressReporting() // allow re-initiation of progress reporting for followup job - GetEnvironmentVariable(EnvironmentVariable) string // get the environment variable or its default value - ClearEnvironmentVariable(EnvironmentVariable) // clears the environment variable SetOutputFormat(OutputFormat) // change the output format of the entire application EnableInputWatcher() // depending on the command, we may allow user to give input through Stdin EnableCancelFromStdIn() // allow user to send in `cancel` to stop the job - AddUserAgentPrefix(string) string // append the global user agent prefix, if applicable E2EAwaitContinue() // used by E2E tests E2EAwaitAllowOpenFiles() // used by E2E tests E2EEnableAwaitAllowOpenFiles(enable bool) // used by E2E tests RegisterCloseFunc(func()) SetForceLogging() IsForceLoggingDisabled() bool - DownloadToTempPath() bool MsgHandlerChannel() <-chan *LCMMsg ReportAllJobPartsDone() SetOutputVerbosity(mode OutputVerbosity) @@ -206,10 +202,6 @@ func (lcm *lifecycleMgr) EnableCancelFromStdIn() { lcm.allowCancelFromStdIn = true } -func (lcm *lifecycleMgr) ClearEnvironmentVariable(variable EnvironmentVariable) { - _ = os.Setenv(variable.Name, "") -} - func (lcm *lifecycleMgr) SetOutputFormat(format OutputFormat) { lcm.outputFormat = format } @@ -219,7 +211,7 @@ func (lcm *lifecycleMgr) checkAndStartCPUProfiling() { // the value AZCOPY_PROFILE_CPU indicates the path to save CPU profiling data. // e.g. export AZCOPY_PROFILE_CPU="cpu.prof" // For more details, please refer to https://golang.org/pkg/runtime/pprof/ - cpuProfilePath := lcm.GetEnvironmentVariable(EEnvironmentVariable.ProfileCPU()) + cpuProfilePath := GetEnvironmentVariable(EEnvironmentVariable.ProfileCPU()) if cpuProfilePath != "" { lcm.Info(fmt.Sprintf("pprof start CPU profiling, and saving profiling data to: %q", cpuProfilePath)) f, err := os.Create(cpuProfilePath) @@ -242,7 +234,7 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() { // the value AZCOPY_PROFILE_MEM indicates the path to save memory profiling data. // e.g. export AZCOPY_PROFILE_MEM="mem.prof" // For more details, please refer to https://golang.org/pkg/runtime/pprof/ - memProfilePath := lcm.GetEnvironmentVariable(EEnvironmentVariable.ProfileMemory()) + memProfilePath := GetEnvironmentVariable(EEnvironmentVariable.ProfileMemory()) if memProfilePath != "" { lcm.Info(fmt.Sprintf("pprof start memory profiling, and saving profiling data to: %q", memProfilePath)) f, err := os.Create(memProfilePath) @@ -634,23 +626,6 @@ func (lcm *lifecycleMgr) InitiateProgressReporting(jc WorkController) { }() } -func (lcm *lifecycleMgr) GetEnvironmentVariable(env EnvironmentVariable) string { - value := os.Getenv(env.Name) - if value == "" { - return env.DefaultValue - } - return value -} - -func (lcm *lifecycleMgr) AddUserAgentPrefix(userAgent string) string { - prefix := lcm.GetEnvironmentVariable(EEnvironmentVariable.UserAgentPrefix()) - if len(prefix) > 0 { - userAgent = prefix + " " + userAgent - } - - return userAgent -} - func (_ *lifecycleMgr) awaitChannel(ch chan struct{}, timeout time.Duration) { select { case <-ch: @@ -685,7 +660,7 @@ func (lcm *lifecycleMgr) E2EEnableAwaitAllowOpenFiles(enable bool) { // Fetching `AZCOPY_DISABLE_SYSLOG` from the environment variables and // setting `disableSyslog` flag in LifeCycleManager to avoid Env Vars Lookup redundantly func (lcm *lifecycleMgr) SetForceLogging() { - disableSyslog, err := strconv.ParseBool(lcm.GetEnvironmentVariable(EEnvironmentVariable.DisableSyslog())) + disableSyslog, err := strconv.ParseBool(GetEnvironmentVariable(EEnvironmentVariable.DisableSyslog())) if err != nil { // By default, we'll retain the current behaviour. i.e. To log in Syslog/WindowsEventLog if not specified by the user disableSyslog = false @@ -697,15 +672,6 @@ func (lcm *lifecycleMgr) IsForceLoggingDisabled() bool { return lcm.disableSyslog } -func (lcm *lifecycleMgr) DownloadToTempPath() bool { - ret, err := strconv.ParseBool(lcm.GetEnvironmentVariable(EEnvironmentVariable.DownloadToTempPath())) - if err != nil { - // By default we'll download to temp path - ret = true - } - return ret -} - func (lcm *lifecycleMgr) MsgHandlerChannel() <-chan *LCMMsg { return lcm.msgHandlerChannel } diff --git a/common/oauthTokenManager.go b/common/oauthTokenManager.go index bd1b30181..385993456 100644 --- a/common/oauthTokenManager.go +++ b/common/oauthTokenManager.go @@ -324,7 +324,7 @@ var stashedEnvOAuthTokenExists = false // Note: This is useful for only checking whether the env var exists, please use getTokenInfoFromEnvVar // directly in the case getting token info is necessary. func EnvVarOAuthTokenInfoExists() bool { - if lcm.GetEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) == "" && !stashedEnvOAuthTokenExists { + if GetEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) == "" && !stashedEnvOAuthTokenExists { return false } stashedEnvOAuthTokenExists = true @@ -341,14 +341,14 @@ func IsErrorEnvVarOAuthTokenInfoNotSet(err error) bool { // getTokenInfoFromEnvVar gets token info from environment variable. func (uotm *UserOAuthTokenManager) getTokenInfoFromEnvVar(ctx context.Context) (*OAuthTokenInfo, error) { - rawToken := lcm.GetEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) + rawToken := GetEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) if rawToken == "" { return nil, errors.New(ErrorCodeEnvVarOAuthTokenInfoNotSet) } // Remove the env var after successfully fetching once, // in case of env var is further spreading into child processes unexpectedly. - lcm.ClearEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) + ClearEnvironmentVariable(EEnvironmentVariable.OAuthTokenInfo()) tokenInfo, err := jsonToTokenInfo([]byte(rawToken)) if err != nil { diff --git a/common/version.go b/common/version.go index cd360150f..f7314d9a8 100644 --- a/common/version.go +++ b/common/version.go @@ -5,3 +5,13 @@ const UserAgent = "AzCopy/" + AzcopyVersion const S3ImportUserAgent = "S3Import " + UserAgent const GCPImportUserAgent = "GCPImport " + UserAgent const BenchmarkUserAgent = "Benchmark " + UserAgent + +// AddUserAgentPrefix appends the global user agent prefix, if applicable +func AddUserAgentPrefix(userAgent string) string { + prefix := GetEnvironmentVariable(EEnvironmentVariable.UserAgentPrefix()) + if len(prefix) > 0 { + userAgent = prefix + " " + userAgent + } + + return userAgent +} diff --git a/jobsAdmin/JobsAdmin.go b/jobsAdmin/JobsAdmin.go index 20201b424..8ff6df337 100755 --- a/jobsAdmin/JobsAdmin.go +++ b/jobsAdmin/JobsAdmin.go @@ -170,7 +170,7 @@ func getMaxRamForChunks() int64 { // return the user-specified override value, if any envVar := common.EEnvironmentVariable.BufferGB() - overrideString := common.GetLifecycleMgr().GetEnvironmentVariable(envVar) + overrideString := common.GetEnvironmentVariable(envVar) if overrideString != "" { overrideValue, err := strconv.ParseFloat(overrideString, 64) if err != nil { diff --git a/jobsAdmin/init.go b/jobsAdmin/init.go index 9ade0a4b7..a5da943b8 100755 --- a/jobsAdmin/init.go +++ b/jobsAdmin/init.go @@ -60,7 +60,7 @@ func MainSTE(concurrency ste.ConcurrencySettings, targetRateInMegaBitsPerSec flo // TODO: We may want to list listen first and terminate if there is already an instance listening // if we've a custom mime map - if path := common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.MimeMapping()); path != "" { + if path := common.GetEnvironmentVariable(common.EEnvironmentVariable.MimeMapping()); path != "" { data, err := os.ReadFile(path) if err != nil { return err diff --git a/main.go b/main.go index 7dd6410b7..e2aa8f825 100644 --- a/main.go +++ b/main.go @@ -33,8 +33,8 @@ import ( var glcm = common.GetLifecycleMgr() func main() { - azcopyLogPathFolder := common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.LogLocation()) // user specified location for log files - azcopyJobPlanFolder := common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.JobPlanLocation()) // user specified location for plan files + azcopyLogPathFolder := common.GetEnvironmentVariable(common.EEnvironmentVariable.LogLocation()) // user specified location for log files + azcopyJobPlanFolder := common.GetEnvironmentVariable(common.EEnvironmentVariable.JobPlanLocation()) // user specified location for plan files // note: azcopyAppPathFolder is the default location for all AzCopy data (logs, job plans, oauth token on Windows) // but all the above can be put elsewhere as they can become very large diff --git a/main_unix.go b/main_unix.go index a83400df3..096560299 100644 --- a/main_unix.go +++ b/main_unix.go @@ -71,8 +71,7 @@ func ProcessOSSpecificInitialization() (int, error) { // GetAzCopyAppPath returns the path of Azcopy folder in local appdata. // Azcopy folder in local appdata contains all the files created by azcopy locally. func GetAzCopyAppPath() string { - lcm := common.GetLifecycleMgr() - localAppData := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) + localAppData := common.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) azcopyAppDataFolder := path.Join(localAppData, ".azcopy") return azcopyAppDataFolder } diff --git a/main_windows.go b/main_windows.go index c3ca18496..81ef490bf 100644 --- a/main_windows.go +++ b/main_windows.go @@ -45,8 +45,7 @@ func ProcessOSSpecificInitialization() (int, error) { // GetAzCopyAppPath returns the path of Azcopy in local appdata. func GetAzCopyAppPath() string { - lcm := common.GetLifecycleMgr() - userProfile := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) + userProfile := common.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) azcopyAppDataFolder := strings.ReplaceAll(path.Join(userProfile, ".azcopy"), "/", `\`) return azcopyAppDataFolder diff --git a/ste/concurrency.go b/ste/concurrency.go index 6ab9e27d1..cfb41af91 100644 --- a/ste/concurrency.go +++ b/ste/concurrency.go @@ -47,7 +47,7 @@ func (i *ConfiguredInt) GetDescription() string { // tryNewConfiguredInt populates a ConfiguredInt from an environment variable, or returns nil if env var is not set func tryNewConfiguredInt(envVar common.EnvironmentVariable) *ConfiguredInt { - override := common.GetLifecycleMgr().GetEnvironmentVariable(envVar) + override := common.GetEnvironmentVariable(envVar) if override != "" { val, err := strconv.ParseInt(override, 10, 32) if err != nil { @@ -77,7 +77,7 @@ func (b *ConfiguredBool) GetDescription() string { // tryNewConfiguredBool populates a ConfiguredInt from an environment variable, or returns nil if env var is not set func tryNewConfiguredBool(envVar common.EnvironmentVariable) *ConfiguredBool { - override := common.GetLifecycleMgr().GetEnvironmentVariable(envVar) + override := common.GetEnvironmentVariable(envVar) if override != "" { val, err := strconv.ParseBool(override) if err != nil { @@ -177,7 +177,7 @@ func getMainPoolSize(numOfCPUs int, requestAutoTune bool) (initial int, max *Con envVar := common.EEnvironmentVariable.ConcurrencyValue() - if common.GetLifecycleMgr().GetEnvironmentVariable(envVar) == "AUTO" { + if common.GetEnvironmentVariable(envVar) == "AUTO" { // Allow user to force auto-tuning from the env var, even when not in benchmark mode // Might be handy in some S2S cases, where we know that release 10.2.1 was using too few goroutines // This feature will probably remain undocumented for at least one release cycle, while we consider diff --git a/ste/pacer-autoPacer.go b/ste/pacer-autoPacer.go index 4ed9a39e2..d0c990963 100644 --- a/ste/pacer-autoPacer.go +++ b/ste/pacer-autoPacer.go @@ -78,7 +78,7 @@ var ( func newPageBlobAutoPacer(bytesPerSecond int64, expectedBytesPerRequest int64, isFair bool, logger common.ILogger) autopacer { shouldPaceOncer.Do(func() { - raw := common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.PacePageBlobs()) + raw := common.GetEnvironmentVariable(common.EEnvironmentVariable.PacePageBlobs()) shouldPacePageBlobs = strings.ToLower(raw) != "false" }) diff --git a/ste/pageRangeOptimizer.go b/ste/pageRangeOptimizer.go index 9a9bad167..a8cc96a0a 100644 --- a/ste/pageRangeOptimizer.go +++ b/ste/pageRangeOptimizer.go @@ -51,7 +51,7 @@ func withNoRetryForBlob(ctx context.Context) context.Context { func (p *pageRangeOptimizer) fetchPages() { // don't fetch page blob list if optimizations are not desired, // the lack of page list indicates that there's data everywhere - if !strings.EqualFold(common.GetLifecycleMgr().GetEnvironmentVariable( + if !strings.EqualFold(common.GetEnvironmentVariable( common.EEnvironmentVariable.OptimizeSparsePageBlobTransfers()), "true") { return } diff --git a/ste/sender-blockBlob.go b/ste/sender-blockBlob.go index bd1364be9..9f2f1a476 100644 --- a/ste/sender-blockBlob.go +++ b/ste/sender-blockBlob.go @@ -384,7 +384,7 @@ func (s *blockBlobSenderBase) buildCommittedBlockMap() { changedChunkSize := "buildCommittedBlockMap: Chunksize mismatch on uncommitted blocks" list := make(map[int]string) - if common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.DisableBlobTransferResume()) == "true" { + if common.GetEnvironmentVariable(common.EEnvironmentVariable.DisableBlobTransferResume()) == "true" { return } diff --git a/ste/sourceInfoProvider-GCP.go b/ste/sourceInfoProvider-GCP.go index 9f49bed12..595537fb3 100644 --- a/ste/sourceInfoProvider-GCP.go +++ b/ste/sourceInfoProvider-GCP.go @@ -59,8 +59,7 @@ func newGCPSourceInfoProvider(jptm IJobPartTransferMgr) (ISourceInfoProvider, er if err != nil { return nil, err } - glcm := common.GetLifecycleMgr() - jsonKey, err = os.ReadFile(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials())) + jsonKey, err = os.ReadFile(common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials())) if err != nil { return nil, fmt.Errorf("Cannot read JSON key file. Please verify you have correctly set GOOGLE_APPLICATION_CREDENTIALS environment variable") } diff --git a/ste/testJobPartTransferManager_test.go b/ste/testJobPartTransferManager_test.go index 55d8fb32d..4255a6444 100644 --- a/ste/testJobPartTransferManager_test.go +++ b/ste/testJobPartTransferManager_test.go @@ -286,7 +286,7 @@ func (t *testJobPartTransferManager) S2SSourceClientOptions() azcore.ClientOptio } else if t.fromTo.From() == common.ELocation.Benchmark() || t.fromTo.To() == common.ELocation.Benchmark() { userAgent = common.BenchmarkUserAgent } else { - userAgent = common.GetLifecycleMgr().AddUserAgentPrefix(common.UserAgent) + userAgent = common.AddUserAgentPrefix(common.UserAgent) } telemetryOptions := policy.TelemetryOptions{ApplicationID: userAgent} diff --git a/ste/xfer-remoteToLocal-file.go b/ste/xfer-remoteToLocal-file.go index dfb791945..ca7e9e512 100644 --- a/ste/xfer-remoteToLocal-file.go +++ b/ste/xfer-remoteToLocal-file.go @@ -27,6 +27,7 @@ import ( "io" "os" "path/filepath" + "strconv" "strings" "github.com/Azure/azure-storage-azcopy/v10/common" @@ -569,7 +570,13 @@ func tryDeleteFile(info *TransferInfo, jptm IJobPartTransferMgr) { // download to a temp path we return a temp path in format // /actual/parent/path/.azDownload-- func (info *TransferInfo) getDownloadPath() string { - if common.GetLifecycleMgr().DownloadToTempPath() && info.SourceSize > 0 { // 0-byte files don't need a rename. + downloadToTempPath, err := strconv.ParseBool(common.GetEnvironmentVariable(common.EEnvironmentVariable.DownloadToTempPath())) + if err != nil { + // By default, we'll download to temp path + downloadToTempPath = true + } + + if downloadToTempPath && info.SourceSize > 0 { // 0-byte files don't need a rename. parent, fileName := filepath.Split(info.Destination) fileName = fmt.Sprintf(azcopyTempDownloadPrefix, info.JobID.String()) + fileName return filepath.Join(parent, fileName) diff --git a/ste/xferVersionPolicy.go b/ste/xferVersionPolicy.go index e966c9403..b80d1ddfc 100644 --- a/ste/xferVersionPolicy.go +++ b/ste/xferVersionPolicy.go @@ -32,7 +32,7 @@ type serviceAPIVersionOverride struct{} var ServiceAPIVersionOverride = serviceAPIVersionOverride{} // DefaultServiceApiVersion is the default value of service api version that is set as value to the ServiceAPIVersionOverride in every Job's context. -var DefaultServiceApiVersion = common.GetLifecycleMgr().GetEnvironmentVariable(common.EEnvironmentVariable.DefaultServiceApiVersion()) +var DefaultServiceApiVersion = common.GetEnvironmentVariable(common.EEnvironmentVariable.DefaultServiceApiVersion()) type versionPolicy struct { } diff --git a/testSuite/cmd/common.go b/testSuite/cmd/common.go index dd8617e44..3552477f0 100644 --- a/testSuite/cmd/common.go +++ b/testSuite/cmd/common.go @@ -28,9 +28,8 @@ type createS3ResOptions struct { } func createS3ClientWithMinio(o createS3ResOptions) *minio.Client { - lcm := common.GetLifecycleMgr() - accessKeyID := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID()) - secretAccessKey := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey()) + accessKeyID := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID()) + secretAccessKey := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey()) if accessKeyID == "" || secretAccessKey == "" { fmt.Println("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY should be set before creating the S3 client") diff --git a/testSuite/cmd/platdiff_windows.go b/testSuite/cmd/platdiff_windows.go index e6aa95427..4959b8110 100644 --- a/testSuite/cmd/platdiff_windows.go +++ b/testSuite/cmd/platdiff_windows.go @@ -9,8 +9,7 @@ import ( // GetAzCopyAppPath returns the path of Azcopy in local appdata. func GetAzCopyAppPath() string { - lcm := common.GetLifecycleMgr() - userProfile := lcm.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) + userProfile := common.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir()) azcopyAppDataFolder := path.Join(userProfile, ".azcopy") if err := os.Mkdir(azcopyAppDataFolder, os.ModeDir); err != nil && !os.IsExist(err) { return ""