Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Clean up Lifecycle Manager #2897

Merged
merged 4 commits into from
Jan 15, 2025
Merged

Clean up Lifecycle Manager #2897

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
96c0ee4
Clean up GetEnvVar and ClearEnvVar
gapra-msft Dec 19, 2024
f4858c1
Clean up DownloadToTempPath
gapra-msft Dec 19, 2024
162751c
Clean up AddUserAgentPrefix
gapra-msft Dec 19, 2024
9a880ab
Merge branch 'main' into gapra/cleanUpLcm
wonwuakpa-msft Jan 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cmd/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1349,7 +1349,7 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)

// Use the concurrency environment value
concurrencyEnvVar := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue())
concurrencyEnvVar := common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue())

pipingUploadParallelism := pipingUploadParallelism
if concurrencyEnvVar != "" {
Expand Down Expand Up @@ -1485,7 +1485,7 @@ func (cca *CookedCopyCmdArgs) getSrcCredential(ctx context.Context, jpo *common.
func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion)
// Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value
if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) && glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" {
if jobsAdmin.JobsAdmin != nil && (cca.FromTo.From() == common.ELocation.File() || cca.FromTo.To() == common.ELocation.File()) && common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" {
jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
}

Expand Down Expand Up @@ -1971,7 +1971,7 @@ func getPerfDisplayText(perfDiagnosticStrings []string, constraint common.PerfCo
}

func shouldDisplayPerfStates() bool {
return glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != ""
return common.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != ""
}

func isStdinPipeIn() (bool, error) {
Expand Down
40 changes: 20 additions & 20 deletions cmd/credentialUtil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,32 +89,32 @@ func GetOAuthTokenManagerInstance() (*common.UserOAuthTokenManager, error) {
var err error
autoOAuth.Do(func() {
var lca loginCmdArgs
autoLoginType := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType()))
autoLoginType := strings.ToLower(common.GetEnvironmentVariable(common.EEnvironmentVariable.AutoLoginType()))
if autoLoginType == "" {
glcm.Info("Autologin not specified.")
return
}

if tenantID := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.TenantID()); tenantID != "" {
if tenantID := common.GetEnvironmentVariable(common.EEnvironmentVariable.TenantID()); tenantID != "" {
lca.tenantID = tenantID
}

if endpoint := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AADEndpoint()); endpoint != "" {
if endpoint := common.GetEnvironmentVariable(common.EEnvironmentVariable.AADEndpoint()); endpoint != "" {
lca.aadEndpoint = endpoint
}

// Fill up lca
lca.loginType = autoLoginType
switch autoLoginType {
case common.EAutoLoginType.SPN().String():
lca.applicationID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ApplicationID())
lca.certPath = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePath())
lca.certPass = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
lca.clientSecret = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
lca.applicationID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ApplicationID())
lca.certPath = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePath())
lca.certPass = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
lca.clientSecret = common.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
case common.EAutoLoginType.MSI().String():
lca.identityClientID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityClientID())
lca.identityObjectID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityObjectID())
lca.identityResourceID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityResourceString())
lca.identityClientID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityClientID())
lca.identityObjectID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityObjectID())
lca.identityResourceID = common.GetEnvironmentVariable(common.EEnvironmentVariable.ManagedIdentityResourceString())
case common.EAutoLoginType.Device().String():
case common.EAutoLoginType.AzCLI().String():
case common.EAutoLoginType.PsCred().String():
Expand Down Expand Up @@ -174,7 +174,7 @@ var stashedEnvCredType = ""
func GetCredTypeFromEnvVar() common.CredentialType {
rawVal := stashedEnvCredType
if stashedEnvCredType == "" {
rawVal = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CredentialType())
rawVal = common.GetEnvironmentVariable(common.EEnvironmentVariable.CredentialType())
if rawVal == "" {
return common.ECredentialType.Unknown()
}
Expand All @@ -183,7 +183,7 @@ func GetCredTypeFromEnvVar() common.CredentialType {

// Remove the env var after successfully fetching once,
// in case of env var is further spreading into child processes unexpectedly.
glcm.ClearEnvironmentVariable(common.EEnvironmentVariable.CredentialType())
common.ClearEnvironmentVariable(common.EEnvironmentVariable.CredentialType())

// Try to get the value set.
var credType common.CredentialType
Expand Down Expand Up @@ -364,7 +364,7 @@ func isPublic(ctx context.Context, blobResourceURL string, cpkOptions common.Cpk
RetryDelay: ste.UploadRetryDelay,
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
ApplicationID: common.AddUserAgentPrefix(common.UserAgent),
}, nil, ste.LogOptions{}, nil)

blobClient, _ := blob.NewClientWithNoCredential(bURLParts.String(), &blob.ClientOptions{ClientOptions: clientOptions})
Expand Down Expand Up @@ -397,7 +397,7 @@ func mdAccountNeedsOAuth(ctx context.Context, blobResourceURL string, cpkOptions
RetryDelay: ste.UploadRetryDelay,
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
ApplicationID: common.AddUserAgentPrefix(common.UserAgent),
}, nil, ste.LogOptions{}, nil)

blobClient, _ := blob.NewClientWithNoCredential(blobResourceURL, &blob.ClientOptions{ClientOptions: clientOptions})
Expand Down Expand Up @@ -454,8 +454,8 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
}

if location == common.ELocation.S3() {
accessKeyID := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID())
secretAccessKey := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey())
accessKeyID := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID())
secretAccessKey := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey())
if accessKeyID == "" || secretAccessKey == "" {
credType = common.ECredentialType.S3PublicBucket()
public = true
Expand All @@ -467,7 +467,7 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
}

if location == common.ELocation.GCP() {
googleAppCredentials := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials())
googleAppCredentials := common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials())
if googleAppCredentials == "" {
return common.ECredentialType.Unknown(), false, errors.New("GOOGLE_APPLICATION_CREDENTIALS environment variable must be set before using GCP transfer feature")
}
Expand Down Expand Up @@ -508,8 +508,8 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
// BlobFS currently supports Shared key. Remove this piece of code, once
// we deprecate that.
if location == common.ELocation.BlobFS() {
name := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AccountName())
key := glcm.GetEnvironmentVariable(common.EEnvironmentVariable.AccountKey())
name := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountName())
key := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountKey())
if name != "" && key != "" { // TODO: To remove, use for internal testing, SharedKey should not be supported from commandline
credType = common.ECredentialType.SharedKey()
warnIfSharedKeyAuthForDatalake()
Expand Down Expand Up @@ -591,7 +591,7 @@ func createClientOptions(logger common.ILoggerResetable, srcCred *common.ScopedC
RetryDelay: ste.UploadRetryDelay,
MaxRetryDelay: ste.UploadMaxRetryDelay,
}, policy.TelemetryOptions{
ApplicationID: glcm.AddUserAgentPrefix(common.UserAgent),
ApplicationID: common.AddUserAgentPrefix(common.UserAgent),
}, ste.NewAzcopyHTTPClient(frontEndMaxIdleConnectionsPerHost), logOptions, srcCred)
}

Expand Down
2 changes: 1 addition & 1 deletion cmd/env.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ var envCmd = &cobra.Command{
Long: envCmdLongDescription,
Run: func(cmd *cobra.Command, args []string) {
for _, env := range common.VisibleEnvironmentVariables {
val := glcm.GetEnvironmentVariable(env)
val := common.GetEnvironmentVariable(env)
if env.Hidden && !showSensitive {
val = "REDACTED"
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/login.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@ var lgCmd = &cobra.Command{
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
loginCmdArg.certPass = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
loginCmdArg.clientSecret = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
loginCmdArg.certPass = common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
loginCmdArg.clientSecret = common.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
loginCmdArg.persistToken = true

if loginCmdArg.certPass != "" || loginCmdArg.clientSecret != "" {
Expand Down
5 changes: 3 additions & 2 deletions cmd/root.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,9 @@ var rootCmd = &cobra.Command{
Short: rootCmdShortDescription,
Long: rootCmdLongDescription,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if glcm.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout()) != "" {
timeout, err := time.ParseDuration(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout()) + "m")
requestTryTimeout := common.GetEnvironmentVariable(common.EEnvironmentVariable.RequestTryTimeout())
if requestTryTimeout != "" {
timeout, err := time.ParseDuration(requestTryTimeout + "m")
if err == nil {
ste.UploadTryTimeout = timeout
}
Expand Down
2 changes: 1 addition & 1 deletion cmd/zc_traverser_blob.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ func newBlobTraverser(rawURL string, serviceClient *service.Client, ctx context.
isDFS: isDFS,
}

disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning()))
disableHierarchicalScanning := strings.ToLower(common.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning()))

// disableHierarchicalScanning should be true for permanent delete
if (disableHierarchicalScanning == "false" || disableHierarchicalScanning == "") && includeDeleted && (includeSnapshot || includeVersion) {
Expand Down
2 changes: 1 addition & 1 deletion cmd/zc_traverser_gcp_service.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (t *gcpServiceTraverser) Traverse(preprocessor objectMorpher, processor obj
}

func newGCPServiceTraverser(rawURL *url.URL, ctx context.Context, getProperties bool, incrementEnumerationCounter enumerationCounterFunc) (*gcpServiceTraverser, error) {
projectID = glcm.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleCloudProject())
projectID = common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleCloudProject())
t := &gcpServiceTraverser{
ctx: ctx,
incrementEnumerationCounter: incrementEnumerationCounter,
Expand Down
19 changes: 0 additions & 19 deletions cmd/zt_interceptors_for_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@ package cmd

import (
"fmt"
"os"

"github.com/Azure/azure-storage-azcopy/v10/common"
)

Expand Down Expand Up @@ -83,10 +81,6 @@ type mockedLifecycleManager struct {
outputFormat common.OutputFormat
}

func (m *mockedLifecycleManager) DownloadToTempPath() bool {
return false
}

func (m *mockedLifecycleManager) ReportAllJobPartsDone() {
}

Expand Down Expand Up @@ -143,24 +137,11 @@ func (*mockedLifecycleManager) SurrenderControl()
func (*mockedLifecycleManager) RegisterCloseFunc(func()) {}
func (mockedLifecycleManager) AllowReinitiateProgressReporting() {}
func (*mockedLifecycleManager) InitiateProgressReporting(common.WorkController) {}
func (*mockedLifecycleManager) ClearEnvironmentVariable(env common.EnvironmentVariable) {
_ = os.Setenv(env.Name, "")
}
func (*mockedLifecycleManager) GetEnvironmentVariable(env common.EnvironmentVariable) string {
value := os.Getenv(env.Name)
if value == "" {
return env.DefaultValue
}
return value
}
func (m *mockedLifecycleManager) SetOutputFormat(format common.OutputFormat) {
m.outputFormat = format
}
func (*mockedLifecycleManager) EnableInputWatcher() {}
func (*mockedLifecycleManager) EnableCancelFromStdIn() {}
func (*mockedLifecycleManager) AddUserAgentPrefix(userAgent string) string {
return userAgent
}

func (*mockedLifecycleManager) SetForceLogging() {}

Expand Down
2 changes: 1 addition & 1 deletion common/ProxyLookupCache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func init() {
lookupMethod: GetProxyFunc(),
}

ev := GetLifecycleMgr().GetEnvironmentVariable(EEnvironmentVariable.CacheProxyLookup())
ev := GetEnvironmentVariable(EEnvironmentVariable.CacheProxyLookup())
if strings.ToLower(ev) == "true" {
GlobalProxyLookup = c.getProxy
} else {
Expand Down
8 changes: 4 additions & 4 deletions common/clientFactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@ import (
)

func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error) {
name := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountName())
key := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
name := GetEnvironmentVariable(EEnvironmentVariable.AccountName())
key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
// If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables
if name == "" || key == "" {
return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential")
Expand All @@ -37,8 +37,8 @@ func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error) {
}

func GetBlobSharedKeyCredential() (*blob.SharedKeyCredential, error) {
name := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountName())
key := lcm.GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
name := GetEnvironmentVariable(EEnvironmentVariable.AccountName())
key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
// If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables
if name == "" || key == "" {
return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential")
Expand Down
13 changes: 6 additions & 7 deletions common/credentialFactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,13 @@ func (o CredentialOpOptions) panicError(err error) {

// CreateS3Credential creates AWS S3 credential according to credential info.
func CreateS3Credential(ctx context.Context, credInfo CredentialInfo, options CredentialOpOptions) (*credentials.Credentials, error) {
glcm := GetLifecycleMgr()
switch credInfo.CredentialType {
case ECredentialType.S3PublicBucket():
return credentials.NewStatic("", "", "", credentials.SignatureAnonymous), nil
case ECredentialType.S3AccessKey():
accessKeyID := glcm.GetEnvironmentVariable(EEnvironmentVariable.AWSAccessKeyID())
secretAccessKey := glcm.GetEnvironmentVariable(EEnvironmentVariable.AWSSecretAccessKey())
sessionToken := glcm.GetEnvironmentVariable(EEnvironmentVariable.AwsSessionToken())
accessKeyID := GetEnvironmentVariable(EEnvironmentVariable.AWSAccessKeyID())
secretAccessKey := GetEnvironmentVariable(EEnvironmentVariable.AWSSecretAccessKey())
sessionToken := GetEnvironmentVariable(EEnvironmentVariable.AwsSessionToken())

// create and return s3 credential
return credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), nil // S3 uses V4 signature
Expand Down Expand Up @@ -185,11 +184,11 @@ func GetCpkInfo(cpkInfo bool) *blob.CPKInfo {
}

// fetch EncryptionKey and EncryptionKeySHA256 from the environment variables
glcm := GetLifecycleMgr()
encryptionKey := glcm.GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKey())
encryptionKeySHA256 := glcm.GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKeySHA256())
encryptionKey := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKey())
encryptionKeySHA256 := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKeySHA256())
encryptionAlgorithmAES256 := blob.EncryptionAlgorithmTypeAES256

glcm := GetLifecycleMgr()
if encryptionKey == "" || encryptionKeySHA256 == "" {
glcm.Error("fatal: failed to fetch cpk encryption key (" + EEnvironmentVariable.CPKEncryptionKey().Name +
") or hash (" + EEnvironmentVariable.CPKEncryptionKeySHA256().Name + ") from environment variables")
Expand Down
15 changes: 15 additions & 0 deletions common/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ package common
import (
"encoding/json"
"fmt"
"os"
"reflect"
"runtime"
"strings"
Expand All @@ -37,6 +38,20 @@ type EnvironmentVariable struct {
Hidden bool
}

// GetEnvironmentVariable gets the environment variable or its default value
func GetEnvironmentVariable(env EnvironmentVariable) string {
value := os.Getenv(env.Name)
if value == "" {
return env.DefaultValue
}
return value
}

// ClearEnvironmentVariable clears the environment variable
func ClearEnvironmentVariable(variable EnvironmentVariable) {
_ = os.Setenv(variable.Name, "")
}

// This array needs to be updated when a new public environment variable is added
// Things are here, rather than in command line parameters for one of two reasons:
// 1. They are optional and obscure (e.g. performance tuning parameters) or
Expand Down
Loading
Loading