Skip to content

Commit

Permalink
Improved some logging messages and --log-level NONE behavior (#2845)
Browse files Browse the repository at this point in the history
  • Loading branch information
dphulkar-msft authored Jan 16, 2025
1 parent 74ff288 commit 7cecb73
Show file tree
Hide file tree
Showing 16 changed files with 171 additions and 35 deletions.
21 changes: 14 additions & 7 deletions cmd/copy.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,11 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
azcopyScanningLogger.CloseLog()
})

// if no logging, set this empty so that we don't display the log location
if azcopyLogVerbosity == common.LogNone {
azcopyLogPathFolder = ""
}

fromTo, err := ValidateFromTo(raw.src, raw.dst, raw.fromTo) // TODO: src/dst
if err != nil {
return cooked, err
Expand Down Expand Up @@ -549,7 +554,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
}

if cooked.FromTo.To() == common.ELocation.None() && strings.EqualFold(raw.metadata, common.MetadataAndBlobTagsClearFlag) { // in case of Blob, BlobFS and Files
glcm.Info("*** WARNING *** Metadata will be cleared because of input --metadata=clear ")
glcm.Warn("*** WARNING *** Metadata will be cleared because of input --metadata=clear ")
}
cooked.metadata = raw.metadata
if err = validateMetadataString(cooked.metadata); err != nil {
Expand All @@ -573,7 +578,7 @@ func (raw rawCopyCmdArgs) cook() (CookedCopyCmdArgs, error) {
return cooked, errors.New("blob tags can only be set when transferring to blob storage")
}
if cooked.FromTo.To() == common.ELocation.None() && strings.EqualFold(raw.blobTags, common.MetadataAndBlobTagsClearFlag) { // in case of Blob and BlobFS
glcm.Info("*** WARNING *** BlobTags will be cleared because of input --blob-tags=clear ")
glcm.Warn("*** WARNING *** BlobTags will be cleared because of input --blob-tags=clear ")
}
blobTags := common.ToCommonBlobTagsMap(raw.blobTags)
err = validateBlobTagsKeyValue(blobTags)
Expand Down Expand Up @@ -901,7 +906,7 @@ var includeWarningOncer = &sync.Once{}
func (raw *rawCopyCmdArgs) warnIfHasWildcard(oncer *sync.Once, paramName string, value string) {
if strings.Contains(value, "*") || strings.Contains(value, "?") {
oncer.Do(func() {
glcm.Info(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
glcm.Warn(fmt.Sprintf("*** Warning *** The %s parameter does not support wildcards. The wildcard "+
"character provided will be interpreted literally and will not have any wildcard effect. To use wildcards "+
"(in filenames only, not paths) use include-pattern or exclude-pattern", paramName))
})
Expand Down Expand Up @@ -1689,11 +1694,13 @@ func (cca *CookedCopyCmdArgs) waitUntilJobCompletion(blocking bool) {
// print initial message to indicate that the job is starting
// if on dry run mode do not want to print message since no job is being done
if !cca.dryrunMode {
// Output the log location if log-level is set to other then NONE
var logPathFolder string
if azcopyLogPathFolder != "" {
logPathFolder = fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID)
}
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(),
fmt.Sprintf("%s%s%s.log",
azcopyLogPathFolder,
common.OS_PATH_SEPARATOR,
cca.jobID),
logPathFolder,
cca.isCleanupJob,
cca.cleanupJobMessage))
}
Expand Down
13 changes: 9 additions & 4 deletions cmd/copyEnumeratorHelper.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,18 @@ package cmd

import (
"fmt"
"math/rand"

"github.com/Azure/azure-storage-azcopy/v10/common"
"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
"math/rand"
)

var EnumerationParallelism = 1
var EnumerationParallelStatFiles = false

// addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order.
func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *CookedCopyCmdArgs) error {
// Source and destination paths are and should be relative paths.
// Source and destination paths are and should be relative paths.

// dispatch the transfers once the number reaches NumOfFilesPerDispatchJobPart
// we do this so that in the case of large transfer, the transfer engine can get started
Expand Down Expand Up @@ -69,8 +70,12 @@ func dispatchFinalPart(e *common.CopyJobPartOrderRequest, cca *CookedCopyCmdArgs
Rpc(common.ERpcCmd.CopyJobPartOrder(), (*common.CopyJobPartOrderRequest)(e), &resp)

if !resp.JobStarted {
// Output the log location and such
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID), cca.isCleanupJob, cca.cleanupJobMessage))
// Output the log location if log-level is set to other then NONE
var logPathFolder string
if azcopyLogPathFolder != "" {
logPathFolder = fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID)
}
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), logPathFolder, cca.isCleanupJob, cca.cleanupJobMessage))

if cca.dryrunMode {
return nil
Expand Down
6 changes: 3 additions & 3 deletions cmd/copyEnumeratorInit.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
// check against seenFailedContainers so we don't spam the job log with initialization failed errors
if _, ok := seenFailedContainers[dstContainerName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
logDstContainerCreateFailureOnce.Do(func() {
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
glcm.Warn("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
})
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Failed to create destination container %s. The transfer will continue if the container exists", dstContainerName), common.LogWarning)
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Error %s", err), common.LogDebug)
Expand Down Expand Up @@ -194,7 +194,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
// check against seenFailedContainers so we don't spam the job log with initialization failed errors
if _, ok := seenFailedContainers[bucketName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
logDstContainerCreateFailureOnce.Do(func() {
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
glcm.Warn("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
})
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail).", bucketName), common.LogWarning)
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Error %s", err), common.LogDebug)
Expand All @@ -215,7 +215,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde

if _, ok := seenFailedContainers[dstContainerName]; err != nil && jobsAdmin.JobsAdmin != nil && !ok {
logDstContainerCreateFailureOnce.Do(func() {
glcm.Info("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
glcm.Warn("Failed to create one or more destination container(s). Your transfers may still succeed if the container already exists.")
})
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("failed to initialize destination container %s; the transfer will continue (but be wary it may fail).", resName), common.LogWarning)
jobsAdmin.JobsAdmin.LogToJobLog(fmt.Sprintf("Error %s", err), common.LogDebug)
Expand Down
12 changes: 11 additions & 1 deletion cmd/jobsResume.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,12 @@ type resumeJobController struct {
// if blocking is specified to false, then another goroutine spawns and wait out the job
func (cca *resumeJobController) waitUntilJobCompletion(blocking bool) {
// print initial message to indicate that the job is starting
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID), false, ""))
// Output the log location if log-level is set to other then NONE
var logPathFolder string
if azcopyLogPathFolder != "" {
logPathFolder = fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID)
}
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), logPathFolder, false, ""))

// initialize the times necessary to track progress
cca.jobStartTime = time.Now()
Expand Down Expand Up @@ -350,6 +355,11 @@ func (rca resumeCmdArgs) process() error {
return fmt.Errorf("error parsing the jobId %s. Failed with error %s", rca.jobID, err.Error())
}

// if no logging, set this empty so that we don't display the log location
if azcopyLogVerbosity == common.LogNone {
azcopyLogPathFolder = ""
}

includeTransfer := make(map[string]int)
excludeTransfer := make(map[string]int)

Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,7 @@ func init() {
rootCmd.PersistentFlags().Float64Var(&cmdLineCapMegaBitsPerSecond, "cap-mbps", 0, "Caps the transfer rate, in megabits per second. Moment-by-moment throughput might vary slightly from the cap. If this option is set to zero, or it is omitted, the throughput isn't capped.")
rootCmd.PersistentFlags().StringVar(&outputFormatRaw, "output-type", "text", "Format of the command's output. The choices include: text, json. The default value is 'text'.")
rootCmd.PersistentFlags().StringVar(&outputVerbosityRaw, "output-level", "default", "Define the output verbosity. Available levels: essential, quiet.")
rootCmd.PersistentFlags().StringVar(&logVerbosityRaw, "log-level", "INFO", "Define the log verbosity for the log file, available levels: INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').")
rootCmd.PersistentFlags().StringVar(&logVerbosityRaw, "log-level", "INFO", "Define the log verbosity for the log file, available levels: DEBUG(detailed trace), INFO(all requests/responses), WARNING(slow responses), ERROR(only failed requests), and NONE(no output logs). (default 'INFO').")

rootCmd.PersistentFlags().StringVar(&cmdLineExtraSuffixesAAD, trustedSuffixesNameAAD, "", "Specifies additional domain suffixes where Azure Active Directory login tokens may be sent. The default is '"+
trustedSuffixesAAD+"'. Any listed here are added to the default. For security, you should only put Microsoft Azure domains here. Separate multiple entries with semi-colons.")
Expand Down
12 changes: 11 additions & 1 deletion cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,11 @@ func (raw *rawSyncCmdArgs) cook() (cookedSyncCmdArgs, error) {
azcopyScanningLogger.CloseLog()
})

// if no logging, set this empty so that we don't display the log location
if azcopyLogVerbosity == common.LogNone {
azcopyLogPathFolder = ""
}

// this if statement ladder remains instead of being separated to help determine valid combinations for sync
// consider making a map of valid source/dest combos and consolidating this to generic source/dest setups, akin to the lower if statement
// TODO: if expand the set of source/dest combos supported by sync, update this method the declarative test framework:
Expand Down Expand Up @@ -505,7 +510,12 @@ func (cca *cookedSyncCmdArgs) scanningComplete() bool {
// if blocking is specified to false, then another goroutine spawns and wait out the job
func (cca *cookedSyncCmdArgs) waitUntilJobCompletion(blocking bool) {
// print initial message to indicate that the job is starting
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID), false, ""))
// Output the log location if log-level is set to other then NONE
var logPathFolder string
if azcopyLogPathFolder != "" {
logPathFolder = fmt.Sprintf("%s%s%s.log", azcopyLogPathFolder, common.OS_PATH_SEPARATOR, cca.jobID)
}
glcm.Init(common.GetStandardInitOutputBuilder(cca.jobID.String(), logPathFolder, false, ""))

// initialize the times necessary to track progress
cca.jobStartTime = time.Now()
Expand Down
4 changes: 3 additions & 1 deletion common/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ func GetStandardInitOutputBuilder(jobID string, logFileLocation string, isCleanu
sb.WriteString(cleanupHeader)
} else {
sb.WriteString("\nJob " + jobID + " has started\n")
sb.WriteString("Log file is located at: " + logFileLocation)
if logFileLocation != "" {
sb.WriteString("Log file is located at: " + logFileLocation)
}
sb.WriteString("\n")
}
return sb.String()
Expand Down
9 changes: 7 additions & 2 deletions e2etest/newe2e_task_validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -175,15 +175,20 @@ func ValidateListOutput(a Asserter, stdout AzCopyStdout, expectedObjects map[AzC
a.Assert("summary must match", Equal{}, listStdout.Summary, DerefOrZero(expectedSummary))
}

func ValidateMessageOutput(a Asserter, stdout AzCopyStdout, message string) {
func ValidateMessageOutput(a Asserter, stdout AzCopyStdout, message string, shouldContain bool) {
if dryrunner, ok := a.(DryrunAsserter); ok && dryrunner.Dryrun() {
return
}
var contains bool
for _, line := range stdout.RawStdout() {
if strings.Contains(line, message) {
return
contains = true
break
}
}
if (!contains && !shouldContain) || (contains && shouldContain) {
return
}
fmt.Println(stdout.String())
a.Error(fmt.Sprintf("expected message (%s) not found in azcopy output", message))
}
Expand Down
92 changes: 92 additions & 0 deletions e2etest/zt_azcopy_flags_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
// Copyright © Microsoft <[email protected]>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package e2etest

import (
"time"

"github.com/Azure/azure-storage-azcopy/v10/common"
)

func init() {
suiteManager.RegisterSuite(&FlagsFunctionalitySuite{})
}

type FlagsFunctionalitySuite struct{}

// @brief This test verifies that when the --log-level is set to NONE,
// AzCopy does not display the "Log files located" message on the console.
func (s *FlagsFunctionalitySuite) Scenario_LogLevelNone(svm *ScenarioVariationManager) {
azCopyVerb := ResolveVariation(svm, []AzCopyVerb{AzCopyVerbCopy, AzCopyVerbSync}) // Calculate verb early to create the destination object early
// Scale up from service to object
dstObj := CreateResource[ContainerResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{common.ELocation.Local(), common.ELocation.Blob(), common.ELocation.File(), common.ELocation.BlobFS()})), ResourceDefinitionContainer{}).GetObject(svm, "test", common.EEntityType.File())

// The object must exist already if we're syncing.
if azCopyVerb == AzCopyVerbSync {
dstObj.Create(svm, NewZeroObjectContentContainer(0), ObjectProperties{})

if !svm.Dryrun() {
// Make sure the LMT is in the past
time.Sleep(time.Second * 10)
}
}

body := NewRandomObjectContentContainer(SizeFromString("10K"))
// Scale up from service to object
srcObj := CreateResource[ObjectResourceManager](svm, GetRootResource(svm, ResolveVariation(svm, []common.Location{ /*common.ELocation.Local(), */ common.ELocation.Blob() /*, common.ELocation.File(), common.ELocation.BlobFS()*/})), ResourceDefinitionObject{
ObjectName: pointerTo("test"),
Body: body,
})

// no local->local
if srcObj.Location().IsLocal() && dstObj.Location().IsLocal() {
svm.InvalidateScenario()
return
}

sasOpts := GenericAccountSignatureValues{}
logLevel := common.LogLevel.None(common.LogNone)
outputType := common.EOutputFormat.Text()

stdOut, _ := RunAzCopy(
svm,
AzCopyCommand{
Verb: azCopyVerb,
Targets: []ResourceManager{
TryApplySpecificAuthType(srcObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{
SASTokenOptions: sasOpts,
}),
TryApplySpecificAuthType(dstObj, EExplicitCredentialType.SASToken(), svm, CreateAzCopyTargetOptions{
SASTokenOptions: sasOpts,
}),
},
Flags: CopyFlags{
CopySyncCommonFlags: CopySyncCommonFlags{
Recursive: pointerTo(true),
GlobalFlags: GlobalFlags{
LogLevel: &logLevel,
OutputType: &outputType,
},
},
},
})
ValidateMessageOutput(svm, stdOut, "Log file is located at", false)
}
9 changes: 5 additions & 4 deletions e2etest/zt_newe2e_basic_functionality_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package e2etest

import (
"strconv"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-storage-azcopy/v10/common"
"strconv"
"time"
)

func init() {
Expand Down Expand Up @@ -347,7 +348,7 @@ func (s *BasicFunctionalitySuite) Scenario_SingleFileUploadDownload_EmptySAS(svm
})

// Validate that the stdout contains the missing sas message
ValidateMessageOutput(svm, stdout, "Please authenticate using Microsoft Entra ID (https://aka.ms/AzCopy/AuthZ), use AzCopy login, or append a SAS token to your Azure URL.")
ValidateMessageOutput(svm, stdout, "Please authenticate using Microsoft Entra ID (https://aka.ms/AzCopy/AuthZ), use AzCopy login, or append a SAS token to your Azure URL.", true)
}

func (s *BasicFunctionalitySuite) Scenario_Sync_EmptySASErrorCodes(svm *ScenarioVariationManager) {
Expand Down Expand Up @@ -534,5 +535,5 @@ func (s *BasicFunctionalitySuite) Scenario_TagsPermission(svm *ScenarioVariation
},
)

ValidateMessageOutput(svm, stdOut, "Authorization failed during an attempt to set tags, please ensure you have the appropriate Tags permission")
ValidateMessageOutput(svm, stdOut, "Authorization failed during an attempt to set tags, please ensure you have the appropriate Tags permission", true)
}
3 changes: 2 additions & 1 deletion e2etest/zt_newe2e_file_oauth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package e2etest

import (
"fmt"

"github.com/Azure/azure-storage-azcopy/v10/common"
)

Expand Down Expand Up @@ -35,5 +36,5 @@ func (s *FileOAuthTestSuite) Scenario_FileBlobOAuthError(svm *ScenarioVariationM
ShouldFail: true,
})

ValidateMessageOutput(svm, stdout, fmt.Sprintf("S2S %s from Azure File authenticated with Azure AD to Blob/BlobFS is not supported", azCopyVerb))
ValidateMessageOutput(svm, stdout, fmt.Sprintf("S2S %s from Azure File authenticated with Azure AD to Blob/BlobFS is not supported", azCopyVerb), true)
}
2 changes: 1 addition & 1 deletion e2etest/zt_newe2e_jobs_clean_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,5 @@ func (s *JobsCleanSuite) Scenario_JobsCleanBasic(svm *ScenarioVariationManager)
},
})

ValidateMessageOutput(svm, jobsCleanOutput, cmd.JobsCleanupSuccessMsg)
ValidateMessageOutput(svm, jobsCleanOutput, cmd.JobsCleanupSuccessMsg, true)
}
5 changes: 3 additions & 2 deletions e2etest/zt_newe2e_list_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
package e2etest

import (
"strings"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
blobsas "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
"github.com/Azure/azure-storage-azcopy/v10/cmd"
"github.com/Azure/azure-storage-azcopy/v10/common"
"strings"
)

func init() {
Expand Down Expand Up @@ -820,7 +821,7 @@ func (s *ListSuite) Scenario_EmptySASErrorCodes(svm *ScenarioVariationManager) {
})

// Validate that the stdout contains these error URLs
ValidateMessageOutput(svm, stdout, "https://aka.ms/AzCopyError/NoAuthenticationInformation")
ValidateMessageOutput(svm, stdout, "https://aka.ms/AzCopyError/NoAuthenticationInformation", true)
}

func (s *ListSuite) Scenario_VirtualDirectoryHandling(svm *ScenarioVariationManager) {
Expand Down
Loading

0 comments on commit 7cecb73

Please sign in to comment.