@@ -26,6 +26,7 @@ import (
2626 "encoding/json"
2727 "errors"
2828 "fmt"
29+ "github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
2930 "io"
3031 "math"
3132 "net/url"
@@ -66,7 +67,7 @@ type rawCopyCmdArgs struct {
6667 src string
6768 dst string
6869 fromTo string
69- //blobUrlForRedirection string
70+ // blobUrlForRedirection string
7071
7172 // new include/exclude only apply to file names
7273 // implemented for remove (and sync) only
@@ -1312,9 +1313,9 @@ func (cca *CookedCopyCmdArgs) processRedirectionUpload(blobResource common.Resou
13121313// dispatches the job order (in parts) to the storage engine
13131314func (cca * CookedCopyCmdArgs ) processCopyJobPartOrders () (err error ) {
13141315 ctx := context .WithValue (context .TODO (), ste .ServiceAPIVersionOverride , ste .DefaultServiceApiVersion )
1315- // Make AUTO default for Azure Files since Azure Files throttles too easily.
1316- if ste .JobsAdmin != nil && (cca .FromTo .From () == common .ELocation .File () || cca .FromTo .To () == common .ELocation .File ()) {
1317- ste .JobsAdmin .SetConcurrencySettingsToAuto ()
1316+ // Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value
1317+ if jobsAdmin .JobsAdmin != nil && (cca .FromTo .From () == common .ELocation .File () || cca .FromTo .To () == common .ELocation .File ()) && glcm . GetEnvironmentVariable ( common . EEnvironmentVariable . ConcurrencyValue ()) == "" {
1318+ jobsAdmin .JobsAdmin .SetConcurrencySettingsToAuto ()
13181319 }
13191320
13201321 // Note: credential info here is only used by remove at the moment.
@@ -1556,6 +1557,55 @@ func (cca *CookedCopyCmdArgs) ReportProgressOrExit(lcm common.LifecycleMgr) (tot
15561557 // if json is not desired, and job is done, then we generate a special end message to conclude the job
15571558 duration := time .Now ().Sub (cca .jobStartTime ) // report the total run time of the job
15581559
1560+ var computeThroughput = func () float64 {
1561+ // compute the average throughput for the last time interval
1562+ bytesInMb := float64 (float64 (summary .BytesOverWire - cca .intervalBytesTransferred ) / float64 (base10Mega ))
1563+ timeElapsed := time .Since (cca .intervalStartTime ).Seconds ()
1564+
1565+ // reset the interval timer and byte count
1566+ cca .intervalStartTime = time .Now ()
1567+ cca .intervalBytesTransferred = summary .BytesOverWire
1568+
1569+ return common .Iffloat64 (timeElapsed != 0 , bytesInMb / timeElapsed , 0 ) * 8
1570+ }
1571+ glcm .Progress (func (format common.OutputFormat ) string {
1572+ if format == common .EOutputFormat .Json () {
1573+ jsonOutput , err := json .Marshal (summary )
1574+ common .PanicIfErr (err )
1575+ return string (jsonOutput )
1576+ } else {
1577+ // abbreviated output for cleanup jobs
1578+ if cca .isCleanupJob {
1579+ return cleanupStatusString
1580+ }
1581+
1582+ // if json is not needed, then we generate a message that goes nicely on the same line
1583+ // display a scanning keyword if the job is not completely ordered
1584+ var scanningString = " (scanning...)"
1585+ if summary .CompleteJobOrdered {
1586+ scanningString = ""
1587+ }
1588+
1589+ throughput := computeThroughput ()
1590+ throughputString := fmt .Sprintf ("2-sec Throughput (Mb/s): %v" , jobsAdmin .ToFixed (throughput , 4 ))
1591+ if throughput == 0 {
1592+ // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it.
1593+ throughputString = ""
1594+ }
1595+
1596+ // indicate whether constrained by disk or not
1597+ isBenchmark := cca .FromTo .From () == common .ELocation .Benchmark ()
1598+ perfString , diskString := getPerfDisplayText (summary .PerfStrings , summary .PerfConstraint , duration , isBenchmark )
1599+
1600+ return fmt .Sprintf ("%.1f %%, %v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s" ,
1601+ summary .PercentComplete ,
1602+ summary .TransfersCompleted ,
1603+ summary .TransfersFailed ,
1604+ summary .TotalTransfers - (summary .TransfersCompleted + summary .TransfersFailed + summary .TransfersSkipped ),
1605+ summary .TransfersSkipped , summary .TotalTransfers , scanningString , perfString , throughputString , diskString )
1606+ }
1607+ })
1608+
15591609 if jobDone {
15601610 exitCode := cca .getSuccessExitCode ()
15611611 if summary .TransfersFailed > 0 {
@@ -1585,7 +1635,7 @@ TotalBytesTransferred: %v
15851635Final Job Status: %v%s%s
15861636` ,
15871637 summary .JobID .String (),
1588- ste .ToFixed (duration .Minutes (), 4 ),
1638+ jobsAdmin .ToFixed (duration .Minutes (), 4 ),
15891639 summary .FileTransfers ,
15901640 summary .FolderPropertyTransfers ,
15911641 summary .TotalTransfers ,
@@ -1603,7 +1653,7 @@ Final Job Status: %v%s%s
16031653 }
16041654
16051655 // log to job log
1606- jobMan , exists := ste .JobsAdmin .JobMgr (summary .JobID )
1656+ jobMan , exists := jobsAdmin .JobsAdmin .JobMgr (summary .JobID )
16071657 if exists {
16081658 jobMan .Log (pipeline .LogInfo , logStats + "\n " + output )
16091659 }
@@ -1620,56 +1670,6 @@ Final Job Status: %v%s%s
16201670 }
16211671 }
16221672
1623- var computeThroughput = func () float64 {
1624- // compute the average throughput for the last time interval
1625- bytesInMb := float64 (float64 (summary .BytesOverWire - cca .intervalBytesTransferred ) / float64 (base10Mega ))
1626- timeElapsed := time .Since (cca .intervalStartTime ).Seconds ()
1627-
1628- // reset the interval timer and byte count
1629- cca .intervalStartTime = time .Now ()
1630- cca .intervalBytesTransferred = summary .BytesOverWire
1631-
1632- return common .Iffloat64 (timeElapsed != 0 , bytesInMb / timeElapsed , 0 ) * 8
1633- }
1634-
1635- glcm .Progress (func (format common.OutputFormat ) string {
1636- if format == common .EOutputFormat .Json () {
1637- jsonOutput , err := json .Marshal (summary )
1638- common .PanicIfErr (err )
1639- return string (jsonOutput )
1640- } else {
1641- // abbreviated output for cleanup jobs
1642- if cca .isCleanupJob {
1643- return cleanupStatusString
1644- }
1645-
1646- // if json is not needed, then we generate a message that goes nicely on the same line
1647- // display a scanning keyword if the job is not completely ordered
1648- var scanningString = " (scanning...)"
1649- if summary .CompleteJobOrdered {
1650- scanningString = ""
1651- }
1652-
1653- throughput := computeThroughput ()
1654- throughputString := fmt .Sprintf ("2-sec Throughput (Mb/s): %v" , ste .ToFixed (throughput , 4 ))
1655- if throughput == 0 {
1656- // As there would be case when no bits sent from local, e.g. service side copy, when throughput = 0, hide it.
1657- throughputString = ""
1658- }
1659-
1660- // indicate whether constrained by disk or not
1661- isBenchmark := cca .FromTo .From () == common .ELocation .Benchmark ()
1662- perfString , diskString := getPerfDisplayText (summary .PerfStrings , summary .PerfConstraint , duration , isBenchmark )
1663-
1664- return fmt .Sprintf ("%.1f %%, %v Done, %v Failed, %v Pending, %v Skipped, %v Total%s, %s%s%s" ,
1665- summary .PercentComplete ,
1666- summary .TransfersCompleted ,
1667- summary .TransfersFailed ,
1668- summary .TotalTransfers - (summary .TransfersCompleted + summary .TransfersFailed + summary .TransfersSkipped ),
1669- summary .TransfersSkipped , summary .TotalTransfers , scanningString , perfString , throughputString , diskString )
1670- }
1671- })
1672-
16731673 return
16741674}
16751675
@@ -1763,7 +1763,7 @@ func init() {
17631763 cpCmd := & cobra.Command {
17641764 Use : "copy [source] [destination]" ,
17651765 Aliases : []string {"cp" , "c" },
1766- SuggestFor : []string {"cpy" , "cy" , "mv" }, //TODO why does message appear twice on the console
1766+ SuggestFor : []string {"cpy" , "cy" , "mv" }, // TODO why does message appear twice on the console
17671767 Short : copyCmdShortDescription ,
17681768 Long : copyCmdLongDescription ,
17691769 Example : copyCmdExample ,
0 commit comments