Cleanup before sync migration

azcopy/credentialUtil.go

@@ -270,7 +270,7 @@ func isPublic(ctx context.Context, blobResourceURL string, cpkOptions common.Cpk
 	return false, nil
 }
 
-// mdAccountNeedsOAuth pings the passed in md account, and checks if we need additional token with Disk-socpe
+// mdAccountNeedsOAuth pings the passed in md account, and checks if we need additional token with Disk-scope
 func mdAccountNeedsOAuth(ctx context.Context, blobResourceURL string, cpkOptions common.CpkOptions) (bool, error) {
 	// This request will not be logged. This can fail, and too many Cx do not like this.
 	clientOptions := ste.NewClientOptions(policy.RetryOptions{

azcopy/optionsUtil.go

@@ -0,0 +1,50 @@
+// Copyright © 2025 Microsoft <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package azcopy
+
+import (
+	"errors"
+	"fmt"
+	"math"
+)
+
+// BlockSizeInBytes converts a FLOATING POINT number of MiB, to a number of bytes
+// A non-nil error is returned if the conversion is not possible to do accurately (e.g. it comes out of a fractional number of bytes)
+// The purpose of using floating point is to allow specialist users (e.g. those who want small block sizes to tune their read IOPS)
+// to use fractions of a MiB. E.g.
+// 0.25 = 256 KiB
+// 0.015625 = 16 KiB
+func BlockSizeInBytes(rawBlockSizeInMiB float64) (int64, error) {
+	if rawBlockSizeInMiB < 0 {
+		return 0, errors.New("negative block size not allowed")
+	}
+	rawSizeInBytes := rawBlockSizeInMiB * 1024 * 1024 // internally we use bytes, but users' convenience the command line uses MiB
+	if rawSizeInBytes > math.MaxInt64 {
+		return 0, errors.New("block size too big for int64")
+	}
+	const epsilon = 0.001 // arbitrarily using a tolerance of 1000th of a byte
+	_, frac := math.Modf(rawSizeInBytes)
+	isWholeNumber := frac < epsilon || frac > 1.0-epsilon // frac is very close to 0 or 1, so rawSizeInBytes is (very close to) an integer
+	if !isWholeNumber {
+		return 0, fmt.Errorf("while fractional numbers of MiB are allowed as the block size, the fraction must result to a whole number of bytes. %.12f MiB resolves to %.3f bytes", rawBlockSizeInMiB, rawSizeInBytes)
+	}
+	return int64(math.Round(rawSizeInBytes)), nil
+}

azcopy/pathUtils.go

@@ -23,9 +23,11 @@ package azcopy
 import (
 	"fmt"
 	"net/url"
+	"strings"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/pkg/errors"
 )
 
 func GetContainerName(path string, location common.Location) (string, error) {
@@ -69,3 +71,82 @@ func GetContainerName(path string, location common.Location) (string, error) {
 		return "", fmt.Errorf("cannot get container name on location type %s", location.String())
 	}
 }
+
+// ----- LOCATION LEVEL HANDLING -----
+type LocationLevel uint8
+
+var ELocationLevel LocationLevel = 0
+
+func (LocationLevel) Account() LocationLevel   { return 0 } // Account is never used in AzCopy, but is in testing to understand resource management.
+func (LocationLevel) Service() LocationLevel   { return 1 }
+func (LocationLevel) Container() LocationLevel { return 2 }
+func (LocationLevel) Object() LocationLevel    { return 3 } // An Object can be a directory or object.
+
+// Uses syntax to assume the "level" of a location.
+// This is typically used to
+func DetermineLocationLevel(location string, locationType common.Location, source bool) (LocationLevel, error) {
+	switch locationType {
+	// In local, there's no such thing as a service.
+	// As such, we'll treat folders as containers, and files as objects.
+	case common.ELocation.Local():
+		level := LocationLevel(ELocationLevel.Object())
+		if strings.Contains(location, "*") {
+			return ELocationLevel.Container(), nil
+		}
+
+		if strings.HasSuffix(location, "/") {
+			level = ELocationLevel.Container()
+		}
+
+		if !source {
+			return level, nil // Return the assumption.
+		}
+
+		fi, err := common.OSStat(location)
+
+		if err != nil {
+			return level, nil // Return the assumption.
+		}
+
+		if fi.IsDir() {
+			return ELocationLevel.Container(), nil
+		} else {
+			return ELocationLevel.Object(), nil
+		}
+	case common.ELocation.Benchmark():
+		return ELocationLevel.Object(), nil // we always benchmark to a subfolder, not the container root
+
+	case common.ELocation.Blob(),
+		common.ELocation.File(),
+		common.ELocation.FileNFS(),
+		common.ELocation.BlobFS(),
+		common.ELocation.S3(),
+		common.ELocation.GCP():
+		URL, err := url.Parse(location)
+
+		if err != nil {
+			return ELocationLevel.Service(), err
+		}
+
+		// GenericURLParts determines the correct resource URL parts to make use of
+		bURL := common.NewGenericResourceURLParts(*URL, locationType)
+
+		if strings.Contains(bURL.GetContainerName(), "*") && bURL.GetObjectName() != "" {
+			return ELocationLevel.Service(), errors.New("can't use a wildcarded container name and specific blob name in combination")
+		}
+
+		if bURL.GetObjectName() != "" {
+			return ELocationLevel.Object(), nil
+		} else if bURL.GetContainerName() != "" && !strings.Contains(bURL.GetContainerName(), "*") {
+			return ELocationLevel.Container(), nil
+		} else {
+			return ELocationLevel.Service(), nil
+		}
+	default: // Probably won't ever hit this
+		return ELocationLevel.Service(), fmt.Errorf("getting level of location is impossible on location %s", locationType)
+	}
+}
+
+func StartsWith(s string, t string) bool {
+	return len(s) >= len(t) && strings.EqualFold(s[0:len(t)], t)
+}

azcopy/remoteClientUtils.go

@@ -0,0 +1,164 @@
+// Copyright © 2025 Microsoft <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package azcopy
+
+import (
+	"context"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/Azure/azure-storage-azcopy/v10/traverser"
+)
+
+func GetSourceServiceClient(ctx context.Context,
+	source common.ResourceString,
+	loc common.Location,
+	trailingDot common.TrailingDotOption,
+	cpk common.CpkOptions,
+	uotm *common.UserOAuthTokenManager) (*common.ServiceClient, common.CredentialType, error) {
+	srcCredType, _, err := GetCredentialTypeForLocation(ctx,
+		loc,
+		source,
+		true,
+		cpk,
+		uotm)
+	if err != nil {
+		return nil, srcCredType, err
+	}
+	var tc azcore.TokenCredential
+	if srcCredType.IsAzureOAuth() {
+		// Get token from env var or cache.
+		tokenInfo, err := uotm.GetTokenInfo(ctx)
+		if err != nil {
+			return nil, srcCredType, err
+		}
+
+		tc, err = tokenInfo.GetTokenCredential()
+		if err != nil {
+			return nil, srcCredType, err
+		}
+	}
+
+	var srcReauthTok *common.ScopedAuthenticator
+	if at, ok := tc.(common.AuthenticateToken); ok { // We don't need two different tokens here since it gets passed in just the same either way.
+		// This will cause a reauth with StorageScope, which is fine, that's the original Authenticate call as it stands.
+		srcReauthTok = (*common.ScopedAuthenticator)(common.NewScopedCredential(at, common.ECredentialType.OAuthToken()))
+	}
+
+	options := traverser.CreateClientOptions(common.AzcopyCurrentJobLogger, nil, srcReauthTok)
+
+	// Create Source Client.
+	var azureFileSpecificOptions any
+	if loc == common.ELocation.File() || loc == common.ELocation.FileNFS() {
+		azureFileSpecificOptions = &common.FileClientOptions{
+			AllowTrailingDot: trailingDot == common.ETrailingDotOption.Enable(),
+		}
+	}
+
+	srcServiceClient, err := common.GetServiceClientForLocation(
+		loc,
+		source,
+		srcCredType,
+		tc,
+		&options,
+		azureFileSpecificOptions,
+	)
+	if err != nil {
+		return nil, srcCredType, err
+	}
+	return srcServiceClient, srcCredType, nil
+}
+
+func GetDestinationServiceClient(ctx context.Context,
+	destination common.ResourceString,
+	fromTo common.FromTo,
+	srcCredType common.CredentialType,
+	trailingDot common.TrailingDotOption,
+	cpk common.CpkOptions,
+	uotm *common.UserOAuthTokenManager) (*common.ServiceClient, common.CredentialType, error) {
+	dstCredType, _, err := GetCredentialTypeForLocation(ctx,
+		fromTo.To(),
+		destination,
+		false,
+		cpk,
+		uotm)
+	if err != nil {
+		return nil, dstCredType, err
+	}
+	var tc azcore.TokenCredential
+	if dstCredType.IsAzureOAuth() {
+		// Get token from env var or cache.
+		tokenInfo, err := uotm.GetTokenInfo(ctx)
+		if err != nil {
+			return nil, dstCredType, err
+		}
+
+		tc, err = tokenInfo.GetTokenCredential()
+		if err != nil {
+			return nil, dstCredType, err
+		}
+	}
+
+	var dstReauthTok *common.ScopedAuthenticator
+	if at, ok := tc.(common.AuthenticateToken); ok { // We don't need two different tokens here since it gets passed in just the same either way.
+		// This will cause a reauth with StorageScope, which is fine, that's the original Authenticate call as it stands.
+		dstReauthTok = (*common.ScopedAuthenticator)(common.NewScopedCredential(at, common.ECredentialType.OAuthToken()))
+	}
+
+	var srcTokenCred *common.ScopedToken
+	if fromTo.IsS2S() && srcCredType.IsAzureOAuth() {
+		// Get token from env var or cache.
+		srcTokenInfo, err := uotm.GetTokenInfo(ctx)
+		if err != nil {
+			return nil, dstCredType, err
+		}
+
+		sourceTc, err := srcTokenInfo.GetTokenCredential()
+		if err != nil {
+			return nil, dstCredType, err
+		}
+		srcTokenCred = common.NewScopedCredential(sourceTc, srcCredType)
+	}
+
+	options := traverser.CreateClientOptions(common.AzcopyCurrentJobLogger, srcTokenCred, dstReauthTok)
+
+	// Create Destination Client.
+	var azureFileSpecificOptions any
+	if fromTo.To() == common.ELocation.File() || fromTo.To() == common.ELocation.FileNFS() {
+		azureFileSpecificOptions = &common.FileClientOptions{
+			AllowTrailingDot:       trailingDot == common.ETrailingDotOption.Enable(),
+			AllowSourceTrailingDot: trailingDot == common.ETrailingDotOption.Enable() && fromTo.To() == common.ELocation.File(),
+		}
+	}
+
+	dstServiceClient, err := common.GetServiceClientForLocation(
+		fromTo.To(),
+		destination,
+		dstCredType,
+		tc,
+		&options,
+		azureFileSpecificOptions,
+	)
+	if err != nil {
+		return nil, dstCredType, err
+	}
+	return dstServiceClient, dstCredType, nil
+}