Create msi for windows

azcopy.wxs

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+  <Product Id="*" Name="azcopy" Language="1033" Version="10.28.0" Manufacturer="Microsoft" UpgradeCode="abeee33a-0cca-4060-89d7-0687f7374f7a">
+    <Package InstallerVersion="200" Compressed="yes" InstallScope="perMachine" />
+
+    <Media Id="1" Cabinet="azcopy.cab" EmbedCab="yes" />
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+      <Directory Id="ProgramFilesFolder">
+        <Directory Id="INSTALLDIR" Name="azcopy">
+          <Component Id="azcopy.exe" Guid="abeee33a-0cca-4060-89d7-0687f7374f7a">
+            <File Id="azcopy.exe" Source="azcopy_executable_path" />
+          </Component>
+        </Directory>
+      </Directory>
+    </Directory>
+
+    <Feature Id="ProductFeature" Title="AzCopy" Level="1">
+      <ComponentRef Id="azcopy.exe" />
+    </Feature>
+  </Product>
+</Wix>

cmd/copyEnumeratorInit.go

@@ -314,7 +314,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 	return NewCopyEnumerator(traverser, filters, processor, finalizer), nil
 }
 
-// This is condensed down into an individual function as we don't end up re-using the destination traverser at all.
+// This is condensed down into an individual function as we don't end up reusing the destination traverser at all.
 // This is just for the directory check.
 func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *context.Context) bool {
 	var err error

cmd/zt_parseSize_test.go

@@ -21,9 +21,10 @@
 package cmd
 
 import (
+	"testing"
+
 	"github.com/stretchr/testify/assert"
 	chk "gopkg.in/check.v1"
-	"testing"
 )
 
 type parseSizeSuite struct{}
@@ -55,7 +56,7 @@
 	_, err = ParseSizeString("123T", "foo-bar") // we don't support terabytes
 	a.Equal(expectedError, err.Error())
 
-	_, err = ParseSizeString("abcK", "foo-bar")
+	_, err = ParseSizeString("abck", "foo-bar")
 	a.Equal(expectedError, err.Error())
 
-}
+}

common/version.go

@@ -1,6 +1,6 @@
 package common
 
-const AzcopyVersion = "10.28.0-Preview"
+const AzcopyVersion = "10.28.0"
 const UserAgent = "AzCopy/" + AzcopyVersion
 const S3ImportUserAgent = "S3Import " + UserAgent
 const GCPImportUserAgent = "GCPImport " + UserAgent

e2etest/newe2e_account_registry.go

@@ -2,15 +2,16 @@
 
 import (
 	"fmt"
+	"strings"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/google/uuid"
-	"strings"
 )
 
 // AccountRegistry is a set of accounts that are intended to be initialized when the tests start running.
 // Suites and tests should not add to this pool.
 // todo: long-term, support flexible static configuration of accounts.
-var AccountRegistry = map[string]AccountResourceManager{} // For re-using accounts across testing
+var AccountRegistry = map[string]AccountResourceManager{} // For reusing accounts across testing
 
 func GetAccount(a Asserter, AccountName string) AccountResourceManager {
 	targetAccount, ok := AccountRegistry[AccountName]
@@ -111,8 +112,8 @@
 		EAccountType.PremiumBlockBlobs(), EAccountType.PremiumHNSEnabled(), EAccountType.HierarchicalNamespaceEnabled():
 		azureAcct, ok := arm.(*AzureAccountResourceManager)
 		a.Assert("account manager must be azure account", Equal{}, ok, true)
-
+		e2etest/newe2e_task_runazcopy.go:115
 		armAcct := azureAcct.ManagementClient()
 		a.Assert("cannot delete an account that does not have a management client associated", Not{IsNil{}}, armAcct)
 
 		a.NoError("delete account", armAcct.Delete())
@@ -127,7 +128,7 @@
 	PremiumPageBlobAcct string = "PremiumPageBlob"
 )
 
 func AccountRegistryInitHook(a Asserter) {
 	if GlobalConfig.StaticResources() {
 		acctInfo := GlobalConfig.E2EAuthConfig.StaticStgAcctInfo
 
@@ -154,7 +155,7 @@
 	}
 }
 
 func AccountRegistryCleanupHook(a Asserter) {
 	if GlobalConfig.StaticResources() {
 		return // no need to attempt cleanup
 	}
@@ -165,4 +166,4 @@
 			a.Assert("Delete account", NoError{}, managementClient.Delete())
 		}
 	}
 }

e2etest/newe2e_task_runazcopy.go

@@ -112,7 +112,7 @@ type AzCopyCommand struct {
 }
 
 type AzCopyEnvironment struct {
-	// `env:"XYZ"` is re-used but does not inherit the traits of config's env trait. Merely used for low-code mapping.
+	// `env:"XYZ"` is reused but does not inherit the traits of config's env trait. Merely used for low-code mapping.
 
 	LogLocation                  *string `env:"AZCOPY_LOG_LOCATION,defaultfunc:DefaultLogLoc"`
 	JobPlanLocation              *string `env:"AZCOPY_JOB_PLAN_LOCATION,defaultfunc:DefaultPlanLoc"`

release-pipeline.yml

@@ -387,6 +387,8 @@ stages:
         variables:
           - name: root_dir
             value: '$(System.DefaultWorkingDirectory)'
+          - name: work_dir
+            value: '$(Build.SourcesDirectory)'
 
         steps:
           - task: GoTool@0
@@ -442,6 +444,45 @@ stages:
               GOOS: windows
               CGO_ENABLED: 0
 
+          - powershell: |
+              # Copy the azcopy executable to the working directory with a new name
+              Copy-Item -Path "$(Build.ArtifactStagingDirectory)\azcopy_windows_amd64.exe" -Destination "$(work_dir)\azcopy.exe" -Force
+
+              # Get AzCopy version
+              $azcopyVersionOutput = & "$(work_dir)\azcopy.exe" --version
+              Write-Host "Raw AzCopy version output: $azcopyVersionOutput"
+
+              # Extract the version using regex
+              if ($azcopyVersionOutput -match '(\d+\.\d+\.\d+)$') {
+                  $azcopyVersion = $matches[1]  # Access the first capture group
+                  Write-Host "AzCopy version: $azcopyVersion"
+              } else {
+                  Write-Host "Failed to extract AzCopy version. Raw output: $azcopyVersionOutput"
+              }
+
+              # Output the version to be captured as a pipeline variable
+              Write-Host "##vso[task.setvariable variable=azcopyVersion]$azcopyVersion"
+            displayName: 'Copy Windows Amd64 required files for packaging'
+
+          - template: setup/create_windows_msi.yml
+            parameters:
+              working_dir: $(work_dir)
+              build_dir: $(Build.ArtifactStagingDirectory)
+              azcopy_version: $(azcopyVersion)
+              azcopy_msi_name: 'azcopy_windows_amd64.msi'
+
+          - powershell: |
+              # Copy the azcopy executable to the working directory with a new name
+              Copy-Item -Path "$(Build.ArtifactStagingDirectory)\azcopy_windows_arm64.exe" -Destination "$(work_dir)\azcopy.exe" -Force
+            displayName: 'Copy Windows Arm64 required files for packaging'
+
+          - template: setup/create_windows_msi.yml
+            parameters:
+              working_dir: $(work_dir)
+              build_dir: $(Build.ArtifactStagingDirectory)
+              azcopy_version: $(azcopyVersion)
+              azcopy_msi_name: 'azcopy_windows_arm64.msi'
+
           - task: PublishBuildArtifacts@1
             inputs:
               artifactName: 'azCopy-windows-temp'
@@ -785,11 +826,12 @@ stages:
             parameters:
               FolderPath: $(Build.ArtifactStagingDirectory)/azCopy-windows-temp
               ESRP_AZCOPY_WIN_KEY_CODE: $(ESRP_AZCOPY_WIN_KEY_CODE)
-              Pattern: 'azcopy_windows*'
+              Pattern: '*.exe;*.msi'
 
           - script: |
               rm -rf azCopy-windows-temp/*.md
               cp azCopy-windows-temp/* .
+              rm -rf *.msi
               mv azCopy-windows-temp $(work_dir)
             displayName: 'Add signed windows binary to staging directory'
             workingDirectory: '$(Build.ArtifactStagingDirectory)'
@@ -799,6 +841,18 @@ stages:
             inputs:
               artifactName: 'azCopy-binaries'
             displayName: 'Publish Signed Artifacts'
+
+          - script: |
+              rm -rf  $(Build.ArtifactStagingDirectory)/*
+              mv $(work_dir)/azCopy-windows-temp/*.msi .
+            displayName: 'Add signed windows binary to staging directory'
+            workingDirectory: '$(Build.ArtifactStagingDirectory)'
+
+          # Push signed msi to artifact directory
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: 'azCopy-windows-signed'
+            displayName: 'Publish Signed Artifacts'
 
           # Validate signed images have md5sum changed
           - script: |
@@ -2194,4 +2248,4 @@ stages:
                     exit 1
                   fi
                 workingDirectory: $(root_dir)
-                displayName: "Updating version number"
+                displayName: "Updating version number"

setup/create_windows_msi.yml

@@ -0,0 +1,121 @@
+parameters:
+  working_dir: ''
+  build_dir: ''
+  azcopy_version: ''
+  azcopy_msi_name: ''
+
+steps:
+- powershell: |
+    # Check if Chocolatey is installed
+    if (!(Get-Command choco.exe -ErrorAction SilentlyContinue)) {
+        Write-Host "Chocolatey is not installed. Installing Chocolatey..."
+        Set-ExecutionPolicy Bypass -Scope Process -Force; 
+        [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; 
+        Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1'))
+    } else {
+        Write-Host "Chocolatey is already installed."
+    }
+
+    # Install WiX Toolset using Chocolatey
+    Write-Host "Installing WiX Toolset via Chocolatey..."
+    choco install wixtoolset -y
+    Write-Host "WiX Toolset installation completed."
+  displayName: 'Install Chocolatey and WiX Toolset'
+
+- powershell: |
+    # Script to create MSI in an Azure Pipeline
+
+    # Step 1: Set Variables
+    $workDir = "${{ parameters.working_dir }}"
+    $wixToolsPath = (where.exe candle.exe) -replace '\\candle.exe$', ''  # Remove candle.exe from path
+    $wxsFilePath = "$workDir\azcopy.wxs"
+    $wixObjPath = "$workDir\azcopy.wixobj"
+    $outputMsi = "${{ parameters.build_dir }}\${{ parameters.azcopy_msi_name }}"
+
+    # Step 2: Update the fields in the .wxs file
+    Write-Host "Updating Uuid in .wxs file..."
+    $uuid = [guid]::NewGuid().Guid.ToString()
+
+    # Load the XML file with namespace handling
+    [xml]$wixXml = Get-Content $wxsFilePath
+
+    # Define the XML namespace manager
+    $namespaceManager = New-Object System.Xml.XmlNamespaceManager($wixXml.NameTable)
+    $namespaceManager.AddNamespace("wix", "http://schemas.microsoft.com/wix/2006/wi")
+
+    # Update fields using the namespace
+    $productNode = $wixXml.SelectSingleNode("//wix:Product", $namespaceManager)
+    $productNode.Name = "AzCopy v10"
+    $productNode.Version = "${{ parameters.azcopy_version }}"
+    $productNode.Manufacturer = "Updated Manufacturer"
+    $productNode.UpgradeCode = $uuid
+
+    # Locate the Component node using the namespace
+    $component = $wixXml.SelectSingleNode("//wix:Directory[@Id='INSTALLDIR']/wix:Component", $namespaceManager)
+
+    # Check if Component element exists
+    if ($component) {
+        # Check if Guid attribute exists and set it, or add it if missing
+        if ($component.Attributes["Guid"]) {
+            $component.Attributes["Guid"].Value = $uuid
+        } else {
+            $component.SetAttribute("Guid", $uuid)
+        }
+        Write-Host "Guid attribute updated successfully."
+    } else {
+        Write-Host "Component element not found in XML structure."
+    }
+
+    # Update the file source path
+    $file = $component.SelectSingleNode("wix:File", $namespaceManager)
+    $file.Source = "$workDir\azcopy.exe" 
+
+    # Save the updated XML back to the file
+    $wixXml.Save($wxsFilePath)
+
+    # Print the updated content of the .wxs file
+    Write-Host "Updated content of the .wxs file:"
+    Get-Content $wxsFilePath
+
+    Write-Host "WiX XML file updated successfully."
+
+    # Step 3: Compile the .wxs file using Candle
+    Write-Host "Compiling .wxs file..."
+    & "$wixToolsPath\candle.exe" -out $wixObjPath $wxsFilePath
+
+    # Step 4: Link the .wixobj to create the MSI using Light
+    Write-Host "Linking to create MSI..."
+    & "$wixToolsPath\light.exe" -out $outputMsi $wixObjPath
+
+    Write-Host "MSI creation process completed!"
+
+    # Define the file path for the .wixpdb file
+    $filePath = "$outputMsi\*.wixpdb"
+
+    # Remove the azcopy.wixpdb file as it is not required.
+    # .wixpdb file is a debugging file generated by WiX Toolset.
+    # Check if the file exists
+    if (Test-Path -Path $filePath) {
+        # Remove the .wixpdb file if it exists
+        Remove-Item -Path $filePath -Force
+        Write-Host "The .wixpdb file has been deleted."
+    } else {
+        Write-Host "No .wixpdb file found to delete."
+    }
+
+    # Verify the file has been deleted
+    if (-not (Test-Path -Path $filePath)) {
+        Write-Host "File deletion confirmed."
+    } else {
+        Write-Host "File still exists."
+    }
+
+    # Step 5: Install the MSI and verify the installation
+    Write-Host "Installing the MSI for verification ..."
+    msiexec /i $outputMsi
+    if ($LASTEXITCODE -eq 0) {
+    Write-Host "Installation was successful."
+    } else {
+        Write-Host "Installation failed with exit code $LASTEXITCODE."
+    }
+    azcopy.exe --version

setup/esrp_sign_windows.yml

@@ -1,7 +1,7 @@
 parameters:
   FolderPath: ''
   ESRP_AZCOPY_WIN_KEY_CODE: ''
-  Pattern: '*.exe'
+  Pattern: ''
 
 steps:
 - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@5

ste/md5Comparer.go

@@ -23,6 +23,7 @@ package ste
 import (
 	"bytes"
 	"errors"
+
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
 
@@ -64,7 +65,7 @@ func (c *md5Comparer) Check() error {
 		switch c.validationOption {
 		// This code would never be triggered anymore due to the early check that now occurs in xfer-remoteToLocal.go
 		case common.EHashValidationOption.FailIfDifferentOrMissing():
-			panic("Transfer should've pre-emptively failed with a missing MD5.")
+			panic("Transfer should've preemptively failed with a missing MD5.")
 		case common.EHashValidationOption.FailIfDifferent(),
 			common.EHashValidationOption.LogOnly():
 			c.logAsMissing()

ste/pageRangeOptimizer.go

@@ -22,16 +22,17 @@ package ste
 
 import (
 	"context"
+	"strings"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/pageblob"
 	"github.com/Azure/azure-storage-azcopy/v10/common"
-	"strings"
 )
 
 // isolate the logic to fetch page ranges for a page blob, and check whether a given range has data
 // for two purposes:
 //  1. capture the necessary info to do so, so that fetchPages can be invoked anywhere
-//  2. open to extending the logic, which could be re-used for both download and s2s scenarios
+//  2. open to extending the logic, which could be reused for both download and s2s scenarios
 type pageRangeOptimizer struct {
 	srcPageBlobClient *pageblob.Client
 	ctx               context.Context