feat: add containerd2 to windows

e2e/windows/README.md

@@ -77,9 +77,17 @@ windows 23H2 gen1 and windows 23H2 gen2.
 
 ## Generate and Update current windows test images
 
-Currently, we use the latest test images "/subscriptions/$AZURE_E2E_IMAGE_SUBSCRIPTION_ID/resourceGroups/$AZURE_E2E_IMAGE_RESOURCE_GROUP_NAME/providers/Microsoft.Compute/galleries/$AZURE_E2E_IMAGE_GALLERY_NAME/images/windows-$WINDOWS_E2E_IMAGE/versions/latest" (
-$WINDOWS_E2E_IMAGE could be 2019-containerd, 2022-containerd, 2022-containerd-gen2, 23H2, 23H2-gen2) as is referred
-by `IMAGE_REFERENCE` in `e2e-scenario.sh`.
+Currently, we use the latest test images `/subscriptions/$AZURE_E2E_IMAGE_SUBSCRIPTION_ID/resourceGroups/$AZURE_E2E_IMAGE_RESOURCE_GROUP_NAME/providers/Microsoft.Compute/galleries/$AZURE_E2E_IMAGE_GALLERY_NAME/images/windows-$WINDOWS_E2E_IMAGE/versions/latest`.
+
+supported versions of `$WINDOWS_E2E_IMAGE` are
+
+- 2019-containerd, 
+- 2022-containerd, 
+- 2022-containerd-gen2, 
+- 23H2, 
+- 23H2-gen2
+
+as is referred by `IMAGE_REFERENCE` in `e2e-scenario.sh`.
 
 To generate new windows test images, we can
 use [`[TEST All VHDs] AKS Windows VHD Build - Msft Tenant`](https://msazure.visualstudio.com/CloudNativeCompute/_build?definitionId=210712&_a=summary).
@@ -100,7 +108,13 @@ To obtain an example template, there are two ways:
   Enter MC resource group -> Enter the existing vmss -> Click `Export template` on the left bar. Then you can see the
   example template.
 - **From az cli**:
-    1. VMSS_RESOURCE_Id=$(az resource show --resource-group $MC_RESOURCE_GROUP_NAME --name $MC_WIN_VMSS_NAME
-       --resource-type Microsoft.Compute/virtualMachineScaleSets --query id --output tsv)
-    1. az group export --resource-group $MC_RESOURCE_GROUP_NAME --resource-ids $VMSS_RESOURCE_Id
-       --include-parameter-default-value > template.json
+  ```
+  VMSS_RESOURCE_Id=$(az resource show --resource-group $MC_RESOURCE_GROUP_NAME \
+  --name $MC_WIN_VMSS_NAME  \
+  --resource-type Microsoft.Compute/virtualMachineScaleSets \
+  --query id --output tsv)
+
+  az group export --resource-group $MC_RESOURCE_GROUP_NAME  \ 
+  --resource-ids $VMSS_RESOURCE_Id \
+  --include-parameter-default-value > template.json
+  ```

parts/windows/windowscsehelper.ps1

@@ -153,10 +153,13 @@ $global:ErrorCodeNames = @(
 
 # NOTE: KubernetesVersion does not contain "v"
 $global:MinimalKubernetesVersionWithLatestContainerd = "1.28.0" # Will change it to the correct version when we support new Windows containerd version
+$global:MinimalKubernetesVersionWithLatestContainerd2 = "1.32.0" # Will change it to the correct version when we support new Windows containerd version
 # Although the contianerd package url is set in AKS RP code now, we still need to update the following variables for AgentBaker Windows E2E tests.
 $global:StableContainerdPackage = "v1.6.35-azure.1/binaries/containerd-v1.6.35-azure.1-windows-amd64.tar.gz"
 # The latest containerd version
 $global:LatestContainerdPackage = "v1.7.20-azure.1/binaries/containerd-v1.7.20-azure.1-windows-amd64.tar.gz"
+# The latest containerd2 version
+$global:LatestContainerd2Package = "v2.0.1-azure.1/binaries/containerd-v2.0.1-azure.1-windows-amd64.tar.gz"
 
 $global:EventsLoggingDir = "C:\WindowsAzure\Logs\Plugins\Microsoft.Compute.CustomScriptExtension\Events\"
 $global:TaskName = ""
@@ -412,7 +415,10 @@ function Install-Containerd-Based-On-Kubernetes-Version {
   if ($ContainerdUrl.EndsWith("/")) {
     Write-Log "ContainerdURL is $ContainerdUrl"
     $containerdPackage=$global:StableContainerdPackage
-    if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) {
+    if (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd2) -ge 0) {
+        $containerdPackage = $global:LatestContainerd2Package
+        Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd2 so the latest containerd version 2.x $containerdPackage is used"
+    } elseif (([version]$KubernetesVersion).CompareTo([version]$global:MinimalKubernetesVersionWithLatestContainerd) -ge 0) {
         $containerdPackage=$global:LatestContainerdPackage
         Write-Log "Kubernetes version $KubernetesVersion is greater than or equal to $global:MinimalKubernetesVersionWithLatestContainerd so the latest containerd version $containerdPackage is used"
     } else {

parts/windows/windowscsehelper.tests.ps1

@@ -39,6 +39,12 @@ Describe 'Install-Containerd-Based-On-Kubernetes-Version' {
     Assert-MockCalled -CommandName "Install-Containerd" -Exactly -Times 1 -ParameterFilter { $ContainerdUrl -eq $expectedURL }
   }
 
+  It 'k8s version is equal to MinimalKubernetesVersionWithContainerd2' {
+    $expectedURL = "https://acs-mirror.azureedge.net/containerd/windows/" + $global:LatestContainerd2Package
+    & Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl "https://acs-mirror.azureedge.net/containerd/windows/" -KubernetesVersion "1.32.0" -CNIBinDir "cniBinPath" -CNIConfDir "cniConfigPath" -KubeDir "kubeDir"
+    Assert-MockCalled -CommandName "Install-Containerd" -Exactly -Times 1 -ParameterFilter { $ContainerdUrl -eq $expectedURL }
+  }
+
   It 'full URL is set' {
     $expectedURL = "https://privatecotnainer.com/windows-containerd-v1.2.3.tar.gz"
     & Install-Containerd-Based-On-Kubernetes-Version -ContainerdUrl "https://privatecotnainer.com/windows-containerd-v1.2.3.tar.gz" -KubernetesVersion "1.26.1" -CNIBinDir "cniBinPath" -CNIConfDir "cniConfigPath" -KubeDir "kubeDir"

staging/cse/windows/README → staging/cse/windows/README.md

@@ -1,6 +1,7 @@
 # Unit Tests
-1. Execute unit tests on pipeline. The yaml file of unit test pipeline is .github/workflows/validate-windows-ut.yml.
-1. Execute unit tests locally:
+
+- Execute unit tests on pipeline. The yaml file of unit test pipeline is `.github/workflows/validate-windows-ut.yml`.
+- Execute unit tests locally:
   1. Please install [pester](https://github.com/pester/Pester) with the command `Install-Module -Name pester -SkipPublisherCheck -Force`.
     - Doc: https://pester.dev/docs/quick-start
     - Courses: https://docs.microsoft.com/en-us/shows/testing-powershell-with-pester/
@@ -10,7 +11,9 @@
     - `Invoke-Pester staging/cse/windows/*.tests.ps1 -Passthru`
 
 # Test with AKS RP
-1. Run below commands to build a test package
+
+- Run below commands to build a test package
+
 ```bash
 branchName="master"
 currentCseVersion="v0.0.50" # `WindowsCSEScriptsPackage` defined in `parts/windows/kuberneteswindowssetup.ps1`
@@ -21,42 +24,43 @@ mkdir -p temp-work-folder/aks-windows-cse
 cd temp-work-folder
 curl -O "https://acs-mirror.azureedge.net/aks/windows/cse/aks-windows-cse-scripts-$currentCseVersion.zip"
 pushd aks-windows-cse
-    unzip ../aks-windows-cse-scripts-*.zip
-    rm ../*.zip
-
-	files=("azurecnifunc.ps1" "calicofunc.ps1" "configfunc.ps1" "containerdfunc.ps1" "containerdtemplate.toml" "kubeletfunc.ps1" "kubernetesfunc.ps1" "nvidiagpudriverfunc.ps1")
-	for file in ${files[@]}; do
-	    echo "Downloading $file from $url/$file"
-	   curl -O "$url/$file"
-	done
-
-	pushd debug
-		files=("VFP.psm1" "captureNetworkFlows.ps1" "collect-windows-logs.ps1" "collectlogs.ps1" "dumpVfpPolicies.ps1" "helper.psm1" "hns.psm1" "hns.v2.psm1" "networkhealth.ps1" "portReservationTest.ps1" "starthnstrace.cmd" "starthnstrace.ps1" "startpacketcapture.cmd" "startpacketcapture.ps1" "stoppacketcapture.cmd")
-		for file in ${files[@]}; do
-		    echo "Downloading $file from $url/debug/$file"
-		   curl -O "$url/debug/$file"
-		done
-	popd
-	
-	pushd provisioningscripts
-		files=("cleanupnetwork.ps1" "hnsremediator.ps1" "hostsconfigagent.ps1" "kubeletstart.ps1" "kubeproxystart.ps1" "loggenerator.ps1" "windowslogscleanup.ps1" "windowsnodereset.ps1" "windowssecuretls.ps1")
-		for file in ${files[@]}; do
-		    echo "Downloading $file from $url/provisioningscripts/$file"
-		   curl -O "$url/provisioningscripts/$file"
-		done
-	popd
-
-	zip -r "../aks-windows-cse-scripts-$testCseVersion.zip" *
+  unzip ../aks-windows-cse-scripts-*.zip
+  rm ../*.zip
+  
+  files=("azurecnifunc.ps1" "calicofunc.ps1" "configfunc.ps1" "containerdfunc.ps1" "containerdtemplate.toml" "kubeletfunc.ps1" "kubernetesfunc.ps1" "nvidiagpudriverfunc.ps1")
+  for file in ${files[@]}; do
+    echo "Downloading $file from $url/$file"
+    curl -O "$url/$file"
+  done
+  
+  pushd debug
+    files=("VFP.psm1" "captureNetworkFlows.ps1" "collect-windows-logs.ps1" "collectlogs.ps1" "dumpVfpPolicies.ps1" "helper.psm1" "hns.psm1" "hns.v2.psm1" "networkhealth.ps1" "portReservationTest.ps1" "starthnstrace.cmd" "starthnstrace.ps1" "startpacketcapture.cmd" "startpacketcapture.ps1" "stoppacketcapture.cmd")
+    for file in ${files[@]}; do
+      echo "Downloading $file from $url/debug/$file"
+      curl -O "$url/debug/$file"
+    done
+  popd
+  
+  pushd provisioningscripts
+    files=("cleanupnetwork.ps1" "hnsremediator.ps1" "hostsconfigagent.ps1" "kubeletstart.ps1" "kubeproxystart.ps1" "loggenerator.ps1" "windowslogscleanup.ps1" "windowsnodereset.ps1" "windowssecuretls.ps1")
+    for file in ${files[@]}; do
+      echo "Downloading $file from $url/provisioningscripts/$file"
+      curl -O "$url/provisioningscripts/$file"
+    done
+  popd
+  
+  zip -r "../aks-windows-cse-scripts-$testCseVersion.zip" *
 popd
 ```
-1. Upload the test package to your Azure storage container `$web`
+
+- Upload the test package to your Azure storage container `$web`
   - You need to enable `Static website` in your storage account on Azure portal
   - You need to note down your `Primary endpoint` of `Static website` of your storage account
   - The test package download URL should be `[Primary endpoint]/aks-windows-cse-scripts-[$testCseVersion].zip`
-1. Test it with AKS RP in the toggle `windows-cse-package-url.yaml`
+- Test it with AKS RP in the toggle `windows-cse-package-url.yaml`
 
 # AKS Windows CSE Scripts Package
-All files except *.test.ps1 and README will be published in AKS Windows CSE Scripts Package.
+All files except `*.test.ps1` and `README.md` will be published in AKS Windows CSE Scripts Package.
 
 ## v0.0.50
 - fix: do not use return in updating kubelet node labels #5151

vhdbuilder/packer/generate-windows-vhd-configuration.ps1

@@ -160,12 +160,13 @@ $global:map = @{
     # Different from other packages which are downloaded/cached and used later only during CSE, windows containerd is installed
     # during building the Windows VHD to cache container images.
     # We use the latest containerd package to start containerd then cache images, and the latest one is expected to be
-    # specified by AKS PR for most of the cases. BUT as long as there's a new unpacked image version, we should keep the
+    # specified by AKS RP for most of the cases. BUT as long as there's a new unpacked image version, we should keep the
     # versions synced.
     "c:\akse-cache\containerd\"   = @(
         $defaultContainerdPackageUrl,
         "https://acs-mirror.azureedge.net/containerd/windows/v1.7.17-azure.1/binaries/containerd-v1.7.17-azure.1-windows-amd64.tar.gz",
-        "https://acs-mirror.azureedge.net/containerd/windows/v1.7.20-azure.1/binaries/containerd-v1.7.20-azure.1-windows-amd64.tar.gz"
+        "https://acs-mirror.azureedge.net/containerd/windows/v1.7.20-azure.1/binaries/containerd-v1.7.20-azure.1-windows-amd64.tar.gz",
+        "https://mobyartifacts.azureedge.net/moby/moby-containerd/2.0.1+azure/windows/windows_amd64/moby-containerd-2.0.1+azure-u1.amd64.zip"
     );
     "c:\akse-cache\csi-proxy\"    = @(
         "https://acs-mirror.azureedge.net/csi-proxy/v1.1.2-hotfix.20230807/binaries/csi-proxy-v1.1.2-hotfix.20230807.tar.gz"