diff --git a/scripts/azure-pipelines/android/Dockerfile b/.github/workflows/android/Dockerfile similarity index 100% rename from scripts/azure-pipelines/android/Dockerfile rename to .github/workflows/android/Dockerfile diff --git a/scripts/azure-pipelines/android/create-docker-image.ps1 b/.github/workflows/android/create-docker-image.ps1 similarity index 94% rename from scripts/azure-pipelines/android/create-docker-image.ps1 rename to .github/workflows/android/create-docker-image.ps1 index 1bb0ee4293e59d..496617c8f01681 100644 --- a/scripts/azure-pipelines/android/create-docker-image.ps1 +++ b/.github/workflows/android/create-docker-image.ps1 @@ -2,7 +2,7 @@ $Date = (Get-Date -Format 'yyyy-MM-dd') $ResourceGroupName = "PrAnd-1ES" -$ContainerRegistryName = "vcpkgandroidwus3" +$ContainerRegistryName = "vcpkgandroidwus" $ErrorActionPreference = 'Stop' $registry = Get-AzContainerRegistry -ResourceGroupName $ResourceGroupName -Name $ContainerRegistryName diff --git a/scripts/azure-pipelines/android/example-ubuntu-provision.sh b/.github/workflows/android/example-ubuntu-provision.sh similarity index 100% rename from scripts/azure-pipelines/android/example-ubuntu-provision.sh rename to .github/workflows/android/example-ubuntu-provision.sh diff --git a/scripts/azure-pipelines/linux/changing-linux-packages.md b/.github/workflows/linux/changing-linux-packages.md similarity index 100% rename from scripts/azure-pipelines/linux/changing-linux-packages.md rename to .github/workflows/linux/changing-linux-packages.md diff --git a/scripts/azure-pipelines/linux/managed-image.json b/.github/workflows/linux/managed-image.json similarity index 100% rename from scripts/azure-pipelines/linux/managed-image.json rename to .github/workflows/linux/managed-image.json diff --git a/scripts/azure-pipelines/linux/provision-image.sh b/.github/workflows/linux/provision-image.sh old mode 100755 new mode 100644 similarity index 100% rename from scripts/azure-pipelines/linux/provision-image.sh rename to .github/workflows/linux/provision-image.sh diff --git a/scripts/azure-pipelines/patch-tuesday-checklist.md b/.github/workflows/patch-tuesday-checklist.md similarity index 91% rename from scripts/azure-pipelines/patch-tuesday-checklist.md rename to .github/workflows/patch-tuesday-checklist.md index 11ca3f0f823a27..a65f09857db954 100644 --- a/scripts/azure-pipelines/patch-tuesday-checklist.md +++ b/.github/workflows/patch-tuesday-checklist.md @@ -9,7 +9,7 @@ "C++ VCPKG Validation" about vulnerable software we are installing in the VMs and update that. (Most often PowerShell needs to be updated) * [ ] Check for any other software for the Windows images we wish to update and make the edits to do - so in `scripts/azure-pipelines/windows` + so in `.github/windows` * [ ] Check for any updates possible to `vcpkgTools.xml`. Note that PowerShell currently uses the 7.2.x series due to customer reported problems on older Windows with 7.3.x and later. * [ ] Update the first line of android/Dockerfile with the current 'focal' image according to @@ -22,4 +22,4 @@ * [ ] In the Azure portal, mark the newly created image as the 'latest' one. * [ ] Mint a new macOS base box. (See instructions in `scripts/azure-pipelines/osx/README.md`) * [ ] Deploy the new base box to all hosts. -* [ ] Update the software on the CTI's machine #12 to match. +* [ ] Update the software on the CTI's machine #1 to match. diff --git a/.github/workflows/test_modified_ports.yml b/.github/workflows/test_modified_ports.yml index fa1d8f68a6527e..d3dbc998d8d2de 100644 --- a/.github/workflows/test_modified_ports.yml +++ b/.github/workflows/test_modified_ports.yml @@ -3,21 +3,206 @@ on: push: branches: - onboard-actions + pull_request_target: + branches: + - onboard-actions + schedule: + - cron: "0 4 * * 1,3,5" +permissions: + # IMPORTANT + # https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/ + # This workflow is configured to run on `pull_request_target`, which is necessary to have access + # to id-token: write, which we need to authenticate to the Azure Storage account needed for + # vcpkg's binary caching practical. + # However, this means that build scripts of ports, which consist of 3rd party build scripts, + # effectively have permssions granted here. + # contents: read *must* be kept to prevent those untrusted build scripts writing to + # github.com/microsoft/vcpkg. + # On the Azure side, we assume that untrusted code has control of the 'VcpkgPrFleet' identity + # we use to authenticate with Storage, firewall the accounts to be accessible only to the + # hardware and VMs that run our PRs, and forbid interacting with any of those components using + # corporate credentials. (For instance, the VMs where PRs are built cannot be logged into at all) + id-token: write + contents: read # Do *NOT* change to write jobs: - Windows: + test: + strategy: + matrix: + conf: + - triplet: 'arm64-windows' + pool: windows + - triplet: 'arm64-windows-static-md' + pool: windows + - triplet: 'arm64-uwp' + pool: windows + - triplet: 'x86-windows' + pool: windows + extra-checks: true + - triplet: 'x64-windows' + pool: windows + - triplet: 'x64-windows-static' + pool: windows + - triplet: 'x64-windows-static-md' + pool: windows + - triplet: 'x64-uwp' + pool: windows + - triplet: 'x64-linux' + pool: linux + - triplet: 'arm-neon-android' + pool: android + - triplet: 'x64-android' + pool: android + - triplet: 'arm64-android' + pool: android runs-on: - self-hosted - - "1ES.Pool=vcpkg-windows-wus" + - "1ES.Pool=${{ matrix.conf.pool == 'windows' && 'vcpkg-windows-ephemeral-wus' || matrix.conf.pool == 'linux' && 'vcpkg-linux-ephemeral-wus' || matrix.conf.pool == 'android' && 'vcpkg-android-ephemeral-wus' }}" + continue-on-error: true + timeout-minutes: 2880 # 2 days + env: + VCPKG_DOWNLOADS: ${{ matrix.conf.pool == 'windows' && 'D:\downloads' || '/mnt/vcpkg-ci/downloads' }} + WORKING_ROOT: ${{ matrix.conf.pool == 'windows' && 'D:\' || '/mnt/vcpkg-ci' }} + ARTIFACT_STAGING: ${{ matrix.conf.pool == 'windows' && 'D:\artifactstaging' || '/mnt/vcpkg-ci/artifactstaging' }} steps: - - name: Checkout + - name: Make Working Directories (Windows) + if: ${{ success() && matrix.conf.pool == 'windows' }} + run: | + mkdir D:\downloads + mkdir D:\artifactstaging + - name: Make Working Directories (non-Windows) + if: ${{ success() && matrix.conf.pool != 'windows' }} + run: | + sudo mkdir ${{ env.WORKING_ROOT }} -m=777 + sudo mkdir ${{ env.WORKING_ROOT }}/failure-logs -m=777 + sudo mkdir ${{ env.VCPKG_DOWNLOADS }} -m=777 + sudo mkdir ${{ env.ARTIFACT_STAGING }} -m=777 + - name: Checkout (Windows) uses: actions/checkout@v4 + if: ${{ success() && matrix.conf.pool == 'windows' }} + env: + PATH: c:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer\Git\cmd;$PATH with: # fetch-depth 50 tries to ensure we capture the whole history of the branch fetch-depth: 50 + ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout (Non-Windows) + uses: actions/checkout@v4 + if: ${{ success() && matrix.conf.pool != 'windows' }} + with: + # fetch-depth 50 tries to ensure we capture the whole history of the branch + fetch-depth: 50 + ref: ${{ github.event.pull_request.head.sha }} + - name: Azure Login + uses: azure/login@v2 + with: + client-id: a81b4cd3-9d8d-4cb9-9a74-f2038f24f224 + subscription-id: 7fcb00fa-a761-49de-8a2f-d67190e62882 + tenant-id: 72f988bf-86f1-41af-91ab-2d7cd011db47 + - name: Bootstrap (Windows) + if: ${{ success() && matrix.conf.pool == 'windows' }} + run: ./scripts/bootstrap.ps1 + - name: Bootstrap (Non-Windows) + if: ${{ success() && matrix.conf.pool != 'windows' && matrix.conf.pool != 'android' }} + run: ./scripts/bootstrap.sh + - name: Format Manifests + if: ${{ success() && matrix.conf.extra-checks }} + run: ./vcpkg.exe format-manifest --all + - name: Create Format Manifest Diff + if: ${{ success() && matrix.conf.extra-checks }} + env: + PATH: c:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer\Git\cmd;$PATH + run: ./scripts/azure-pipelines/Create-PRDiff.ps1 -DiffFile "${{ env.ARTIFACT_STAGING }}/format.diff" + - name: 'Publish Artifact: Format Diff' + uses: actions/upload-artifact@v4 + if: ${{ failure() && !cancelled() && matrix.conf.extra-checks }} + with: + name: 'format.diff' + path: "${{ env.ARTIFACT_STAGING }}/format.diff" + retention-days: 7 + if-no-files-found: error + - name: "*** Test Modified Ports (non-Android)" + if: ${{ success() && matrix.conf.pool != 'android' }} + shell: pwsh + run: | + $current = Get-Date -AsUtc + $endDate = $current.AddDays(2) + $end = Get-Date -Date $endDate -UFormat '+%Y-%m-%dT%H:%MZ' + Write-Host "Getting Asset Cache SAS" + $assetSas = az storage container generate-sas --name cache --account-name vcpkgassetcachewus3 --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv | Out-String + if ($LastExitCode -ne 0) { + Write-Error "Failed to get Asset Cache SAS" + return 1 + } + + $assetSas = $assetSas.Trim() + Write-Host "Getting Binary Cache SAS" + $binarySas = az storage container generate-sas --name cache --account-name vcpkgbinarycachewus --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv | Out-String + if ($LastExitCode -ne 0) { + Write-Error "Failed to get Binary Cache SAS" + return 1 + } - - name: Bootstrap - run: ./bootstrap-vcpkg.sh - - - name: Example + $binarySas = $binarySas.Trim() + $env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkgassetcachewus3.blob.core.windows.net/cache,$assetSas,readwrite" + if ($IsWindows) { # FIXME: Git in the images + $env:PATH += ";c:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer\Git\cmd" + } + + & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ matrix.conf.triplet }} -BuildReason ${{ github.event_name }} -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $env:ARTIFACT_STAGING + - name: "*** Test Modified Ports (Android)" + if: ${{ success() && matrix.conf.pool == 'android' }} + run: | + az acr login --name vcpkgandroidwus + docker pull vcpkgandroidwus.azurecr.io/vcpkg-android:2024-10-21 + docker run --rm \ + --mount type=bind,source=${{ github.workspace }},target=/vcpkg \ + vcpkgandroidwus.azurecr.io/vcpkg-android:2024-10-21 \ + ./bootstrap-vcpkg.sh + end=`date -u -d "2 days" '+%Y-%m-%dT%H:%MZ'` + assetSas=`az storage container generate-sas --name cache --account-name vcpkgassetcachewus3 --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv` + binarySas=`az storage container generate-sas --name cache --account-name vcpkgbinarycachewus --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv` + echo Minting SAS tokens valid through $end + USER=$(id --user) + docker run --init -i \ + -a stderr \ + -a stdout \ + --user $USER \ + --mount type=bind,source=${{ github.workspace }},target=/vcpkg \ + --mount type=bind,source=/mnt/vcpkg-ci/failure-logs,target=/vcpkg/failure-logs \ + --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \ + --env X_VCPKG_ASSET_SOURCES="x-azurl,https://vcpkgassetcachewus3.blob.core.windows.net/cache,$assetSas,readwrite" \ + vcpkgandroidwus.azurecr.io/vcpkg-android:2024-10-21 \ + pwsh \ + -File /vcpkg/scripts/azure-pipelines/test-modified-ports.ps1 \ + -Triplet ${{ matrix.conf.triplet }} \ + -BuildReason ${{ github.event_name }} \ + -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus.blob.core.windows.net/cache,$binarySas" \ + -WorkingRoot /mnt/vcpkg-ci \ + -ArtifactStagingDirectory /mnt/vcpkg-ci/artifactstaging + - name: 'Validate version files' + if: ${{ success() && matrix.conf.extra-checks }} + env: + PATH: c:\Program Files\Microsoft Visual Studio\2022\Enterprise\Common7\IDE\CommonExtensions\Microsoft\TeamFoundation\Team Explorer\Git\cmd;$PATH + run: 'scripts/azure-pipelines/windows/validate-version-files.ps1' + - name: 'Publish Artifact: failure logs for ${{ matrix.conf.triplet }}' + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: 'failure logs for ${{ matrix.conf.triplet }}' + path: '${{ env.ARTIFACT_STAGING }}/failure-logs' + retention-days: 7 + if-no-files-found: ignore + - name: 'Build a file list for all packages' + if: ${{ !cancelled() }} shell: pwsh - run: Write-Host 'Running on 1ES Hosted GitHub Runners' + run: | + ./vcpkg fetch python3 + & $(./vcpkg fetch python3) ./scripts/file_script.py "$env:WORKING_ROOT/installed/vcpkg/info/" + - name: 'Publish Artifact: file lists for ${{ matrix.conf.triplet }}' + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: 'file lists for ${{ matrix.conf.triplet }}' + path: scripts/list_files + retention-days: 7 + if-no-files-found: ignore diff --git a/scripts/azure-pipelines/windows/bootstrap-from-source.cmd b/.github/workflows/windows/bootstrap-from-source.cmd similarity index 100% rename from scripts/azure-pipelines/windows/bootstrap-from-source.cmd rename to .github/workflows/windows/bootstrap-from-source.cmd diff --git a/scripts/azure-pipelines/windows/create-image.ps1 b/.github/workflows/windows/create-image.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/create-image.ps1 rename to .github/workflows/windows/create-image.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-azure-cli.ps1 b/.github/workflows/windows/deploy-azure-cli.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-azure-cli.ps1 rename to .github/workflows/windows/deploy-azure-cli.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-cuda.ps1 b/.github/workflows/windows/deploy-cuda.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-cuda.ps1 rename to .github/workflows/windows/deploy-cuda.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-cudnn.ps1 b/.github/workflows/windows/deploy-cudnn.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-cudnn.ps1 rename to .github/workflows/windows/deploy-cudnn.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-install-disk.ps1 b/.github/workflows/windows/deploy-install-disk.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-install-disk.ps1 rename to .github/workflows/windows/deploy-install-disk.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 b/.github/workflows/windows/deploy-inteloneapi.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-inteloneapi.ps1 rename to .github/workflows/windows/deploy-inteloneapi.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-mpi.ps1 b/.github/workflows/windows/deploy-mpi.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-mpi.ps1 rename to .github/workflows/windows/deploy-mpi.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-pwsh.ps1 b/.github/workflows/windows/deploy-pwsh.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-pwsh.ps1 rename to .github/workflows/windows/deploy-pwsh.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-settings.txt b/.github/workflows/windows/deploy-settings.txt similarity index 100% rename from scripts/azure-pipelines/windows/deploy-settings.txt rename to .github/workflows/windows/deploy-settings.txt diff --git a/scripts/azure-pipelines/windows/deploy-tlssettings.ps1 b/.github/workflows/windows/deploy-tlssettings.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-tlssettings.ps1 rename to .github/workflows/windows/deploy-tlssettings.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-visual-studio.ps1 b/.github/workflows/windows/deploy-visual-studio.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-visual-studio.ps1 rename to .github/workflows/windows/deploy-visual-studio.ps1 diff --git a/scripts/azure-pipelines/windows/deploy-windows-sdks.ps1 b/.github/workflows/windows/deploy-windows-sdks.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/deploy-windows-sdks.ps1 rename to .github/workflows/windows/deploy-windows-sdks.ps1 diff --git a/scripts/azure-pipelines/windows/disk-space.ps1 b/.github/workflows/windows/disk-space.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/disk-space.ps1 rename to .github/workflows/windows/disk-space.ps1 diff --git a/scripts/azure-pipelines/windows/provision-entire-image.ps1 b/.github/workflows/windows/provision-entire-image.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/provision-entire-image.ps1 rename to .github/workflows/windows/provision-entire-image.ps1 diff --git a/scripts/azure-pipelines/windows/sysprep.ps1 b/.github/workflows/windows/sysprep.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/sysprep.ps1 rename to .github/workflows/windows/sysprep.ps1 diff --git a/scripts/azure-pipelines/windows/utility-prefix.ps1 b/.github/workflows/windows/utility-prefix.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/utility-prefix.ps1 rename to .github/workflows/windows/utility-prefix.ps1 diff --git a/scripts/azure-pipelines/windows/validate-version-files.ps1 b/.github/workflows/windows/validate-version-files.ps1 similarity index 100% rename from scripts/azure-pipelines/windows/validate-version-files.ps1 rename to .github/workflows/windows/validate-version-files.ps1 diff --git a/scripts/azure-pipelines/android/azure-pipelines.yml b/scripts/azure-pipelines/android/azure-pipelines.yml deleted file mode 100644 index 3fdc9025887b3c..00000000000000 --- a/scripts/azure-pipelines/android/azure-pipelines.yml +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT -# - -parameters: - - name: jobName - type: string - - name: dockerImage - type: string - - name: tripletPattern - displayName: 'Enable the triplets which contain this substring' - type: string - default: '' - -jobs: -- job: ${{ parameters.jobName }} - condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) - pool: - name: 'PrAnd-1ES-Pool' - workspace: - clean: resources - timeoutInMinutes: 1440 # 1 day - cancelTimeoutInMinutes: 1 - variables: - - name: WORKING_ROOT - value: /mnt/vcpkg-ci - - name: VCPKG_DOWNLOADS - value: /mnt/vcpkg-ci/downloads - - name: ANDROID_NDK_HOME - value: /android-ndk-r27c - steps: - # Note: /mnt is the Azure machines' temporary disk. - - bash: | - sudo mkdir /home/agent -m=777 - sudo chown `id -u` /home/agent - sudo mkdir ${{ variables.WORKING_ROOT }} -m=777 - sudo rm -rf ${{ variables.WORKING_ROOT }}/failure-logs - sudo mkdir ${{ variables.WORKING_ROOT }}/failure-logs -m=777 - sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777 - exit 0 - displayName: 'Create working directories' - - bash: | - CONTAINERS=$(docker ps -a -q) - if [ -n "$CONTAINERS" ]; then - docker rm -f $CONTAINERS - fi - displayName: 'Stop and remove all docker containers' - - task: AzureCLI@2 - displayName: '*** Test Modified Ports' - inputs: - azureSubscription: 'VcpkgPrFleet' - scriptType: bash - scriptLocation: 'inlineScript' # Be very very careful that the exit code from the last pwsh is reported correctly - inlineScript: | - az acr login --name vcpkgandroidwus3 - docker pull ${{ parameters.dockerImage }} - docker run --rm \ - --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \ - ${{ parameters.dockerImage }} \ - ./bootstrap-vcpkg.sh - end=`date -u -d "2 days" '+%Y-%m-%dT%H:%MZ'` - assetSas=`az storage container generate-sas --name cache --account-name vcpkgassetcachewus3 --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv` - binarySas=`az storage container generate-sas --name cache --account-name vcpkgbinarycachewus3 --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv` - echo Minting SAS tokens valid through $end - USER=$(id --user) - docker run --init -i \ - -a stderr \ - -a stdout \ - --user $USER \ - --mount type=bind,source=$(Build.Repository.LocalPath),target=/vcpkg \ - --mount type=bind,source=$(WORKING_ROOT)/failure-logs,target=/vcpkg/failure-logs \ - --mount type=bind,source=/mnt/vcpkg-ci,target=/mnt/vcpkg-ci \ - --env X_VCPKG_ASSET_SOURCES="x-azurl,https://vcpkgassetcachewus3.blob.core.windows.net/cache,$assetSas,readwrite" \ - --env ANDROID_NDK_HOME="${{ variables.ANDROID_NDK_HOME }}" \ - ${{ parameters.dockerImage }} \ - pwsh \ - -File /vcpkg/scripts/azure-pipelines/test-modified-ports.ps1 \ - -Triplet ${{ replace(parameters.jobName, '_', '-') }} \ - -BuildReason $(Build.Reason) \ - -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus3.blob.core.windows.net/cache,$binarySas" \ - -WorkingRoot ${{ variables.WORKING_ROOT }} - - bash: | - CONTAINERS=$(docker ps -a -q) - if [ -n "$CONTAINERS" ]; then - docker rm -f $CONTAINERS - fi - displayName: 'Stop and remove all docker containers' - condition: always() - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - inputs: - PathtoPublish: '$(WORKING_ROOT)/failure-logs' - ArtifactName: "failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') - - bash: | - python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/ - displayName: 'Build a file list for all packages' - condition: always() - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: file lists for ${{ replace(parameters.jobName, '_', '-') }}" - condition: always() - inputs: - PathtoPublish: scripts/list_files - ArtifactName: "file lists for ${{ replace(parameters.jobName, '_', '-') }}" - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: ne(variables['XML_RESULTS_FILE'], '') - inputs: - testRunTitle: ${{ replace(parameters.jobName, '_', '-') }} - testResultsFormat: xUnit - testResultsFiles: $(XML_RESULTS_FILE) - platform: ${{ replace(parameters.jobName, '_', '-') }} diff --git a/scripts/azure-pipelines/azure-pipelines.yml b/scripts/azure-pipelines/azure-pipelines.yml index a938207de71727..917db92d900c16 100644 --- a/scripts/azure-pipelines/azure-pipelines.yml +++ b/scripts/azure-pipelines/azure-pipelines.yml @@ -1,9 +1,6 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: MIT # -variables: - linux-docker-image: 'vcpkgandroidwus3.azurecr.io/vcpkg-android:2024-10-21' - parameters: - name: vcpkgToolSha displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' @@ -15,48 +12,6 @@ parameters: default: '-' jobs: -- template: windows/azure-pipelines.yml - parameters: - jobName: x86_windows - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: x64_windows - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: x64_windows_static - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: x64_windows_static_md - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: x64_uwp - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: arm64_windows - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: windows/azure-pipelines.yml - parameters: - jobName: arm64_uwp - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - - template: osx/azure-pipelines.yml parameters: jobName: x64_osx @@ -70,27 +25,3 @@ jobs: poolName: 'PrOsx-2024-07-12-arm64' vcpkgToolSha: ${{ parameters.vcpkgToolSha }} tripletPattern: ${{ parameters.tripletPattern }} - -- template: linux/azure-pipelines.yml - parameters: - jobName: x64_linux - vcpkgToolSha: ${{ parameters.vcpkgToolSha }} - tripletPattern: ${{ parameters.tripletPattern }} - -- template: android/azure-pipelines.yml - parameters: - jobName: arm_neon_android - dockerImage: $(linux-docker-image) - tripletPattern: ${{ parameters.tripletPattern }} - -- template: android/azure-pipelines.yml - parameters: - jobName: x64_android - dockerImage: $(linux-docker-image) - tripletPattern: ${{ parameters.tripletPattern }} - -- template: android/azure-pipelines.yml - parameters: - jobName: arm64_android - dockerImage: $(linux-docker-image) - tripletPattern: ${{ parameters.tripletPattern }} diff --git a/scripts/azure-pipelines/linux/azure-pipelines.yml b/scripts/azure-pipelines/linux/azure-pipelines.yml deleted file mode 100644 index 352381aaff4d0a..00000000000000 --- a/scripts/azure-pipelines/linux/azure-pipelines.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT -# - -parameters: - - name: vcpkgToolSha - displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' - type: string - default: 'use default' - - name: jobName - type: string - default: 'x64_linux' - - name: tripletPattern - displayName: 'Enable the triplets which contain this substring' - type: string - default: '' - -jobs: -- job: x64_linux - condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) - pool: - name: PrLin-1ES-Pool - workspace: - clean: resources - timeoutInMinutes: 1440 # 1 day - variables: - - name: WORKING_ROOT - value: /mnt/vcpkg-ci - - name: VCPKG_DOWNLOADS - value: /mnt/vcpkg-ci/downloads - steps: - # Note: /mnt is the Azure machines' temporary disk. - - bash: | - sudo mkdir /home/agent -m=777 - sudo chown `id -u` /home/agent - sudo mkdir ${{ variables.WORKING_ROOT }} -m=777 - sudo mkdir ${{ variables.VCPKG_DOWNLOADS }} -m=777 - exit 0 - displayName: 'Create working directories' - - bash: ./bootstrap-vcpkg.sh - displayName: 'Bootstrap vcpkg' - condition: eq('use default', '${{ parameters.vcpkgToolSha }}') - - bash: ./scripts/azure-pipelines/bootstrap-from-source.sh ${{ parameters.vcpkgToolSha }} - displayName: "Build vcpkg with CMake" - condition: ne('use default', '${{ parameters.vcpkgToolSha }}') - - task: AzureCLI@2 - displayName: '*** Test Modified Ports' - inputs: - azureSubscription: 'VcpkgPrFleet' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $current = Get-Date -AsUtc - $endDate = $current.AddDays(2) - $end = Get-Date -Date $endDate -UFormat '+%Y-%m-%dT%H:%MZ' - $assetSas = az storage container generate-sas --name cache --account-name vcpkgassetcachewus3 --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv | Out-String - $assetSas = $assetSas.Trim() - $binarySas = az storage container generate-sas --name cache --account-name vcpkgbinarycachewus3 --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv | Out-String - $binarySas = $binarySas.Trim() - $env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkgassetcachewus3.blob.core.windows.net/cache,$assetSas,readwrite" - & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus3.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/failure-logs' - ArtifactName: "failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') - - bash: | - python3 scripts/file_script.py /mnt/vcpkg-ci/installed/vcpkg/info/ - displayName: 'Build a file list for all packages' - condition: always() - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: file lists for ${{ replace(parameters.jobName, '_', '-') }}" - condition: always() - inputs: - PathtoPublish: scripts/list_files - ArtifactName: "file lists for ${{ replace(parameters.jobName, '_', '-') }}" - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: ne(variables['XML_RESULTS_FILE'], '') - inputs: - testRunTitle: ${{ replace(parameters.jobName, '_', '-') }} - testResultsFormat: xUnit - testResultsFiles: $(XML_RESULTS_FILE) - platform: ${{ replace(parameters.jobName, '_', '-') }} - configuration: static diff --git a/scripts/azure-pipelines/osx/azure-pipelines.yml b/scripts/azure-pipelines/osx/azure-pipelines.yml index d424602a472883..ed23c23806cd07 100644 --- a/scripts/azure-pipelines/osx/azure-pipelines.yml +++ b/scripts/azure-pipelines/osx/azure-pipelines.yml @@ -73,12 +73,3 @@ jobs: inputs: PathtoPublish: scripts/list_files ArtifactName: "file lists for ${{ replace(parameters.jobName, '_', '-') }}" - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: ne(variables['XML_RESULTS_FILE'], '') - inputs: - testRunTitle: ${{ replace(parameters.jobName, '_', '-') }} - testResultsFormat: xUnit - testResultsFiles: $(XML_RESULTS_FILE) - platform: ${{ replace(parameters.jobName, '_', '-') }} - configuration: static diff --git a/scripts/azure-pipelines/test-modified-ports.ps1 b/scripts/azure-pipelines/test-modified-ports.ps1 index 3978ce0d3f90ec..6a032ef0a9e1a0 100755 --- a/scripts/azure-pipelines/test-modified-ports.ps1 +++ b/scripts/azure-pipelines/test-modified-ports.ps1 @@ -91,7 +91,7 @@ if ([string]::IsNullOrWhiteSpace($BinarySourceStub)) { if ([string]::IsNullOrWhiteSpace($BuildReason)) { Write-Host 'Build reason not specified, defaulting to using binary caching in read write mode.' } - elseif ($BuildReason -eq 'PullRequest') { + elseif ($BuildReason -eq 'PullRequest' -or $BuildReason -eq 'pull_request' -or $BuildReason -eq 'push') { Write-Host 'Build reason was Pull Request, using binary caching in read write mode, skipping failures.' $skipFailuresArg = @('--skip-failures') } @@ -110,7 +110,6 @@ if ($IsWindows) { } $failureLogs = Join-Path $ArtifactStagingDirectory 'failure-logs' -$xunitFile = Join-Path $ArtifactStagingDirectory "$Triplet-results.xml" if ($IsWindows) { mkdir empty @@ -190,14 +189,11 @@ if (($BuildReason -eq 'PullRequest') -and -not $NoParentHashes) # but changes must trigger at least some testing. Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake" Copy-Item "scripts/buildsystems/vcpkg.cmake" -Destination "scripts/test_ports/cmake-user" -& "./vcpkg$executableExtension" ci "--triplet=$Triplet" --failure-logs=$failureLogs --x-xunit=$xunitFile "--ci-baseline=$PSScriptRoot/../ci.baseline.txt" @commonArgs @cachingArgs @parentHashes @skipFailuresArg +& "./vcpkg$executableExtension" ci "--triplet=$Triplet" --failure-logs=$failureLogs "--ci-baseline=$PSScriptRoot/../ci.baseline.txt" @commonArgs @cachingArgs @parentHashes @skipFailuresArg $lastLastExitCode = $LASTEXITCODE $failureLogsEmpty = (-Not (Test-Path $failureLogs) -Or ((Get-ChildItem $failureLogs).count -eq 0)) Write-Host "##vso[task.setvariable variable=FAILURE_LOGS_EMPTY]$failureLogsEmpty" - -Write-Host "##vso[task.setvariable variable=XML_RESULTS_FILE]$xunitFile" - if ($lastLastExitCode -ne 0) { Write-Error "vcpkg ci testing failed; this is usually a bug in a port. Check for failure logs attached to the run in Azure Pipelines." diff --git a/scripts/azure-pipelines/windows/azure-pipelines.yml b/scripts/azure-pipelines/windows/azure-pipelines.yml deleted file mode 100644 index eaace013fd1909..00000000000000 --- a/scripts/azure-pipelines/windows/azure-pipelines.yml +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# SPDX-License-Identifier: MIT -# - -parameters: - - name: vcpkgToolSha - displayName: 'Custom SHA of vcpkg-tool to use rather than bootstrap' - type: string - default: 'use default' - - name: jobName - type: string - default: 'x86_windows' - - name: tripletPattern - displayName: 'Enable the triplets which contain this substring' - type: string - default: '' - -jobs: -- job: ${{ parameters.jobName }} - condition: and(succeeded(), contains('^${{ replace(parameters.jobName, '_', '-') }}$', '${{ parameters.tripletPattern }}')) - pool: - name: PrWin-1ES - demands: ImageVersionOverride -equals 2024.10.14 - workspace: - clean: resources - timeoutInMinutes: 2880 # 2 days - variables: - - name: WORKING_ROOT - value: D:\ - - name: VCPKG_DOWNLOADS - value: D:\downloads - - name: DiffFile - value: $(Build.ArtifactStagingDirectory)\format.diff - - name: ExtraChecksTriplet - value: x86-windows - steps: - - script: .\bootstrap-vcpkg.bat - displayName: 'Bootstrap vcpkg' - condition: eq('use default', '${{ parameters.vcpkgToolSha }}') - - script: .\scripts\azure-pipelines\windows\bootstrap-from-source.cmd ${{ parameters.vcpkgToolSha }} - displayName: "Build vcpkg with CMake" - condition: ne('use default', '${{ parameters.vcpkgToolSha }}') - - script: '.\vcpkg.exe format-manifest --all' - displayName: 'Format Manifests' - condition: eq('${{ replace(parameters.jobName, '_', '-') }}', '${{ variables.ExtraChecksTriplet }}') - - task: PowerShell@2 - displayName: 'Create Diff' - condition: eq('${{ replace(parameters.jobName, '_', '-') }}', '${{ variables.ExtraChecksTriplet }}') - inputs: - filePath: scripts/azure-pipelines/Create-PRDiff.ps1 - arguments: "-DiffFile '$(DiffFile)'" - pwsh: true - - task: PublishBuildArtifacts@1 - displayName: 'Publish Format and Documentation Diff' - condition: and(eq('${{ replace(parameters.jobName, '_', '-') }}', '${{ variables.ExtraChecksTriplet }}'), failed()) - inputs: - PathtoPublish: '$(DiffFile)' - ArtifactName: 'format.diff' - - task: AzureCLI@2 - displayName: '*** Test Modified Ports' - inputs: - azureSubscription: 'VcpkgPrFleet' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $current = Get-Date -AsUtc - $endDate = $current.AddDays(2) - $end = Get-Date -Date $endDate -UFormat '+%Y-%m-%dT%H:%MZ' - $assetSas = az storage container generate-sas --name cache --account-name vcpkgassetcachewus3 --as-user --auth-mode login --https-only --permissions rcl --expiry $end -o tsv | Out-String - $assetSas = $assetSas.Trim() - $binarySas = az storage container generate-sas --name cache --account-name vcpkgbinarycachewus3 --as-user --auth-mode login --https-only --permissions rclw --expiry $end -o tsv | Out-String - $binarySas = $binarySas.Trim() - $env:X_VCPKG_ASSET_SOURCES = "x-azurl,https://vcpkgassetcachewus3.blob.core.windows.net/cache,$assetSas,readwrite" - & scripts/azure-pipelines/test-modified-ports.ps1 -Triplet ${{ replace(parameters.jobName, '_', '-') }} -BuildReason $(Build.Reason) -BinarySourceStub "x-azblob,https://vcpkgbinarycachewus3.blob.core.windows.net/cache,$binarySas" -WorkingRoot $env:WORKING_ROOT -ArtifactStagingDirectory $(Build.ArtifactStagingDirectory) - - task: PowerShell@2 - displayName: 'Validate version files' - condition: eq('${{ replace(parameters.jobName, '_', '-') }}', '${{ variables.ExtraChecksTriplet }}') - inputs: - filePath: 'scripts/azure-pipelines/windows/validate-version-files.ps1' - pwsh: true - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\failure-logs' - ArtifactName: "failure logs for ${{ replace(parameters.jobName, '_', '-') }}" - condition: ne(variables['FAILURE_LOGS_EMPTY'], 'True') - - task: PowerShell@2 - displayName: 'Build a file list for all packages' - condition: always() - inputs: - targetType: inline - script: | - ./vcpkg.exe fetch python3 - & $(.\vcpkg fetch python3) .\scripts\file_script.py D:\installed\vcpkg\info\ - pwsh: true - - task: PublishBuildArtifacts@1 - displayName: "Publish Artifact: file lists for ${{ replace(parameters.jobName, '_', '-') }}" - condition: always() - inputs: - PathtoPublish: scripts/list_files - ArtifactName: "file lists for ${{ replace(parameters.jobName, '_', '-') }}" - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - condition: ne(variables['XML_RESULTS_FILE'], '') - inputs: - testRunTitle: ${{ replace(parameters.jobName, '_', '-') }} - testResultsFormat: xUnit - testResultsFiles: $(XML_RESULTS_FILE) - platform: ${{ replace(parameters.jobName, '_', '-') }} diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 old mode 100644 new mode 100755 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh old mode 100644 new mode 100755