Skip to content

Commit 77d4cf3

Browse files
wonwuakpa-msftwonwuakpa-msft
authored
Add missing stage in 1ES pipeline, label release jobs correctly (#3233)
1 parent 5a7ad77 commit 77d4cf3

File tree

1 file changed

+134
-18
lines changed

1 file changed

+134
-18
lines changed

‎build-1es-pipeline.yaml‎

Lines changed: 134 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,12 @@ parameters:
3636
displayName: 'Test PMC Connection without Publishing Artifacts to Linux Repos'
3737
type: boolean
3838
default: false
39+
40+
- name: remove_packages
41+
displayName: 'Remove Linux Packages'
42+
type: string
43+
default: ' '
44+
3945
# Do not trigger this pipeline automatically
4046
trigger: none
4147
pr: none
@@ -543,7 +549,7 @@ extends:
543549
exit 1
544550
fi
545551
displayName: 'Copy signed mac binaries'
546-
# SignArtifacts end here
552+
# SignArtifacts end here
547553

548554
- stage: VerifyArtifacts
549555
dependsOn: SignArtifacts
@@ -865,7 +871,7 @@ extends:
865871
m1_container_url="https://azcopyvnextrelease.blob.core.windows.net/%24web/azcopy-m1-drops/azcopy-$(azcopy_version)-$today"
866872
AZCOPY_AUTO_LOGIN_TYPE=AzCLI ./azcopy_bin cp "$(input)/azcopy-binaries-mac/azcopy_darwin_m1_arm64" "$m1_container_url" --recursive --put-md5=true
867873
fi
868-
# UploadToStorage ends here
874+
# UploadToStorage end here
869875
- ${{ if eq(parameters.publish_docker_image, true) }}:
870876
- stage: BuildAndPublishDockerImage
871877
dependsOn:
@@ -923,7 +929,7 @@ extends:
923929
inputs:
924930
image: azcopycontainers.azurecr.io/azure-azcopy-mariner-x86_64.$(azcopy_docker_version):latest
925931
remoteImage: azcopycontainers.azurecr.io/azure-azcopy-mariner-x86_64.$(azcopy_docker_version):latest
926-
932+
927933
- job: LinuxARM64
928934
pool:
929935
name: azcopy-arm-pool
@@ -984,7 +990,10 @@ extends:
984990
- AzCopyVersion
985991
- VerifyArtifacts
986992
jobs:
987-
- job: Job
993+
- job: ReleaseJob
994+
templateContext:
995+
type: releaseJob
996+
isProduction: true
988997
pool:
989998
name: azcopy-pool
990999
image: ubuntu22-1espt
@@ -1002,11 +1011,23 @@ extends:
10021011
echo "##vso[task.setvariable variable=azcopy_github_tag]$azcopy_github_tag"
10031012
displayName: 'AzCopy Tag Name'
10041013
1005-
- task : DownloadPipelineArtifact@2
1006-
displayName: 'Download Linux, Windows, Mac Build Artifacts'
1014+
- task: 1ES.DownloadPipelineArtifact@1
1015+
displayName: 'Download Windows Build Artifacts'
10071016
inputs:
1008-
itemPattern: 'azcopy-*-signed/**'
1009-
downloadPath: $(Build.ArtifactStagingDirectory)
1017+
artifactName: 'azcopy-windows-signed'
1018+
targetPath: $(Build.ArtifactStagingDirectory)/signed
1019+
1020+
- task: 1ES.DownloadPipelineArtifact@1
1021+
displayName: 'Download Linux Build Artifacts'
1022+
inputs:
1023+
artifactName: 'azcopy-linux-signed'
1024+
targetPath: $(Build.ArtifactStagingDirectory)/signed
1025+
1026+
- task: 1ES.DownloadPipelineArtifact@1
1027+
displayName: 'Download Mac Build Artifacts'
1028+
inputs:
1029+
artifactName: 'azcopy-mac-signed'
1030+
targetPath: $(Build.ArtifactStagingDirectory)/signed
10101031

10111032
- script: |
10121033
sudo ls -lRt $(Build.ArtifactStagingDirectory)
@@ -1107,10 +1128,10 @@ extends:
11071128
amd64file=$(ls "$(signed)/mariner/azcopy"*.x86_64.rpm)
11081129
marinerArmFileName="${arm64file/.arm64.rpm/-cm2.arm64.rpm}"
11091130
marinerAmdFileName="${amd64file/.x86_64.rpm/-cm2.x86_64.rpm}"
1110-
1131+
11111132
mv -v "$(signed)/mariner/azcopy"*.arm64.rpm "$marinerArmFileName"
11121133
mv -v "$(signed)/mariner/azcopy"*.x86_64.rpm "$marinerAmdFileName"
1113-
1134+
11141135
mv "$(signed)/mariner/azcopy"*.rpm "$(signed)/"
11151136
rm -r "$(signed)/mariner/"
11161137
displayName: 'Rename Mariner binaries'
@@ -1150,37 +1171,37 @@ extends:
11501171
inlineScript: |
11511172
azcopyAmdDeb=`pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload azcopy*.x86_64.deb`
11521173
echo "Azcopy AMD DEB ID: $azcopyAmdDeb"
1153-
1174+
11541175
azcopyAmdRpmFile=$(ls azcopy-* | grep -vE 'cm2\.x86_64\.rpm|arm64\.deb|x86_64\.deb|arm64\.rpm')
11551176
azcopyAmdRpm=$(pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload $azcopyAmdRpmFile)
11561177
echo "Azcopy AMD RPM ID: $azcopyAmdRpm"
1157-
1178+
11581179
azcopyArmDeb=`pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload azcopy*.arm64.deb`
11591180
echo "Azcopy ARM DEB ID: $azcopyArmDeb"
1160-
1181+
11611182
azcopyArmRpmFile=$(ls azcopy* | grep -vE 'cm2\.arm64\.rpm|arm64\.deb|x86_64\.deb|x86_64\.rpm')
11621183
azcopyArmRpm=$(pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload $azcopyArmRpmFile)
11631184
echo "Azcopy ARM RPM $azcopyArmRpmFile"
11641185
echo "Azcopy ARM RPM ID: $azcopyArmRpm"
1165-
1186+
11661187
# Find the Mariner RPM file and upload
11671188
marinerAmdRpmFile=$(ls azcopy-* | grep 'cm2\.x86_64\.rpm')
11681189
marinerAmdRpm=$(pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload $marinerAmdRpmFile)
11691190
echo "Azcopy mariner AMD RPM $marinerAmdRpmFile"
11701191
echo "Azcopy mariner AMD RPM ID: $marinerAmdRpm"
1171-
1192+
11721193
marinerAarchRpmFile=$(ls azcopy-* | grep 'cm2\.arm64\.rpm')
11731194
marinerAarchRpm=$(pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" --id-only package upload $marinerAarchRpmFile)
11741195
echo "Azcopy mariner ARM RPM: $marinerAarchRpmFile"
11751196
echo "Azcopy mariner ARM RPM ID: $marinerAarchRpm"
1176-
1197+
11771198
is_preview="false"
11781199
echo "##vso[task.setvariable variable=is_preview]$is_preview"
11791200
if [[ $marinerAmdRpmFile == *"preview"* ]]; then
11801201
is_preview="true"
11811202
echo "##vso[task.setvariable variable=is_preview]$is_preview"
11821203
fi
1183-
1204+
11841205
while IFS=, read -r distro archetype repoName releaseName; do
11851206
# If the package is preview, publish to mariner preview package
11861207
if [[ $distro == *"Mariner-"* ]]; then
@@ -1215,4 +1236,99 @@ extends:
12151236
echo "Repo Name: $repoName"
12161237
pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" repo publish $repoName
12171238
fi
1218-
done < <(tail -n +3 ../packages.csv)
1239+
done < <(tail -n +3 ../packages.csv)
1240+
1241+
- ${{ if ne(parameters.remove_packages, '') }}:
1242+
- stage: RemovePackagesFromLinuxRepository
1243+
jobs:
1244+
- job: RemovePackages
1245+
timeoutInMinutes: 120
1246+
pool:
1247+
name: azcopy-pool
1248+
image: ubuntu22-1espt
1249+
os: linux
1250+
variables:
1251+
- group: AZCOPY_SECRET_VAULT
1252+
steps:
1253+
- checkout: none
1254+
1255+
- task: PipAuthenticate@1
1256+
inputs:
1257+
artifactFeeds: 'DevExGlobalFeed'
1258+
displayName: 'Connect to PMC artifact'
1259+
1260+
- script: |
1261+
sudo apt-get clean
1262+
sudo apt-get update --fix-missing
1263+
sudo apt-get install -y tree
1264+
displayName: 'Install Dependencies'
1265+
1266+
- script: |
1267+
pip install pmc-cli
1268+
echo '##vso[task.prependpath]$(HOME)/.local/bin'
1269+
displayName: 'Install pmc-cli'
1270+
1271+
- task: DownloadSecureFile@1
1272+
name: pmcCertificate
1273+
displayName: 'Download pmc pem file'
1274+
inputs:
1275+
secureFile: 'azstorage-devex-kv-blobfuse-release-pmc1-10102025.pem'
1276+
1277+
- task: DownloadSecureFile@1
1278+
name: settings
1279+
displayName: 'Download settings.toml file'
1280+
inputs:
1281+
secureFile: 'settings.toml'
1282+
1283+
- task: AzureCLI@2
1284+
displayName: 'Test PMC installation'
1285+
inputs:
1286+
addSpnToEnvironment: true
1287+
azureSubscription: 'azcopy-release'
1288+
scriptType: bash
1289+
scriptLocation: inlineScript
1290+
inlineScript: |
1291+
pmc --version
1292+
pmc --auth-type wif --base-url "https://pmc-ingest.trafficmanager.net/api/v4" repo list --limit 1
1293+
if [ $? -ne 0 ]; then
1294+
exit 1
1295+
fi
1296+
1297+
- script: |
1298+
# Validate remove_packages param
1299+
if [[ "${{parameters.remove_packages}}" != " " ]] && [[ -n "${{parameters.remove_packages}}" ]]; then
1300+
if ! [[ "${{parameters.remove_packages}}" =~ ^[a-zA-Z0-9._~-]+$ ]]; then
1301+
echo "Error: remove_packages param contains invalid chars. Double check its value."
1302+
exit 1
1303+
fi
1304+
fi
1305+
pkg_id_list=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package deb list --name azcopy --version ${{parameters.remove_packages}})
1306+
1307+
# Loop over all the results and extract the package id for each one
1308+
for PKG_ID in $(echo "$pkg_id_list" | jq -r '.results[].id'); do
1309+
echo "Processing Package ID: $PKG_ID" # Debugging line
1310+
pkg_repo_names=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --package "$PKG_ID")
1311+
1312+
for REPO_NAME in $(echo "$pkg_repo_names" | jq -r '.results[].name'); do
1313+
echo "Found repo: $REPO_NAME $PKG_ID" # Debugging line
1314+
pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --remove-packages $PKG_ID $REPO_NAME
1315+
pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $REPO_NAME
1316+
done
1317+
done
1318+
displayName: 'Remove deb packages from the repository'
1319+
1320+
- script: |
1321+
pkg_id_list=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) package rpm list --name azcopy --version ${{parameters.remove_packages}})
1322+
1323+
# Loop over all the results and extract the package id for each one
1324+
for PKG_ID in $(echo "$pkg_id_list" | jq -r '.results[].id'); do
1325+
echo "Processing Package ID: $PKG_ID" # Debugging line
1326+
pkg_repo_names=$(pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo list --package "$PKG_ID")
1327+
1328+
for REPO_NAME in $(echo "$pkg_repo_names" | jq -r '.results[].name'); do
1329+
echo "Found repo: $REPO_NAME $PKG_ID" # Debugging line
1330+
pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo package update --remove-packages $PKG_ID $REPO_NAME
1331+
pmc --msal-cert-path $(pmcCertificate.secureFilePath) --config $(settings.secureFilePath) repo publish $REPO_NAME
1332+
done
1333+
done
1334+
displayName: 'Remove rpm packages from the repository'

0 commit comments

Comments
 (0)