Skip to content

Commit e1e3325

Browse files
sandeepd-nvsandeepd-nv
authored
Merge pull request #84 from sandeepd-nv/add_ci1
Implement Github CI
2 parents e83073a + 01389ef commit e1e3325

File tree

15 files changed

+718
-0
lines changed

15 files changed

+718
-0
lines changed

Diff for: ‎.github/actions/build/action.yml.j2

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
name: build
2+
3+
description: Build specified project
4+
5+
inputs:
6+
build-type:
7+
required: true
8+
type: string
9+
description: One of ci / release
10+
target-device:
11+
required: true
12+
type: string
13+
host-platform:
14+
required: true
15+
type: string
16+
use-container:
17+
required: true
18+
type: boolean
19+
docker-image:
20+
type: string
21+
required: true
22+
upload-enabled:
23+
required: true
24+
type: boolean
25+
26+
runs:
27+
using: composite
28+
steps:
29+
30+
<% for package_id, package_info in packages.items() %>
31+
- name: Download <<package_info.repo>> (artifacts)
32+
uses: ./.github/actions/download-artifacts
33+
with:
34+
artifact-repo: "<<package_info.repo>>"
35+
artifact-name: "<<package_info.artifact_name | replace_placeholder('repo', package_info.repo) | replace_placeholder('git_tag', package_info.git_tag) >>"
36+
target-device: "${{ inputs.target-device }}"
37+
git_sha: "<<package_info.git_tag>>"
38+
host-platform: ${{ inputs.host-platform }}
39+
dest-dir: ${{ env.ARTIFACTS_DIR }}
40+
dependencies-workflow: <<package_info.artifact_workflow>>
41+
<% endfor %>
42+
43+
<% if packages %>
44+
45+
- name: Display structure of downloaded artifacts
46+
shell: bash --noprofile --norc -xeuo pipefail {0}
47+
run: |
48+
pwd
49+
ls -lahR ${{ env.ARTIFACTS_DIR }}
50+
<% endif %>
51+
52+
- if: ${{ inputs.use-container }}
53+
name: Build (in container)
54+
shell: bash --noprofile --norc -xeuo pipefail {0}
55+
run: |
56+
57+
docker run \
58+
-e AWS_REGION \
59+
-e AWS_SESSION_TOKEN \
60+
-e AWS_ACCESS_KEY_ID \
61+
-e AWS_SECRET_ACCESS_KEY \
62+
-e GITHUB_TOKEN \
63+
-e ARTIFACTS_DIR="$ARTIFACTS_DIR" \
64+
-e UPLOAD_ENABLED="$UPLOAD_ENABLED" \
65+
-e USE_CUDA="$USE_CUDA" \
66+
-e REPO_DIR="$REPO_DIR" \
67+
-e LEGATE_CORE_BUILD_MODE="$LEGATE_CORE_BUILD_MODE" \
68+
-e PYTHON_VERSION="$PYTHON_VERSION" \
69+
-v "${{ env.REPO_DIR }}:${{ env.REPO_DIR }}" \
70+
-v "${{ env.ARTIFACTS_DIR }}:${{ env.ARTIFACTS_DIR }}" \
71+
--rm "${{ inputs.docker-image }}" \
72+
/bin/bash -c "${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint ${{ env.REPO_DIR }}/continuous_integration/scripts/build ${{ inputs.build-type}} ${{ inputs.target-device }}"
73+
74+
- if: ${{ !inputs.use-container }}
75+
name: Build (without container)
76+
shell: bash --noprofile --norc -xeuo pipefail {0}
77+
run: |
78+
"${{ env.REPO_DIR }}/continuous_integration/scripts/entrypoint" "${{ env.REPO_DIR }}/continuous_integration/scripts/build" "${{ inputs.build-type}}" "${{ inputs.target-device }}"
79+
80+
- name: Display structure of the artifacts folder (post build)
81+
shell: bash --noprofile --norc -xeuo pipefail {0}
82+
run: |
83+
sudo chown -R $(whoami) ${{ env.ARTIFACTS_DIR }}
84+
ls -lahR ${{ env.ARTIFACTS_DIR }}
85+
86+
- name: Upload build artifacts
87+
uses: actions/upload-artifact@v4
88+
with:
89+
name: ${{ env.ARTIFACT_NAME }}
90+
path: ${{ env.ARTIFACTS_DIR }}

Diff for: ‎.github/actions/download-artifacts/action.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: download-artifacts
2+
3+
description: Download dependencies (artifacts)
4+
5+
inputs:
6+
artifact-repo:
7+
type: string
8+
require: true
9+
artifact-name:
10+
type: string
11+
require: true
12+
target-device:
13+
type: string
14+
required: true
15+
git_sha:
16+
type: string
17+
required: true
18+
host-platform:
19+
type: string
20+
required: true
21+
dest-dir:
22+
type: string
23+
required: true
24+
dependencies-workflow:
25+
required: true
26+
type: string
27+
28+
runs:
29+
using: composite
30+
steps:
31+
32+
- id: cache
33+
name: Cache conda artifacts
34+
uses: actions/cache@v4
35+
with:
36+
key: "nvidia/{ inputs.artifact-repo }}@${{ inputs.host-platform }}-${{ inputs.git_sha }}-${{ inputs.target-device }}"
37+
path: ${{ inputs.dest-dir }}
38+
39+
- if: steps.cache.outputs.cache-hit != 'true'
40+
name: Download ${{ inputs.artifact-repo }} artifacts
41+
uses: dawidd6/action-download-artifact@v3
42+
with:
43+
path: ${{ inputs.dest-dir }}
44+
repo: nvidia/${{ inputs.artifact-repo }}
45+
check_artifacts: true
46+
commit: ${{ inputs.git_sha }}
47+
workflow_conclusion: ""
48+
workflow: ${{ inputs.dependencies-workflow }}
49+
name: ${{ inputs.artifact-name }}
50+
skip_unpack: true
51+
if_no_artifact_found: fail
52+
allow_forks: false
53+
54+
- if: steps.cache.outputs.cache-hit != 'true'
55+
name: Unpack artifact
56+
shell: bash --noprofile --norc -xeuo pipefail {0}
57+
run: |
58+
cd ${{ inputs.dest-dir }}
59+
unzip *.zip

Diff for: ‎.github/actions/setup/action.yml

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
name: Common setup
2+
3+
inputs:
4+
client-repo:
5+
required: true
6+
type: string
7+
build-type:
8+
required: true
9+
type: string
10+
target-device:
11+
required: true
12+
type: string
13+
host-platform:
14+
required: true
15+
type: string
16+
build-mode:
17+
required: true
18+
type: string
19+
upload-enabled:
20+
required: true
21+
type: boolean
22+
python-version:
23+
required: false
24+
type: string
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- name: Set REPO_DIR and Dump environment
30+
shell: bash --noprofile --norc -xeuo pipefail {0}
31+
run: |
32+
echo "REPO_DIR=$(pwd)" >> $GITHUB_ENV
33+
env
34+
35+
- name: Set environment variables
36+
shell: bash --noprofile --norc -xeuo pipefail {0}
37+
run: |
38+
39+
WITH_TESTS_STR=''
40+
if [[ ("${{ inputs.upload-enabled }}" == "false") && ("${{ inputs.build-type }}" != "ci") ]]; then
41+
WITH_TESTS_STR='-with_tests'
42+
fi
43+
44+
TARGET_PLATFORM='linux-64'
45+
if [[ "${{ inputs.host-platform }}" == "linux-aarch64" ]]; then
46+
TARGET_PLATFORM='linux-aarch64'
47+
fi
48+
49+
BUILD_MODE="${{ inputs.build-mode }}"
50+
BUILD_MODE_STR=""
51+
[ -n "${BUILD_MODE}" ] && BUILD_MODE_STR="-${BUILD_MODE}"
52+
53+
if [[ ("${BUILD_MODE}" == "") || ("${BUILD_MODE}" == "release") ]]; then
54+
# We upload release versions in the default folder.
55+
PKG_DIR="${TARGET_PLATFORM}"
56+
else
57+
PKG_DIR="${BUILD_MODE}/${TARGET_PLATFORM}"
58+
fi
59+
60+
echo "ARTIFACT_NAME=${{ inputs.host-platform }}-${{ inputs.build-type }}-${{ inputs.client-repo }}-python${{ inputs.python-version }}-${{ inputs.target-device }}${BUILD_MODE_STR}${WITH_TESTS_STR}-${{ github.sha }}" >> $GITHUB_ENV
61+
echo "ARTIFACTS_DIR=$(realpath "$(pwd)/dist")" >> $GITHUB_ENV
62+
echo "USE_CUDA=${{ (inputs.target-device == 'cpu' && 'OFF') || 'ON' }}" >> $GITHUB_ENV
63+
echo "UPLOAD_ENABLED=${{ (inputs.upload-enabled == 'true' && 'ON') || 'OFF' }}" >> $GITHUB_ENV
64+
echo "LEGATE_CORE_BUILD_MODE=${BUILD_MODE}" >> $GITHUB_ENV
65+
echo "BUILD_DATE=$(date +%Y%m%d)" >> $GITHUB_ENV
66+
echo "TARGET_PLATFORM=${TARGET_PLATFORM}" >> $GITHUB_ENV
67+
echo "PKG_DIR=${PKG_DIR}" >> $GITHUB_ENV
68+
echo "PYTHON_VERSION=${{ inputs.python-version }}" >> $GITHUB_ENV

Diff for: ‎.github/workflows/ci-gh.yml

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
name: Build and test
2+
3+
concurrency:
4+
group: ${{ startsWith(github.ref_name, 'main') && format('unique-{0}', github.run_id) || format('ci-build-and-test-on-{0}-from-{1}', github.event_name, github.ref_name) }}
5+
cancel-in-progress: true
6+
7+
on:
8+
push:
9+
branches:
10+
- "pull-request/[0-9]+"
11+
- "main"
12+
13+
jobs:
14+
build-and-test:
15+
name: Build and test (${{ matrix.host-platform }}, ${{ matrix.target-device }}, ${{ matrix.build-mode }})
16+
strategy:
17+
fail-fast: false
18+
matrix:
19+
host-platform:
20+
- linux-x64
21+
target-device:
22+
- gpu
23+
build-mode:
24+
- release
25+
upload-enabled:
26+
- false
27+
uses:
28+
./.github/workflows/gh-build-and-test.yml
29+
with:
30+
host-platform: ${{ matrix.host-platform }}
31+
target-device: ${{ matrix.target-device }}
32+
build-mode: ${{ matrix.build-mode }}
33+
build-type: ci
34+
upload-enabled: ${{ matrix.upload-enabled }}
35+
secrets: inherit

Diff for: ‎.github/workflows/gh-build-and-test.yml

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
host-platform:
5+
type: string
6+
required: true
7+
target-device:
8+
type: string
9+
required: true
10+
build-mode:
11+
type: string
12+
required: true
13+
build-type:
14+
type: string
15+
required: true
16+
upload-enabled:
17+
type: boolean
18+
required: true
19+
jobs:
20+
build:
21+
if: ${{ github.repository_owner == 'nvidia' }}
22+
uses:
23+
./.github/workflows/gh-build.yml
24+
with:
25+
client-repo: ${{ github.event.repository.name }}
26+
target-device: ${{ inputs.target-device }}
27+
runs-on: ${{ (inputs.host-platform == 'linux-x64' && 'linux-amd64-cpu16') || (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu16') || (inputs.host-platform == 'mac' && 'macos-latest') }}
28+
build-type: ${{ inputs.build-type }}
29+
use-container: ${{ inputs.host-platform == 'linux-x64' || inputs.host-platform == 'linux-aarch64'}}
30+
host-platform: ${{ inputs.host-platform }}
31+
dependencies-file: ""
32+
build-mode: ${{ inputs.build-mode }}
33+
upload-enabled: ${{ inputs.upload-enabled }}
34+
secrets: inherit

Diff for: ‎.github/workflows/gh-build.yml

+95
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
name: Build
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
client-repo:
7+
required: true
8+
type: string
9+
target-device:
10+
required: true
11+
type: string
12+
runs-on:
13+
required: true
14+
type: string
15+
build-type:
16+
required: true
17+
type: string
18+
description: One of ci / release
19+
use-container:
20+
required: true
21+
type: boolean
22+
host-platform:
23+
required: true
24+
type: string
25+
dependencies-file:
26+
required: true
27+
type: string
28+
description: path to versions.json relative to the target repo dir
29+
build-mode:
30+
required: true
31+
type: string
32+
upload-enabled:
33+
required: true
34+
type: boolean
35+
python-version:
36+
required: false
37+
type: string
38+
39+
jobs:
40+
build:
41+
name: Build (${{ inputs.host-platform }}, ${{ inputs.target-device }}, ${{ inputs.build-type }}, CMake build-mode=${{ inputs.build-mode }}, Python "${{ inputs.python-version }}", Use container=${{ inputs.use-container }} )
42+
43+
permissions:
44+
id-token: write # This is required for configure-aws-credentials
45+
contents: read # This is required for actions/checkout
46+
47+
runs-on: ${{ inputs.runs-on }}
48+
49+
steps:
50+
- name: Checkout ${{ inputs.client-repo }}
51+
uses: actions/checkout@v4
52+
with:
53+
fetch-depth: 0
54+
55+
- name: Setup
56+
uses: ./.github/actions/setup
57+
with:
58+
client-repo: ${{ inputs.client-repo }}
59+
build-type: ${{ inputs.build-type }}
60+
target-device: "${{ inputs.target-device }}"
61+
host-platform: ${{ inputs.host-platform }}
62+
build-mode: ${{ inputs.build-mode }}
63+
upload-enabled: ${{ inputs.upload-enabled }}
64+
python-version: ${{ inputs.python-version }}
65+
66+
- name: Render templates
67+
shell: bash --noprofile --norc -xeuo pipefail {0}
68+
run: |
69+
pip -q install jinja2
70+
71+
DEPENDENCIES_FILE=""
72+
73+
if [ -z "${{ inputs.dependencies-file }}" ]; then
74+
DEPENDENCIES_FILE="${REPO_DIR}/continuous_integration/no_dependencies.json"
75+
else
76+
DEPENDENCIES_FILE="${REPO_DIR}/${{ inputs.dependencies-file }}"
77+
fi
78+
79+
${REPO_DIR}/continuous_integration/scripts/render-template.py .github/actions/build/action.yml.j2 "${DEPENDENCIES_FILE}" .github/actions/build/action.yml
80+
81+
- name: Dump templates
82+
shell: bash --noprofile --norc -xeuo pipefail {0}
83+
run: |
84+
echo ${REPO_DIR}/.github/actions/build/action.yml
85+
cat ${REPO_DIR}/.github/actions/build/action.yml
86+
87+
- name: Call build action
88+
uses: ./.github/actions/build
89+
with:
90+
build-type: ${{ inputs.build-type }}
91+
target-device: "${{ inputs.target-device }}"
92+
host-platform: ${{ inputs.host-platform }}
93+
use-container: ${{ inputs.use-container }}
94+
docker-image: "condaforge/miniforge3:latest"
95+
upload-enabled: ${{ inputs.upload-enabled }}

Diff for: ‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -168,3 +168,6 @@ dmypy.json
168168

169169
# Cython debug symbols
170170
cython_debug/
171+
172+
# Dont ignore
173+
!.github/actions/build/

0 commit comments

Comments
 (0)