test_cloud_runner #401
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test_cloud_runner | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| fre-workflows-repo: | |
| description: "If using fork: Input fork name (example: singhd789/fre-workflows). Leave empty if not using a fork." | |
| required: false | |
| type: string | |
| fre-workflows-branch: | |
| description: 'If using fork: Input branch you are using on your fork.' | |
| required: false | |
| default: 'main' | |
| type: string | |
| fre-cli-repo: | |
| description: 'NOAA-GFDL/fre-cli repo' | |
| required: true | |
| type: choice | |
| default: 'NOAA-GFDL/fre-cli' | |
| options: | |
| - 'NOAA-GFDL/fre-cli' | |
| fre-cli-branch: | |
| description: 'Input which fre-cli branch to check out' | |
| required: true | |
| default: 'main' | |
| type: string | |
| push: | |
| branches: | |
| - main # triggers pipeline on push to main | |
| jobs: | |
| test-cloud: | |
| runs-on: self-hosted | |
| steps: | |
| - name: Checkout fre-workflows fork if specified | |
| if: ${{ github.event.inputs.fre-workflows-repo != ''}} | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.fre-workflows-repo }} | |
| ref: ${{ github.event.inputs.fre-workflows-branch }} | |
| - name: Checkout fre-workflows repository if no fork specified | |
| if: ${{ github.event.inputs.fre-workflows-repo == ''}} | |
| uses: actions/checkout@v4 | |
| - name: Checkout fre-cli | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.inputs.fre-cli-repo }} | |
| ref: ${{ github.event.inputs.fre-cli-branch }} | |
| path: './fre-cli' | |
| submodules: true | |
| - name: Display fre-workflow and fre-cli repos and branches | |
| run: | | |
| # fre-workflows repo and branch | |
| echo "fre-workflows repo: $(git remote -v)" | |
| echo "fre-workflows branch: $(git branch)" | |
| # fre-cli repo and branch | |
| cd fre-cli | |
| echo "fre-cli repo: $(git remote -v)" | |
| echo "fre-cli branch: $(git branch)" | |
| cd - | |
| - name: Run ppp-container | |
| run: | | |
| # Print pwd then run script | |
| echo "Currently in: ${PWD}" | |
| # Set up output directory based on which runner picks up the job | |
| # This is to ensure there are no accessibility issues when using | |
| # multiple runners. | |
| echo "Runner that picked up the job: ${RUNNER_NAME}" | |
| # Define output directory, grid spec file name, and directory where | |
| # data will be linked | |
| gridspec_file=c96_OM4_025_grid_No_mg_drag_v20160808.tar | |
| output_dir=/contrib/container-test/output-${RUNNER_NAME} | |
| data_linkdir=/home/Dana.Singh/workflow-data-${RUNNER_NAME} | |
| ## Remove if paths previously exist - this might actually delete them from the host as well | |
| #rm -rf $output_dir | |
| #rm -rf $data_linkdir/history | |
| #rm -rf $data_linkdir/grid_spec | |
| ## Create data link directory and output directory | |
| mkdir -p $output_dir | |
| mkdir -p $data_linkdir | |
| mkdir -p $data_linkdir/grid_spec | |
| ## Create symbolic links for data (for if multiple runners are used at same time) | |
| ln -sf /contrib2/c96L65_TEST/history ${data_linkdir} | |
| ln -sf /contrib/container-test/grid_spec/${gridspec_file} ${data_linkdir}/grid_spec | |
| # Write environment variable to GITHUB_ENV to make it accessible for other steps | |
| echo "OUTPUT_DIR=$output_dir" >> $GITHUB_ENV | |
| echo "OUTPUT DIR: $output_dir" | |
| echo "HISTORY FILE DIR: $data_linkdir/history" | |
| echo "GRID SPEC DIR: $data_linkdir/grid_spec" | |
| # Set up binds and required paths | |
| bind_paths="--bind ${output_dir}/:/mnt:rw | |
| --bind ${data_linkdir}/history:/mnt/history:ro | |
| --bind ${data_linkdir}/grid_spec/${gridspec_file}:/mnt/${gridspec_file}:ro | |
| --bind /contrib/container-test" | |
| container_path=/contrib/container-test/ppp-wf.sif | |
| runscript_path=${PWD}/for_gh_runner/runscript.sh | |
| # Run the container | |
| singularity exec --writable-tmpfs ${bind_paths} ${container_path} ${runscript_path} | |
| - name: Upload workflow-run log files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fre-workflow-logs | |
| path: ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log | |
| if-no-files-found: error | |
| - name: Print workflow summary | |
| continue-on-error: true | |
| run: | | |
| num_tasks=$(ls -l ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job | wc -l) | |
| echo "number of tasks launched: ${num_tasks}" | |
| echo "" | |
| echo "here's all the job directories with job scripts:" | |
| ls -l ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/????*/*/NN/job | |
| echo "" | |
| echo "here's all the job scripts for 1980 only:" | |
| ls -l ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/1980*/*/NN/job | |
| echo "" | |
| echo "rose-suite conf for workflow:" | |
| cat ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/config/*rose-suite.conf | |
| echo "" | |
| - name: Print pp-starter successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/pp-starter/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print stage-history successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/stage-history/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print regrid-xy successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/regrid-xy*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print data-catalog-final (FAILURE-GUARDED) | |
| continue-on-error: true | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/data-catalog-final/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print mask-atmos-plevel successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/mask-atmos-plevel*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print remap-pp-components successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-pp-components*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print rename-split-to-pp successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/rename-split-to-pp*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print split-netcdf successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/split-netcdf*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print climotology time average successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/climo-*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print combine-climo successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/combine-climo*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print remap-climo successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/remap-climo-*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| - name: Print clean task successes or failures | |
| run: | | |
| set +e | |
| # Successes | |
| success=$(grep -E "CYLC_JOB_EXIT=SUCCEEDED" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) | |
| exit_status_1=$? | |
| # Failures | |
| failure=$(grep -E "CYLC_JOB_EXIT=ERR" ${{ env.OUTPUT_DIR }}/cylc-run/test_pp__ptest__ttest/log/job/*/clean*/0[1-3]/job.status) | |
| exit_status_2=$? | |
| if [ $exit_status_1 -eq 0 ]; then | |
| echo $success | sed 's/ /\n/g' #split string by spaces | |
| elif [ $exit_status_1 -eq 1 ]; then | |
| echo "No succeeded tasks found" | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| if [ $exit_status_2 -eq 0 ]; then | |
| echo "Failures found" | |
| echo $failure | sed 's/ /\n/g' #split string by spaces | |
| exit 1 #exit with error | |
| elif [ $exit_status_2 -eq 1 ]; then | |
| echo "No task failures found." | |
| else | |
| echo "WARNING: error with grep execution" | |
| exit 2 | |
| fi | |
| ## I think this will work (without cylc clean) because the OUTPUT_DIR is bind mounted from the host | |
| - name: Clean up output folder | |
| run: rm -rf ${{ env.OUTPUT_DIR }} | |
| ## Keep in here for now as a just in case reference ## | |
| # - name: Print any succeeded tasks | |
| # run: | | |
| # # List tasks that have succeeded (and completed) in the workflow | |
| # grep -E ":succeeded.*completed" /contrib/container-test/ppp-setup/log.out | |
| # | |
| ## maybe, all the printinig above should be continue-on-error for informational purposes | |
| ## then, this task does the final determination if things ran right or wrong. | |
| ## githubs auto-annotations for warnings etc are good are helping one figure out where to sniff around | |
| # - name: Print any failed tasks | |
| # run: | | |
| # # Search for if there is an ERR status for any task | |
| # if grep -q "failed/ERR" /contrib/container-test/ppp-setup/log.out; then | |
| # echo "Task failures found: CHECK THE LOG FILES UPLOADED AS ARTIFACTS" | |
| # grep "failed/ERR" /contrib/container-test/ppp-setup/log.out | |
| # exit 1 | |
| # fi | |
| # | |
| # echo "Workflow ran successfully" | |
| ##To-do: | |
| # - figure out if container will be hosted on cloud or somehow pulled from registry | |
| # - examine behavior of runner in draft vs not draft mode |