From e985d83f48521917b5a1ed4509f31101cbe6b4d6 Mon Sep 17 00:00:00 2001 From: Zach Dworkin Date: Tue, 6 Aug 2024 09:48:43 -0700 Subject: [PATCH] contrib/intel/jenkins: Remove Intel support for Jenkins Removing Jenkins support from older branches that don't have CI support anymore Signed-off-by: Zach Dworkin --- contrib/intel/jenkins/Jenkinsfile | 124 ------ contrib/intel/jenkins/Jenkinsfile.daily | 338 ---------------- contrib/intel/jenkins/build.py | 101 ----- contrib/intel/jenkins/common.py | 55 --- contrib/intel/jenkins/run.py | 114 ------ contrib/intel/jenkins/runtests.py | 85 ---- contrib/intel/jenkins/tests.py | 502 ------------------------ 7 files changed, 1319 deletions(-) delete mode 100644 contrib/intel/jenkins/Jenkinsfile delete mode 100644 contrib/intel/jenkins/Jenkinsfile.daily delete mode 100755 contrib/intel/jenkins/build.py delete mode 100755 contrib/intel/jenkins/common.py delete mode 100755 contrib/intel/jenkins/run.py delete mode 100755 contrib/intel/jenkins/runtests.py delete mode 100755 contrib/intel/jenkins/tests.py diff --git a/contrib/intel/jenkins/Jenkinsfile b/contrib/intel/jenkins/Jenkinsfile deleted file mode 100644 index 3afa97be94c..00000000000 --- a/contrib/intel/jenkins/Jenkinsfile +++ /dev/null @@ -1,124 +0,0 @@ - -pipeline { - agent { node { label 'master' } } - options { - timestamps() - timeout(activity: true, time: 4, unit: 'HOURS') - } - environment { - JOB_CADENCE = 'PR' - } - - stages { - stage ('build-libfabric') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'libfabric' --ofi_build_mode='dbg' - echo "libfabric build completed" - """ - } - } - } - stage('build-fabtests') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'fabtests' --ofi_build_mode='dbg' - echo 'fabtests build completed' - """ - } - } - } - stage ('copy-build-dirs') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'builddir' --ofi_build_mode='dbg' - echo "copy build dirs complete" - """ - } - } - } - stage('parallel-tests') { - parallel { - stage('eth-tcp-dbg') { - options { skipDefaultCheckout() } - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd /opt${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=tcp --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('eth-udp-shm-dbg') { - agent {node {label 'eth'}} - options { skipDefaultCheckout() } - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd /opt${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=udp --ofi_build_mode='dbg' - python3.7 runtests.py --prov=udp --util=rxd --ofi_build_mode='dbg' - python3.7 runtests.py --prov=shm --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('mlx5-verbs_rxm-dbg') { - agent {node {label 'mlx5'}} - options { skipDefaultCheckout() } - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd /opt${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --ofi_build_mode='dbg' - python3.7 runtests.py --prov=verbs --util=rxm --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('mlx5-verbs_rxd-dbg') { - agent {node {label 'mlx5'}} - options { skipDefaultCheckout() } - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd /opt${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --util=rxd --ofi_build_mode='dbg' - ) - """ - } - } - } - } - } - } - - post { - cleanup { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - dir("${env.WORKSPACE}") { - sh "rm -rf '${env.CI_INSTALL_DIR}/${env.JOB_NAME}/${env.BUILD_NUMBER}/ci_middlewares'" - deleteDir() - } - } - } - } -} diff --git a/contrib/intel/jenkins/Jenkinsfile.daily b/contrib/intel/jenkins/Jenkinsfile.daily deleted file mode 100644 index 6cc0c61ebae..00000000000 --- a/contrib/intel/jenkins/Jenkinsfile.daily +++ /dev/null @@ -1,338 +0,0 @@ - -pipeline { - agent any - options { - timestamps() - timeout(activity: true, time: 4, unit: 'HOURS') - } - environment { - JOB_CADENCE = 'daily' - } - stages { - stage ('build-libfabric') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'libfabric' - python3.7 contrib/intel/jenkins/build.py 'libfabric' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'libfabric' --ofi_build_mode='dl' - echo "libfabric build completed" - """ - } - } - } - stage('build-fabtests') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'fabtests' - python3.7 contrib/intel/jenkins/build.py 'fabtests' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'fabtests' --ofi_build_mode='dl' - echo 'fabtests build completed' - """ - } - } - } - stage ('build-shmem') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'shmem' - python3.7 contrib/intel/jenkins/build.py 'shmem' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'shmem' --ofi_build_mode='dl' - echo 'shmem benchmarks built successfully' - """ - } - } - } - stage ('build OMPI_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' - python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'ompi_benchmarks' --ofi_build_mode='dl' - echo 'mpi benchmarks with ompi - built successfully' - """ - } - } - } - stage('build IMPI_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'impi_benchmarks' - python3.7 contrib/intel/jenkins/build.py 'impi_benchmarks' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'impi_benchmarks' --ofi_build_mode='dl' - echo 'mpi benchmarks with impi - built successfully' - """ - } - } - } - stage('build MPICH_bm') { - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin']) { - sh """ - python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' - python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dbg' - python3.7 contrib/intel/jenkins/build.py 'mpich_benchmarks' --ofi_build_mode='dl' - echo "mpi benchmarks with mpich - built successfully" - """ - } - } - } - stage('parallel-tests') { - parallel { - stage('eth-sockets') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=sockets - ) - """ - } - } - } - stage('eth-tcp') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=tcp - ) - """ - } - } - } - stage('eth-udp-rxd-shm') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=udp - python3.7 runtests.py --prov=udp --util=rxd - python3.7 runtests.py --prov=shm - ) - """ - } - } - } - stage('mlx5-verbs-rxm') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs - python3.7 runtests.py --prov=verbs --util=rxm - - ) - """ - } - } - } - stage('mlx5-verbs-rxd') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --util=rxd - ) - """ - } - } - } - stage('eth-sockets-dbg') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=sockets --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('eth-tcp-dbg') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=tcp --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('eth-udp-rxd-shm-dbg') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=udp --ofi_build_mode='dbg' - python3.7 runtests.py --prov=udp --util=rxd --ofi_build_mode='dbg' - python3.7 runtests.py --prov=shm --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('mlx5-verbs_rxm-dbg') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --ofi_build_mode='dbg' - python3.7 runtests.py --prov=verbs --util=rxm --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('mlx5-verbs_rxd-dbg') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --util=rxd --ofi_build_mode='dbg' - ) - """ - } - } - } - stage('eth-sockets-dl') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=sockets --ofi_build_mode='dl' - ) - """ - } - } - } - stage('eth-tcp-dl') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=tcp --ofi_build_mode='dl' - ) - """ - } - } - } - stage('eth-udp-rxd-shm-dl') { - agent {node {label 'eth'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin/:$PYTHONPATH']) - { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=udp --ofi_build_mode='dl' - python3.7 runtests.py --prov=udp --util=rxd --ofi_build_mode='dl' - python3.7 runtests.py --prov=shm --ofi_build_mode='dl' - ) - """ - } - } - } - stage('mlx5-verbs_rxm-dl') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --ofi_build_mode='dl' - python3.7 runtests.py --prov=verbs --util=rxm --ofi_build_mode='dl' - ) - """ - } - } - } - stage('mlx5-verbs_rxd-dl') { - agent {node {label 'mlx5'}} - steps { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh """ - env - ( - cd ${env.WORKSPACE}/contrib/intel/jenkins/ - python3.7 runtests.py --prov=verbs --util=rxd --ofi_build_mode='dl' - ) - """ - } - } - } - } - } - } - - post { - failure { - mail from: 'notification@jenkins-ci.org', - to: "${env.mailrecepient}", - subject: "${env.JOB_NAME} - Build # ${env.BUILD_NUMBER} - ${currentBuild.result}!", - body: " Check console output at ${env.BUILD_URL} to view the results." - - } - cleanup { - withEnv(['PATH+EXTRA=/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/bin:$PYTHONPATH']) { - sh "rm -rf '/mpibuilddir/mpich-build-dir/${env.JOB_NAME}/${env.BUILD_NUMBER}'" - sh "rm -rf '/mpibuilddir/ompi-build-dir/${env.JOB_NAME}/${env.BUILD_NUMBER}'" - sh "rm -rf '/mpibuilddir/mpich-suite-build-dir/${env.JOB_NAME}/${env.BUILD_NUMBER}'" - dir("${env.WORKSPACE}") { - deleteDir() - } - } - } - } -} diff --git a/contrib/intel/jenkins/build.py b/contrib/intel/jenkins/build.py deleted file mode 100755 index ce78f943c4b..00000000000 --- a/contrib/intel/jenkins/build.py +++ /dev/null @@ -1,101 +0,0 @@ -import os -import sys - -# add jenkins config location to PATH -sys.path.append(os.environ['CI_SITE_CONFIG']) - -import ci_site_config -import argparse -import subprocess -import shlex -import common -import re -import shutil - - -def build_libfabric(libfab_install_path, mode): - - if (os.path.exists(libfab_install_path) != True): - os.makedirs(libfab_install_path) - - config_cmd = ['./configure', '--prefix={}'.format(libfab_install_path)] - enable_prov_val = 'yes' - - if (mode == 'dbg'): - config_cmd.append('--enable-debug') - elif (mode == 'dl'): - enable_prov_val='dl' - - for prov in common.enabled_prov_list: - config_cmd.append('--enable-{}={}'.format(prov, enable_prov_val)) - for prov in common.disabled_prov_list: - config_cmd.append('--enable-{}=no'.format(prov)) - - common.run_command(['./autogen.sh']) - common.run_command(shlex.split(" ".join(config_cmd))) - common.run_command(['make','clean']) - common.run_command(['make']) - common.run_command(['make','install']) - - -def build_fabtests(libfab_install_path, mode): - - os.chdir('{}/fabtests'.format(workspace)) - if (mode == 'dbg'): - config_cmd = ['./configure', '--enable-debug', '--prefix={}' \ - .format(libfab_install_path),'--with-libfabric={}' \ - .format(libfab_install_path)] - else: - config_cmd = ['./configure', '--prefix={}'.format(libfab_install_path), - '--with-libfabric={}'.format(libfab_install_path)] - - common.run_command(['./autogen.sh']) - common.run_command(config_cmd) - common.run_command(['make','clean']) - common.run_command(['make']) - common.run_command(['make', 'install']) - -def copy_build_dir(install_path): - shutil.copytree(ci_site_config.build_dir, '{}/ci_middlewares'.format(install_path)) - -if __name__ == "__main__": -#read Jenkins environment variables - # In Jenkins, JOB_NAME = 'ofi_libfabric/master' vs BRANCH_NAME = 'master' - # job name is better to use to distinguish between builds of different - # jobs but with same branch name. - jobname = os.environ['JOB_NAME'] - buildno = os.environ['BUILD_NUMBER'] - workspace = os.environ['WORKSPACE'] - - parser = argparse.ArgumentParser() - parser.add_argument("build_item", help="build libfabric or fabtests", - choices=['libfabric','fabtests', 'builddir']) - parser.add_argument("--ofi_build_mode", help="select buildmode debug or dl", \ - choices=['dbg','dl']) - - args = parser.parse_args() - build_item = args.build_item - - if (args.ofi_build_mode): - ofi_build_mode = args.ofi_build_mode - else: - ofi_build_mode = 'reg' - - ci_middlewares_install_path = "{installdir}/{jbname}/{bno}" \ - .format(installdir=ci_site_config.install_dir, \ - jbname=jobname, bno=buildno) - install_path = "{installdir}/{jbname}/{bno}/{bmode}" \ - .format(installdir=ci_site_config.install_dir, - jbname=jobname, bno=buildno,bmode=ofi_build_mode) - - p = re.compile('mpi*') - - if (build_item == 'libfabric'): - build_libfabric(install_path, ofi_build_mode) - - elif (build_item == 'fabtests'): - build_fabtests(install_path, ofi_build_mode) - - elif (build_item == 'builddir'): - copy_build_dir(ci_middlewares_install_path) - diff --git a/contrib/intel/jenkins/common.py b/contrib/intel/jenkins/common.py deleted file mode 100755 index f851e477f49..00000000000 --- a/contrib/intel/jenkins/common.py +++ /dev/null @@ -1,55 +0,0 @@ -import collections -import ci_site_config -import subprocess -import sys - -def get_node_name(host, interface): - # This is the pattern we follow in SFS team cluster - return "%s-%s" % (host, interface) - -def run_command(command): - print(" ".join(command)) - p = subprocess.Popen(command, stdout=subprocess.PIPE, text=True) - print(p.returncode) - while True: - out = p.stdout.read(1) - if (out == "" and p.poll() != None): - break - if (out != ""): - sys.stdout.write(out) - sys.stdout.flush() - if (p.returncode != 0): - print("exiting with " + str(p.poll())) - sys.exit(p.returncode) - - -Prov = collections.namedtuple('Prov', 'core util') -prov_list = [ - Prov("psm3", None), - Prov("verbs", None), - Prov("verbs", "rxd"), - Prov("verbs", "rxm"), - Prov("sockets", None), - Prov("tcp", None), - Prov("udp", None), - Prov("udp", "rxd"), - Prov("shm", None), -] -enabled_prov_list = [ - "verbs", - "tcp", - "sockets", - "udp", - "shm" -] -disabled_prov_list = [ - "psm3", - 'usnic', - 'psm', - 'efa', - 'perf', - 'rstream', - 'hook_debug', - 'bgq' - 'mrail' -] diff --git a/contrib/intel/jenkins/run.py b/contrib/intel/jenkins/run.py deleted file mode 100755 index b41a38c35b9..00000000000 --- a/contrib/intel/jenkins/run.py +++ /dev/null @@ -1,114 +0,0 @@ -import tests -import subprocess -import sys -import argparse -import os -import common - -sys.path.append(os.environ['CI_SITE_CONFIG']) -import ci_site_config - -# read Jenkins environment variables -# In Jenkins, JOB_NAME = 'ofi_libfabric/master' vs BRANCH_NAME = 'master' -# job name is better to use to distinguish between builds of different -# jobs but with the same branch name. -fab = os.environ['FABRIC']#args.fabric -jbname = os.environ['JOB_NAME']#args.jobname -bno = os.environ['BUILD_NUMBER']#args.buildno - - -#run fi_info test -def fi_info_test(core, hosts, mode,util=None): - - fi_info_test = tests.FiInfoTest(jobname=jbname,buildno=bno,\ - testname="fi_info", core_prov=core, fabric=fab,\ - hosts=hosts, ofi_build_mode=mode, util_prov=util) - print("running fi_info test for {}-{}-{}".format(core, util, fab)) - fi_info_test.execute_cmd() - - -#runfabtests -def fabtests(core, hosts, mode, util=None): - - runfabtest = tests.Fabtest(jobname=jbname,buildno=bno,\ - testname="runfabtests", core_prov=core, fabric=fab,\ - hosts=hosts, ofi_build_mode=mode, util_prov=util) - - if (runfabtest.execute_condn): - print("running fabtests for {}-{}-{}".format(core, util, fab)) - runfabtest.execute_cmd() - else: - print("skipping {} as execute condition fails"\ - .format(runfabtest.testname)) - print("----------------------------------------------------------------------------------------\n") - - -def shmemtest(core, hosts, mode, util=None): - - runshmemtest = tests.ShmemTest(jobname=jbname,buildno=bno,\ - testname="shmem test", core_prov=core, fabric=fab,\ - hosts=hosts, ofi_build_mode=mode, util_prov=util) - if (runshmemtest.execute_condn): - print("running shmem unit test for {}-{}-{}".format(core, util, fab)) - runshmemtest.execute_cmd("unit") - print("running shmem PRK test for {}-{}-{}".format(core, util, fab)) - runshmemtest.execute_cmd("prk") - print("running shmem ISx test for {}-{}-{}".format(core, util, fab)) - runshmemtest.execute_cmd("isx") - print("running shmem uh test for {}-{}-{}".format(core, util, fab)) - runshmemtest.execute_cmd("uh") - else: - print("skipping {} as execute condition fails"\ - .format(runshmemtest.testname)) - print("----------------------------------------------------------------------------------------\n") - - -#imb-tests -def intel_mpi_benchmark(core, hosts, mpi, mode, util=None): - - imb_test = tests.MpiTestIMB(jobname=jbname,buildno=bno,\ - testname="IntelMPIbenchmark",core_prov=core, fabric=fab,\ - hosts=hosts, mpitype=mpi, ofi_build_mode=mode, util_prov=util) - - if (imb_test.execute_condn == True and imb_test.mpi_gen_execute_condn == True): - print("running imb-tests for {}-{}-{}-{}".format(core, util, fab, mpi)) - imb_test.execute_cmd() - else: - print("skipping {} as execute condition fails"\ - .format(imb_test.testname)) - print("----------------------------------------------------------------------------------------\n") - - -#mpich_test_suite -def mpich_test_suite(core, hosts, mpi, mode, util=None): - - mpich_tests = tests.MpichTestSuite(jobname=jbname,buildno=bno,\ - testname="MpichTestSuite",core_prov=core, fabric=fab,\ - mpitype=mpi, hosts=hosts, ofi_build_mode=mode, \ - util_prov=util) - if (mpich_tests.execute_condn == True and \ - mpich_tests.mpi_gen_execute_condn == True): - print("Running mpich test suite: Spawn coll, comm, dt Tests for {}-{}-{}-{}".format(core, util, fab, mpi)) - os.environ["MPITEST_RETURN_WITH_CODE"] = "1" - mpich_tests.execute_cmd("spawn") - print("----------------------------------------------------------------------------------------\n") - - -#osu benchmark tests -def osu_benchmark(core, hosts, mpi, mode, util=None): - - osu_test = tests.MpiTestOSU(jobname=jbname, buildno=bno, \ - testname="osu-benchmarks",core_prov=core, fabric=fab, mpitype=mpi, \ - hosts=hosts, ofi_build_mode=mode, util_prov=util) - - if (osu_test.execute_condn == True and osu_test.mpi_gen_execute_condn == True): - print("running osu-test for {}-{}-{}-{}".format(core, util, fab, mpi)) - osu_test.execute_cmd() - else: - print("skipping {} as execute condition fails" \ - .format(osu_test.testname)) - print("----------------------------------------------------------------------------------------\n") - - -if __name__ == "__main__": - pass diff --git a/contrib/intel/jenkins/runtests.py b/contrib/intel/jenkins/runtests.py deleted file mode 100755 index e4637587c9e..00000000000 --- a/contrib/intel/jenkins/runtests.py +++ /dev/null @@ -1,85 +0,0 @@ -import argparse -import os -import sys -sys.path.append(os.environ['CI_SITE_CONFIG']) -import ci_site_config -import run -import common - -parser = argparse.ArgumentParser() - -parser.add_argument("--prov", help="core provider", choices=["verbs", \ - "tcp", "udp", "sockets", "shm"]) -parser.add_argument("--util", help="utility provider", choices=["rxd", "rxm"]) -parser.add_argument("--ofi_build_mode", help="specify the build configuration", \ - choices = ["dbg", "dl"]) -parser.add_argument("--test", help="specify test to execute", \ - choices = ["all", "unit", "shmem", "mpi"]) - -args = parser.parse_args() -args_core = args.prov - -args_util = args.util - -if (args.ofi_build_mode): - ofi_build_mode = args.ofi_build_mode -else: - ofi_build_mode='reg' - -if (args.test): - run_test = args.test -else: - run_test = 'all' - -node = (os.environ['NODE_NAME']).split('-')[0] -hosts = [node] -# Note: Temporarily disabling all mpich testing -# due to mpich options issues which is causing -# multiple tests to fail. -#mpilist = ['impi', 'mpich', 'ompi'] -mpilist = ['impi', 'ompi'] - -#this script is executed from /tmp -#this is done since some mpi tests -#look for a valid location before running -# the test on the secondary host(client) -# but jenkins only creates a valid path on -# the primary host (server/test node) - -os.chdir('/tmp/') - -if(args_core): - for host in ci_site_config.node_map[node]: - hosts.append(host) - - if (args_util == None): - if (run_test == 'all' or run_test == 'unit'): - run.fi_info_test(args_core, hosts, ofi_build_mode) - run.fabtests(args_core, hosts, ofi_build_mode) - - if (run_test == 'all' or run_test == 'shmem'): - run.shmemtest(args_core, hosts, ofi_build_mode) - - if (run_test == 'all' or run_test == 'all'): - for mpi in mpilist: - run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode) - run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode) - run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode) - else: - if (run_test == 'all' or run_test == 'unit'): - run.fi_info_test(args_core, hosts, ofi_build_mode, util=args_util) - run.fabtests(args_core, hosts, ofi_build_mode, util=args_util) - - if (run_test == 'all' or run_test == 'shmem'): - run.shmemtest(args_core, hosts, ofi_build_mode, util=args_util) - - if (run_test == 'all' or run_test == 'all'): - for mpi in mpilist: - run.mpich_test_suite(args_core, hosts, mpi, ofi_build_mode, \ - util=args_util) - run.intel_mpi_benchmark(args_core, hosts, mpi, ofi_build_mode, \ - util=args_util) - run.osu_benchmark(args_core, hosts, mpi, ofi_build_mode, \ - util=args_util) -else: - print("Error : Specify a core provider to run tests") diff --git a/contrib/intel/jenkins/tests.py b/contrib/intel/jenkins/tests.py deleted file mode 100755 index 793991ccb76..00000000000 --- a/contrib/intel/jenkins/tests.py +++ /dev/null @@ -1,502 +0,0 @@ -import sys -import os - -print(os.environ['CI_SITE_CONFIG']) -sys.path.append(os.environ['CI_SITE_CONFIG']) # for adding path for ci_site_config - -import subprocess -import re -import ci_site_config -import common -import shlex -from abc import ABC, abstractmethod # abstract base class for creating abstract classes in python - -job_cadence = os.environ['JOB_CADENCE'] - - -# A Jenkins env variable for job name is composed of the name of the jenkins job and the branch name -# it is building for. for e.g. in our case jobname = 'ofi_libfabric/master' -class Test: - - def __init__ (self, jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov=None): - self.jobname = jobname - self.buildno = buildno - self.testname = testname - self.core_prov = core_prov - self.util_prov = "ofi_{}".format(util_prov) if util_prov != None else "" - self.fabric = fabric - self.hosts = hosts - self.ofi_build_mode = ofi_build_mode - self.job_cadence = job_cadence - if (len(hosts) == 2): - self.server = hosts[0] - self.client = hosts[1] - - self.nw_interface = ci_site_config.interface_map[self.fabric] - self.libfab_installpath = "{}/{}/{}/{}".format(ci_site_config.install_dir, - self.jobname, self.buildno, self.ofi_build_mode) - self.ci_middlewares_path = "{}/{}/{}/ci_middlewares" \ - .format(ci_site_config.install_dir, \ - self.jobname, self.buildno) - - self.env = [("FI_VERBS_MR_CACHE_ENABLE", "1"),\ - ("FI_VERBS_INLINE_SIZE", "256")] \ - if self.core_prov == "verbs" else [] - - -class FiInfoTest(Test): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov=None): - - super().__init__(jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov) - - self.fi_info_testpath = "{}/bin".format(self.libfab_installpath) - - @property - def cmd(self): - return "{}/fi_info ".format(self.fi_info_testpath) - - @property - def options(self): - if (self.util_prov): - opts = "-f {} -p {};{}".format(self.fabric, self.core_prov, self.util_prov) - else: - opts = "-f {} -p {}".format(self.fabric, self.core_prov) - - return opts - - def execute_cmd(self): - command = self.cmd + self.options - outputcmd = shlex.split(command) - common.run_command(outputcmd) - - -class Fabtest(Test): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov=None): - - super().__init__(jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov) - self.fabtestpath = "{}/bin".format(self.libfab_installpath) - self.fabtestconfigpath = "{}/share/fabtests".format(self.libfab_installpath) - def get_exclude_file(self): - path = self.libfab_installpath - efile_path = "{}/share/fabtests/test_configs".format(path) - - prov = self.util_prov if self.util_prov else self.core_prov - efile_old = "{path}/{prov}/{prov}.exclude".format(path=efile_path, - prov=prov) - - if self.util_prov: - efile = "{path}/{util_prov}/{core_prov}/exclude".format(path=efile_path, - util_prov=self.util_prov, core_prov=self.core_prov) - else: - efile = "{path}/{prov}/exclude".format(path=efile_path, - prov=self.core_prov) - - if os.path.isfile(efile): - return efile - elif os.path.isfile(efile_old): - return efile_old - else: - print("Exclude file: {} not found!".format(efile)) - return None - - @property - def cmd(self): - return "{}/runfabtests.sh ".format(self.fabtestpath) - - @property - def options(self): - opts = "-T 300 -vvv -p {} -S ".format(self.fabtestpath) - if (self.core_prov == "verbs" and self.nw_interface): - opts = "{} -s {} ".format(opts, common.get_node_name(self.server, - self.nw_interface)) # include common.py - opts = "{} -c {} ".format(opts, common.get_node_name(self.client, - self.nw_interface)) # from common.py - - if (self.core_prov == "shm"): - opts = "{} -s {} ".format(opts, self.server) - opts = "{} -c {} ".format(opts, self.client) - opts += "-N " - - if not re.match(".*sockets|udp.*", self.core_prov): - opts = "{} -t all ".format(opts) - - efile = self.get_exclude_file() - if efile: - opts = "{} -R ".format(opts) - opts = "{} -f {} ".format(opts, efile) - - for key,val in self.env: - opts = "{options} -E {key}={value} ".format(options = opts, - key=key, value=val) - - if self.util_prov: - opts = "{options} {core};{util} ".format(options=opts, - core=self.core_prov, util=self.util_prov) - else: - opts = "{options} {core} ".format(options=opts, - core=self.core_prov) - - if (self.core_prov == "shm"): - opts += "{} {} ".format(self.server, self.server) - else: - opts += "{} {} ".format(self.server, self.client) - - return opts - - @property - def execute_condn(self): - return True if (self.core_prov != 'shm' or \ - self.ofi_build_mode == 'dbg') else False - - def execute_cmd(self): - curdir = os.getcwd() - os.chdir(self.fabtestconfigpath) - command = self.cmd + self.options - outputcmd = shlex.split(command) - common.run_command(outputcmd) - os.chdir(curdir) - - -class ShmemTest(Test): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov=None): - - super().__init__(jobname, buildno, testname, core_prov, fabric, - hosts, ofi_build_mode, util_prov) - - #self.n - number of hosts * number of processes per host - self.n = 4 - # self.ppn - number of processes per node. - self.ppn = 2 - self.shmem_dir = "{}/shmem".format(self.ci_middlewares_path) - - @property - def cmd(self): - #todo: rename mpi_testpath to testpath to make it generic for shmem and mpitest - return "{}/run_shmem.sh ".format(ci_site_config.mpi_testpath) - - def options(self, shmem_testname): - - if self.util_prov: - prov = "{core};{util} ".format(core=self.core_prov, - util=self.util_prov) - else: - prov = self.core_prov - - opts = "-n {n} -hosts {server},{client} -shmem_dir={shmemdir} \ - -libfabric_path={path}/lib -prov '{provider}' -test {test} \ - -server {server} -inf {inf}" \ - .format(n=self.n, server=self.server, client=self.client, \ - shmemdir=self.shmem_dir, path=self.libfab_installpath, \ - provider=prov, test=shmem_testname, \ - inf=ci_site_config.interface_map[self.fabric]) - return opts - - @property - def execute_condn(self): - return True if (self.job_cadence == 'daily' and \ - self.core_prov == "sockets") \ - else False - - def execute_cmd(self, shmem_testname): - command = self.cmd + self.options(shmem_testname) - outputcmd = shlex.split(command) - common.run_command(outputcmd) - - -class MpiTests(Test): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov=None): - - super().__init__(jobname, buildno, testname, core_prov, - fabric, hosts, ofi_build_mode, util_prov) - self.mpi = mpitype - - @property - def cmd(self): - if (self.mpi == "impi" or self.mpi == "mpich"): - self.testpath = ci_site_config.mpi_testpath - return "{}/run_{}.sh ".format(self.testpath,self.mpi) - elif(self.mpi =="ompi"): - self.testpath = "{}/ompi/bin".format(self.ci_middlewares_path) - return "{}/mpirun ".format(self.testpath) - - @property - def options(self): - opts = [] - if (self.mpi == "impi" or self.mpi == "mpich"): - opts = "-n {} -ppn {} -hosts {},{} ".format(self.n,self.ppn, - self.server,self.client) - - if (self.mpi == "impi"): - opts = "{} -mpi_root={} ".format(opts, - ci_site_config.impi_root) - else: - opts = "{} -mpi_root={}/mpich".format(opts, - self.ci_middlewares_path) - - opts = "{} -libfabric_path={}/lib ".format(opts, - self.libfab_installpath) - - if self.util_prov: - opts = "{options} -prov {core};{util} ".format(options=opts, - core=self.core_prov, util=self.util_prov) - else: - opts = "{} -prov {} ".format(opts, self.core_prov) - - for key, val in self.env: - opts = "{} -genv {} {} ".format(opts, key, val) - - elif (self.mpi == "ompi"): - opts = "-np {} ".format(self.n) - hosts = ",".join([":".join([host,str(self.ppn)]) \ - for host in self.hosts]) - - opts = "{} --host {} ".format(opts, hosts) - - if self.util_prov: - opts = "{} --mca mtl_ofi_provider_include {};{} ".format(opts, - self.core_prov,self.util_prov) - else: - opts = "{} --mca mtl_ofi_provider_include {} ".format(opts, - self.core_prov) - - opts += "--mca orte_base_help_aggregate 0 " - opts += "--mca mtl ofi --mca pml cm -tag-output " - for key,val in self.env: - opts = "{} -x {}={} ".format(opts,key,val) - return opts - - @property - def mpi_gen_execute_condn(self): - #Skip MPI tests for udp, verbs(core) providers. - # we would still have MPI tests runnning for - # verbs-rxd and verbs-rxm providers - return True if (self.core_prov != "udp" and \ - self.core_prov != "shm" and \ - (self.core_prov != "verbs" or \ - self.util_prov == "ofi_rxm" or \ - self.util_prov == "ofi_rxd")) else False - - -# IMBtests serves as an abstract class for different -# types of intel MPI benchmarks. Currently we have -# the mpi1 and rma tests enabled which are encapsulated -# in the IMB_mpi1 and IMB_rma classes below. -class IMBtests(ABC): - """ - This is an abstract class for IMB tests. - currently IMB-MPI1 and IMB-RMA tests are - supported. In future there could be more. - All abstract methods must be implemented. - """ - - @property - @abstractmethod - def imb_cmd(self): - pass - - @property - @abstractmethod - def execute_condn(self): - pass - - -class IMBmpi1(IMBtests): - - def __init__(self): - self.additional_tests = [ - "Biband", - "Uniband", - "PingPongAnySource", - "PingPingAnySource", - "PingPongSpecificSource", - "PingPingSpecificSource" - ] - - @property - def imb_cmd(self): - return "{}/bin/IMB-MPI1 -include {}".format(ci_site_config.impi_root, \ - ','.join(self.additional_tests)) - - @property - def execute_condn(self): - return True - - -class IMBrma(IMBtests): - def __init__(self, core_prov): - self.core_prov = core_prov - - @property - def imb_cmd(self): - return "{}/bin/IMB-RMA".format(ci_site_config.impi_root) - - @property - def execute_condn(self): - return True if (self.core_prov != "verbs") else False - - -# MpiTestIMB class inherits from the MPITests class. -# It uses the same options method and class variables as all MPI tests. -# It creates IMB_xxx test objects for each kind of IMB test. -class MpiTestIMB(MpiTests): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov=None): - super().__init__(jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov) - - self.n = 4 - self.ppn = 1 - self.mpi1 = IMBmpi1() - self.rma = IMBrma(self.core_prov) - - @property - def execute_condn(self): - return True if (self.mpi == "impi") else False - - def execute_cmd(self): - command = self.cmd + self.options - if(self.mpi1.execute_condn): - outputcmd = shlex.split(command + self.mpi1.imb_cmd) - common.run_command(outputcmd) - if (self.rma.execute_condn): - outputcmd = shlex.split(command + self.rma.imb_cmd) - common.run_command(outputcmd) - - -class MpichTestSuite(MpiTests): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov=None): - super().__init__(jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov) - self.mpichsuitepath = "{}/{}/mpichsuite/test/mpi/" \ - .format(self.ci_middlewares_path, self.mpi) - self.pwd = os.getcwd() - - def testgroup(self, testgroupname): - - testpath = "{}/{}".format(self.mpichsuitepath, testgroupname) - tests = [] - with open("{}/testlist".format(testpath)) as file: - for line in file: - if(line[0] != '#' and line[0] != '\n'): - tests.append((line.rstrip('\n')).split(' ')) - - return tests - - def options(self, nprocs, timeout=None): - if (self.mpi == "impi" or self.mpi == "mpich"): - if (self.mpi == "impi"): - mpiroot = ci_site_config.impi_root - else: - mpiroot = "{}/mpich".format(self.ci_middlewares_path) - if (self.util_prov): - prov = "\"{};{}\"".format(self.core_prov, self.util_prov) - else: - prov = self.core_prov - - if (timeout != None): - os.environ['MPIEXEC_TIMEOUT']=timeout - - opts = "-n {np} -hosts {s},{c} -mpi_root={mpiroot} \ - -libfabric_path={installpath}/lib -prov {provider} " \ - .format(np=nprocs, s=self.server, c=self.client, \ - provider=prov, mpiroot=mpiroot, \ - installpath=self.libfab_installpath) - - elif (self.mpi == "ompi"): - print(self.mpi) - - return opts - - @property - def execute_condn(self): - return True if (self.mpi == 'impi' \ - and self.core_prov != 'sockets') else False - - def execute_cmd(self, testgroupname): - print("Running Tests: " + testgroupname) - tests = [] - time = None - os.chdir("{}/{}".format(self.mpichsuitepath,testgroupname)) - tests = self.testgroup(testgroupname) - for test in tests: - testname = test[0] - nprocs = test[1] - args = test[2:] - for item in args: - itemlist = item.split('=') - if (itemlist[0] == 'timelimit'): - time = itemlist[1] - opts = self.options(nprocs, timeout=time) - testcmd = self.cmd + opts +"./{}".format(testname) - outputcmd = shlex.split(testcmd) - common.run_command(outputcmd) - os.chdir(self.pwd) - - -class MpiTestOSU(MpiTests): - - def __init__(self, jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov=None): - super().__init__(jobname, buildno, testname, core_prov, fabric, - mpitype, hosts, ofi_build_mode, util_prov) - - self.n = 4 - self.ppn = 2 - self.two_proc_tests = {'osu_latency', - 'osu_bibw', - 'osu_latency_mt', - 'osu_bw','osu_get_latency', - 'osu_fop_latency', - 'osu_acc_latency', - 'osu_get_bw', - 'osu_put_latency', - 'osu_put_bw', - 'osu_put_bibw', - 'osu_cas_latency', - 'osu_get_acc_latency' - } - - self.osu_mpi_path = "{}/{}/osu/libexec/osu-micro-benchmarks/mpi/". \ - format(self.ci_middlewares_path, mpitype) - - @property - def execute_condn(self): - # sockets have some issues with OSU benchmark testing. - return True if ((self.job_cadence == 'daily') and \ - (self.mpi != "ompi" or \ - (self.core_prov != "sockets" and \ - self.ofi_build_mode!="dbg"))) \ - else False - - def execute_cmd(self): - assert(self.osu_mpi_path) - p = re.compile('osu_put*') - for root, dirs, tests in os.walk(self.osu_mpi_path): - for test in tests: - if test in self.two_proc_tests: - self.n=2 - self.ppn=1 - else: - self.n=4 - self.ppn=2 - # for sockets provider skip 'osu_put' benchmark tests as they fail. - if(self.core_prov !='sockets' or p.search(test)== None): - launcher = self.cmd + self.options - osu_cmd = os.path.join(root, test) - command = launcher + osu_cmd - outputcmd = shlex.split(command) - common.run_command(outputcmd)