[CI] use sklearnex azure pipelines template for sklearnex CI step (#2950

) * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update apt.sh * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * remove sklearn conformance scripts * Update apt.sh * Update ci.yml * Update apt.sh
uxlfoundation · Dec 10, 2024 · a38140a · a38140a
1 parent bc1de4d
commit a38140a
Show file tree

Hide file tree

Showing 9 changed files with 44 additions and 530 deletions.
diff --git a/.ci/env/apt.sh b/.ci/env/apt.sh
@@ -34,8 +34,13 @@ function install_dpcpp {
     sudo apt-get install -y intel-oneapi-compiler-dpcpp-cpp-2025.0 intel-oneapi-runtime-libs=2025.0.0-406
 }
 
+function install_tbb {
+    sudo apt-get install -y intel-oneapi-tbb-devel-2022.0
+}
+
 function install_mkl {
-    sudo apt-get install -y intel-oneapi-mkl-devel-2025.0 intel-oneapi-tbb-devel-2022.0
+    sudo apt-get install -y intel-oneapi-mkl-devel-2025.0
+    install_tbb
 }
 
 function install_clang-format {
@@ -97,6 +102,9 @@ function build_sysroot {
 if [ "${component}" == "dpcpp" ]; then
     add_repo
     install_dpcpp
+elif [ "${component}" == "tbb" ]; then
+    add_repo
+    install_tbb
 elif [ "${component}" == "mkl" ]; then
     add_repo
     install_mkl
@@ -126,6 +134,6 @@ elif [ "${component}" == "build-sysroot" ] ; then
     build_sysroot "$2" "$3" "$4" "$5"
 else
     echo "Usage:"
-    echo "   $0 [dpcpp|mkl|gnu-cross-compilers|clang-format|dev-base|qemu-apt|qemu-deb|llvm-version|build-sysroot]"
+    echo "   $0 [dpcpp|tbb|mkl|gnu-cross-compilers|clang-format|dev-base|qemu-apt|qemu-deb|llvm-version|build-sysroot]"
     exit 1
 fi
diff --git a/.ci/pipeline/ci.yml b/.ci/pipeline/ci.yml
@@ -31,6 +31,14 @@ variables:
   WINDOWS_BASEKIT_URL: 'https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b380d914-366b-4b77-a74a-05e3c38b3514/intel-oneapi-base-toolkit-2025.0.0.882_offline.exe'
   WINDOWS_DPCPP_COMPONENTS: 'intel.oneapi.win.mkl.devel:intel.oneapi.win.tbb.devel'
 
+resources:
+  repositories:
+    - repository: sklearnex
+      type: github
+      name: uxlfoundation/scikit-learn-intelex
+      ref: main
+      endpoint: intel-daal-connection
+
 jobs:
 - job: 'FormatterChecks'
   pool:
@@ -81,6 +89,12 @@ jobs:
       targetPath: '$(Build.Repository.LocalPath)/$(release.dir)'
     displayName: 'Upload build artifacts'
     continueOnError: true
+  - task: PublishPipelineArtifact@1
+    inputs:
+      artifactName: 'oneDAL environment'
+      targetPath: '$(Build.Repository.LocalPath)/.ci/env'
+    displayName: 'Upload environment artifacts'
+    continueOnError: true
   - script: |
       source /opt/intel/oneapi/setvars.sh
       .ci/scripts/test.sh --test-kind examples --build-dir $(release.dir) --compiler gnu --interface daal/cpp --build-system cmake
@@ -582,83 +596,41 @@ jobs:
   timeoutInMinutes: 0
   variables:
     release.dir: '__release_lnx_gnu'
-    platform.type : 'lnx32e'
+    platform.type: 'lnx32e'
+    DALROOT: '$(Pipeline.Workspace)/daal/latest'
+    TBBROOT: '/opt/intel/oneapi/tbb/latest'
+    NO_DPC: 1
+    NO_DIST: 1
   pool:
     vmImage: '$(VM_IMAGE)'
     maxParallel: 2
   strategy:
     matrix:
       Python311:
-        python.version: '3.11'
+        PYTHON_VERSION: '3.11'
+        SKLEARN_VERSION: '1.5'
   steps:
+  - checkout: none
+  - script: |
+      git clone https://github.com/uxlfoundation/scikit-learn-intelex.git .
+    displayName: 'Clone sklearnex'
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      artifact: 'oneDAL environment'
+      path: '$(Build.Repository.LocalPath)/.ci/env'
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: '$(platform.type) build'
       path: $(Pipeline.Workspace)
-  - script: .ci/env/apt.sh dev-base
-    displayName: 'apt-get'
   - script: |
-      .ci/scripts/describe_system.sh
-    displayName: 'System info'
-  - script: |
-      conda update -y -q conda
-      conda create -q -y -n CB -c conda-forge python=$(python.version) mpich
-    displayName: 'Conda create'
-  - script: |
-      git clone https://github.com/uxlfoundation/scikit-learn-intelex.git sklearnex
-    displayName: Clone sklearnex
-  - script: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      pip install -r sklearnex/dependencies-dev
-      pip install -r sklearnex/requirements-test.txt
-      pip install tbb==2022.*
-    displayName: Create python environment
-  - script: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      export DALROOT=$(Pipeline.Workspace)/daal/latest
-      source ${DALROOT}/env/vars.sh
-      export PREFIX=${CONDA_PREFIX}
-      cd sklearnex
-      export PYTHON=python
-      ./conda-recipe/build.sh
-    displayName: sklearnex build
+      chmod -R 755 .ci/env
+      .ci/env/apt.sh tbb
+    displayName: 'tbb installation'
+  - template: .ci/pipeline/build-and-test-lnx.yml@sklearnex
   - task: PublishPipelineArtifact@1
     inputs:
       artifactName: '$(platform.type) sklearnex build'
-      targetPath: '$(Build.Repository.LocalPath)/sklearnex'
-    displayName: 'Upload sklearnex build artifacts'
-    continueOnError: true
-  - script: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      source $(Pipeline.Workspace)/daal/latest/env/vars.sh
-      ./sklearnex/conda-recipe/run_test.sh
-    timeoutInMinutes: 20
-    displayName: sklearnex test
-  - script: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      source $(Pipeline.Workspace)/daal/latest/env/vars.sh
-      ret_code=0
-      python -m sklearnex sklearnex/tests/run_examples.py
-      ret_code=$(($ret_code + $?))
-      exit $ret_code
-    displayName: sklearnex examples
-  - script: |
-      source /usr/share/miniconda/etc/profile.d/conda.sh
-      conda activate CB
-      source $(Pipeline.Workspace)/daal/latest/env/vars.sh
-      $(Pipeline.Workspace)/s/.ci/scripts/conformance_test.sh --build-dir $(Pipeline.Workspace) --conda-dir $HOME/miniconda --oneapi-dir /opt/intel/oneapi
-    timeoutInMinutes: 15
-    displayName: Conformance tests
-  - task: PublishPipelineArtifact@1
-    inputs:
-      artifactName: '$(platform.type) conformance tests report'
-      targetPath: '$(Pipeline.Workspace)/s/.ci/scripts/conformance-scripts/'
-    displayName: 'Upload conformance tests artifacts'
-    continueOnError: true
+      targetPath: '$(Build.Repository.LocalPath)/build'
 
 - job: 'WindowsMakeVC'
   timeoutInMinutes: 0

diff --git a/.ci/scripts/conformance-scripts/algorithms.txt b/.ci/scripts/conformance-scripts/algorithms.txt
diff --git a/.ci/scripts/conformance-scripts/download_tests.sh b/.ci/scripts/conformance-scripts/download_tests.sh
diff --git a/.ci/scripts/conformance-scripts/run_tests.py b/.ci/scripts/conformance-scripts/run_tests.py
diff --git a/.ci/scripts/conformance-scripts/run_tests_with_context.py b/.ci/scripts/conformance-scripts/run_tests_with_context.py
diff --git a/.ci/scripts/conformance-scripts/run_xpu_conformance.py b/.ci/scripts/conformance-scripts/run_xpu_conformance.py