[ci] Benchmark presubmit test

.github/workflows/presubmit.yml

@@ -4,6 +4,7 @@ on:
     types: [opened, synchronize, reopened]
 
 jobs:
+
   title_format:
     name: Check PR Title
     if: ${{ github.event.pull_request }}
@@ -40,7 +41,6 @@ jobs:
           git remote add upstream https://github.com/taichi-dev/taichi.git
           git fetch upstream master
           sudo apt install clang-format-10
-
       - name: Cache PIP
         uses: actions/cache@v2
         with:
@@ -50,7 +50,6 @@ jobs:
       - name: Install requirements
         run: |
           python3 -m pip install --user -r requirements_dev.txt
-
       - name: Check code format
         run: |
           python3 misc/code_format.py
@@ -59,237 +58,30 @@ jobs:
           # exit with 1 if there were differences:
           git diff _fake_squash _enforced_format --exit-code
 
-      - name: Pylint
-        run: |
-          # Make sure pylint doesn't regress
-          pylint python/taichi/ --disable=all --enable=C0415
-          if [ $? -eq 0 ]
-          then
-            echo "PASSED: pylint is happy"
-            exit 0
-          else
-            echo "FAILED: please run the pylint command above and make sure it passes"
-            exit 1
-          fi
-
-  build_and_test_cpu_required:
-    # This job will be required to pass before merging to master branch.
-    name: Required Build and Test (CPU)
+  performance_monitoring:
+    name: Performance monitoring (NVGPU)
     needs: check_code_format
     timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: ubuntu-latest
-            python: 3.6
-    runs-on: ${{ matrix.os }}
+    runs-on: [self-hosted, x64, cuda, linux, benchmark]
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
 
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
-
       - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=ON -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-
-  build_and_test_cpu:
-    name: Build and Test (CPU)
-    needs: build_and_test_cpu_required
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.7
-            with_cc: OFF
-            with_cpp_tests: ON
-          - os: ubuntu-latest
-            python: 3.9
-            with_cc: OFF
-            with_cpp_tests: OFF
-          - os: ubuntu-latest
-            python: 3.8
-            with_cc: ON
-            with_cpp_tests: OFF
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
-      - name: Download Pre-Built LLVM 10.0.0
-        run: |
-          python misc/ci_download.py
-          mkdir taichi-llvm
-          cd taichi-llvm
-          unzip ../taichi-llvm.zip
-        env:
-          CI_PLATFORM: ${{ matrix.os }}
-
-      - name: Build & Install
-        run: .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CC:BOOL=${{ matrix.with_cc }} -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=${{ matrix.with_cpp_tests }}
-          CXX: clang++
-      # [DEBUG] Copy this step around to enable debugging inside Github Action instances.
-      #- name: Setup tmate session
-      #  uses: mxschmitt/action-tmate@v3
-      #  with:
-      #    limit-access-to-actor: true
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          RUN_CPP_TESTS: ${{ matrix.with_cpp_tests }}
-
-  build_and_test_gpu_linux:
-    name: Build and Test (GPU)
-    needs: check_code_format
-    runs-on: [self-hosted, cuda, vulkan, cn]
-    timeout-minutes: 60
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - name: Build
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
           .github/workflows/scripts/unix_build.sh
         env:
           LLVM_LIB_ROOT_DIR: /opt/taichi-llvm-10.0.0
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=ON -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=ON
-          BUILD_NUM_THREADS: 8
           LLVM_PATH: /opt/taichi-llvm-10.0.0/bin
           LLVM_DIR: /opt/taichi-llvm-10.0.0/lib/cmake/llvm
-          CXX: clang++-8
-
-      - name: Test
-        run: .github/workflows/scripts/unix_test.sh
-        env:
-          DISPLAY: :1
-          GPU_TEST: ON
-
-  build_and_test_windows:
-    name: Build and Test (Windows)
-    needs: check_code_format
-    runs-on: windows-latest
-    timeout-minutes: 90
-    steps:
-
-      - name: Install 7Zip PowerShell
-        shell: powershell
-        run: Install-Module 7Zip4PowerShell -Force -Verbose
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
-
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Add msbuild to PATH
-        uses: microsoft/[email protected]
-
-      - name: Download And Install Vulkan
-        shell: powershell
-        run: |
-          Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.2.189.0/windows/VulkanSDK-1.2.189.0-Installer.exe" -OutFile VulkanSDK.exe
-          $installer = Start-Process -FilePath VulkanSDK.exe -Wait -PassThru -ArgumentList @("/S");
-          $installer.WaitForExit();
-
-      - name: Build
-        shell: powershell
-        run: |
-          $env:Path += ";C:/VulkanSDK/1.2.189.0/Bin"
-          cd C:\
-          Remove-item alias:curl
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/taichi-llvm-10.0.0-msvc2019.zip -LO
-          7z x taichi-llvm-10.0.0-msvc2019.zip -otaichi_llvm
-          curl --retry 10 --retry-delay 5 https://github.com/taichi-dev/taichi_assets/releases/download/llvm10/clang-10.0.0-win.zip -LO
-          7z x clang-10.0.0-win.zip -otaichi_clang
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          clang --version
-          cd D:\a\taichi\taichi
-          python -m pip install -r requirements_dev.txt
-          cd python
-          git fetch origin master
-          $env:TAICHI_CMAKE_ARGS = $env:CI_SETUP_CMAKE_ARGS
-          python build.py build
-          cd ..\dist
-          $env:WHL = $(dir *.whl)
-          python -m pip install $env:WHL
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-          CI_SETUP_CMAKE_ARGS: -G "Visual Studio 16 2019" -A x64 -DLLVM_DIR=C:\taichi_llvm\lib\cmake\llvm -DTI_WITH_VULKAN:BOOL=ON
-          VULKAN_SDK: C:/VulkanSDK/1.2.189.0
-
-      - name: Test
-        shell: powershell
-        run: |
-          $env:PATH = ";C:\taichi_llvm\bin;C:\taichi_clang\bin;" + $env:PATH
-          python -c "import taichi"
-          python examples/algorithm/laplace.py
-          python bin/taichi diagnose
-          python bin/taichi changelog
-          python bin/taichi test -vr2 -t2
-        env:
-          PYTHON: C:\hostedtoolcache\windows\Python\3.7.9\x64\python.exe
-
-  build_and_test_m1:
-    name: Build and Test (Apple M1)
-    needs: check_code_format
-    timeout-minutes: 60
-    strategy:
-      matrix:
-        include:
-          - os: macos-latest
-            python: 3.8
-    defaults:
-      run:
-        # https://github.com/actions/runner/issues/805#issuecomment-844426478
-        shell: "/usr/bin/arch -arch arm64e /bin/bash --noprofile --norc -eo pipefail {0}"
-    runs-on: [self-hosted, m1]
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: 'recursive'
+          CUDA_TOOLKIT_ROOT_DIR: /usr/local/cuda/
+          CI_SETUP_CMAKE_ARGS: -DTI_WITH_CUDA_TOOLKIT:BOOL=ON
+          BUILD_NUM_THREADS: 8
+          CXX: clang++-10
 
-      - name: Build
+      - name: Run benchmark
         run: |
-          rm -rf $HOME/Library/Python/3.8/lib/python/site-packages/taichi
-          .github/workflows/scripts/unix_build.sh
-        env:
-          CI_SETUP_CMAKE_ARGS: -DTI_WITH_OPENGL:BOOL=OFF -DTI_WITH_CUDA:BOOL=OFF -DTI_WITH_CC:BOOL=OFF -DTI_WITH_VULKAN:BOOL=OFF -DTI_BUILD_TESTS:BOOL=ON
-          CXX: clang++
+          python3 benchmarks/misc/run.py /home/benchmarkbot/benchmark/
 
-      - name: Test
-        run: |
-          export PATH=$PATH:$HOME/Library/Python/3.8/bin
-          python3 examples/algorithm/laplace.py
-          TI_LIB_DIR=`python3 -c "import taichi;print(taichi.__path__[0])" | tail -1`
-          TI_LIB_DIR="$TI_LIB_DIR/lib" ./build/taichi_cpp_tests
-          ti test -vr2 -t4 -x

.github/workflows/scripts/unix_docker_build.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+# Parse ARGs
+for ARGUMENT in "$@"
+do
+    KEY=$(echo $ARGUMENT | cut -f1 -d=)
+    VALUE=$(echo $ARGUMENT | cut -f2 -d=)
+    case "$KEY" in
+            CI_SETUP_CMAKE_ARGS)              SHA=${VALUE} ;;
+            PY)               PY=${VALUE} ;;
+            *)
+    esac
+done
+
+source /home/dev/miniconda/etc/profile.d/conda.sh
+conda activate $PY
+
+cd taichi
+python3 -m pip install --user -r requirements_dev.txt
+python3 -m pip install torch==1.9.0+cu111 -f https://download.pytorch.org/whl/torch_stable.html
+TAICHI_CMAKE_ARGS=$CI_SETUP_CMAKE_ARGS python3 setup.py install --user
+
+export TI_IN_DOCKER=true
+
+ti diagnose
+ti test -vr2 -t2 -k "not ndarray and not torch"
+ti test -vr2 -t1 -k "ndarray or torch"

benchmarks/misc/membound.py

@@ -8,7 +8,7 @@
 test_cases = [fill, saxpy, reduction]
 test_archs = [ti.cuda]
 test_dtype = [ti.i32, ti.i64, ti.f32, ti.f64]
-test_dsize = [(4**i) * kibibyte for i in range(1, 11)]  #[4KB,16KB...1GB]
+test_dsize = [(4**i) * kibibyte for i in range(1, 10)]  #[4KB,16KB...256MB]
 test_repeat = 10
 results_evaluation = [geometric_mean]
 

benchmarks/misc/run.py

@@ -1,4 +1,9 @@
+import datetime
+import os
+import sys
+
 from membound import Membound
+from taichi.core import ti_core as _ti_core
 
 import taichi as ti
 
@@ -18,16 +23,30 @@ def run(self):
         for s in self.suites:
             s.run()
 
-    def write_md(self):
-        filename = f'performance_result.md'
-        with open(filename, 'w') as f:
+    def store_to_path(self, path_with_file_name='./performance_result.md'):
+        with open(path_with_file_name, 'w') as f:
             for arch in test_archs:
                 for s in self.suites:
                     lines = s.mdlines(arch)
                     for line in lines:
                         print(line, file=f)
 
+    def store_with_date_and_commit_id(self, file_dir='./'):
+        current_time = datetime.datetime.now().strftime("%Y%m%dd%Hh%Mm%Ss")
+        commit_hash = _ti_core.get_commit_hash()[:8]
+        file_name = f'perfresult_{current_time}_{commit_hash}.md'
+        path = os.path.join(file_dir, file_name)
+        print('Storing benchmark result to: ' + path)
+        self.store_to_path(path)
+
+
+def main():
+    file_dir = sys.argv[1] if len(sys.argv) > 1 else './'
+    p = PerformanceMonitoring()
+    p.run()
+    p.store_to_path()  # for /benchmark
+    p.store_with_date_and_commit_id(file_dir)  #for postsubmit
+
 
-p = PerformanceMonitoring()
-p.run()
-p.write_md()
+if __name__ == '__main__':
+    main()