[build] Have PyTorch depend on minimal libcaffe2.so instead of libATen.so (pytorch#7399)

* Have PyTorch depend on minimal libcaffe2.so instead of libATen.so

* Build ATen tests as a part of Caffe2 build

* Hopefully cufft and nvcc fPIC fixes

* Make ATen install components optional

* Add tests back for ATen and fix TH build

* Fixes for test_install.sh script

* Fixes for cpp_build/build_all.sh

* Fixes for aten/tools/run_tests.sh

* Switch ATen cmake calls to USE_CUDA instead of NO_CUDA

* Attempt at fix for aten/tools/run_tests.sh

* Fix typo in last commit

* Fix valgrind call after pushd

* Be forgiving about USE_CUDA disable like PyTorch

* More fixes on the install side

* Link all libcaffe2 during test run

* Make cuDNN optional for ATen right now

* Potential fix for non-CUDA builds

* Use NCCL_ROOT_DIR environment variable

* Pass -fPIC through nvcc to base compiler/linker

* Remove THCUNN.h requirement for libtorch gen

* Add Mac test for -Wmaybe-uninitialized

* Potential Windows and Mac fixes

* Move MSVC target props to shared function

* Disable cpp_build/libtorch tests on Mac

* Disable sleef for Windows builds

* Move protos under BUILD_CAFFE2

* Remove space from linker flags passed with -Wl

* Remove ATen from Caffe2 dep libs since directly included

* Potential Windows fixes

* Preserve options while sleef builds

* Force BUILD_SHARED_LIBS flag for Caffe2 builds

* Set DYLD_LIBRARY_PATH and LD_LIBRARY_PATH for Mac testing

* Pass TORCH_CUDA_ARCH_LIST directly in cuda.cmake

* Fixes for the last two changes

* Potential fix for Mac build failure

* Switch Caffe2 to build_caffe2 dir to not conflict

* Cleanup FindMKL.cmake

* Another attempt at Mac cpp_build fix

* Clear cpp-build directory for Mac builds

* Disable test in Mac build/test to match cmake

Author: orionr

.gitignore

@@ -25,6 +25,9 @@ torch/lib/*.lib
 torch/lib/*.dylib*
 torch/lib/*.h
 torch/lib/build
+torch/lib/cmake
+torch/lib/pkgconfig
+torch/lib/protoc
 torch/lib/tmp_install
 torch/lib/include
 torch/lib/torch_shm_manager
@@ -150,7 +153,7 @@ build
 build_host_protoc
 build_android
 build_ios
-build_*
+/build_*
 .build_debug/*
 .build_release/*
 distribute/*

.jenkins/caffe2/build.sh

@@ -96,9 +96,12 @@ if [[ "${BUILD_ENVIRONMENT}" == conda* ]]; then
   exit 0
 fi
 
-# Run cmake from ./build directory
-mkdir -p ./build
-cd ./build
+# Run cmake from ./build_caffe2 directory so it doesn't conflict with
+# standard PyTorch build directory. Eventually these won't need to
+# be separate.
+rm -rf build_caffe2
+mkdir build_caffe2
+cd ./build_caffe2
 
 INSTALL_PREFIX="/usr/local/caffe2"
 CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}")

.jenkins/pytorch/build.sh

@@ -35,8 +35,8 @@ fi
 
 WERROR=1 python setup.py install
 
-# Add the ATen test binaries so that they won't be git clean'ed away
-git add -f aten/build/src/ATen/test
+# Add the test binaries so that they won't be git clean'ed away
+git add -f build/bin
 
 # Testing ATen install
 if [[ "$BUILD_ENVIRONMENT" != *cuda* ]]; then

.jenkins/pytorch/dirty.sh

@@ -3,4 +3,4 @@ set -ex
 upstream="$1"
 pr="$2"
 git diff --name-only "$upstream" "$pr"
-git diff --name-only "$upstream" "$pr" | grep -Eq '^(aten/|.jenkins/pytorch|docs/(make.bat|Makefile|requirements.txt|source)|mypy|requirements.txt|setup.py|test/|third_party/|tools/|\.gitmodules|torch/)'
+git diff --name-only "$upstream" "$pr" | grep -Eq '^(aten/|caffe2/|.jenkins/pytorch|docs/(make.bat|Makefile|requirements.txt|source)|mypy|requirements.txt|setup.py|test/|third_party/|tools/|\.gitmodules|torch/)'

.jenkins/pytorch/macos-build-test.sh

@@ -28,8 +28,18 @@ python test/run_test.py --verbose
 
 # NB: Install outside of source directory (at the same level as the root
 # pytorch folder) so that it doesn't get cleaned away prior to docker push.
+# But still clean it before we perform our own build.
+#
 CPP_BUILD="$PWD/../cpp-build"
+rm -rf $CPP_BUILD
+mkdir -p $CPP_BUILD
 WERROR=1 VERBOSE=1 tools/cpp_build/build_all.sh "$CPP_BUILD"
 
-python tools/download_mnist.py --quiet -d test/cpp/api/mnist
-"$CPP_BUILD"/libtorch/bin/test_api
+# TODO; Enable tests on Mac as soon as possible
+#python tools/download_mnist.py --quiet -d test/cpp/api/mnist
+#
+# # Unfortunately it seems like the test can't load from miniconda3
+# # without these paths being set
+# export DYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH:$PWD/miniconda3/lib"
+# export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$PWD/miniconda3/lib"
+# "$CPP_BUILD"/libtorch/bin/test_api

.jenkins/pytorch/test.sh

@@ -31,13 +31,14 @@ time python test/run_test.py --verbose
 
 # Test ATen
 if [[ "$BUILD_ENVIRONMENT" != *asan* ]]; then
-  echo "Testing ATen"
+  echo "Running ATen tests with pytorch lib"
   TORCH_LIB_PATH=$(python -c "import site; print(site.getsitepackages()[0])")/torch/lib
   # NB: the ATen test binaries don't have RPATH set, so it's necessary to
   # put the dynamic libraries somewhere were the dynamic linker can find them.
   # This is a bit of a hack.
-  ln -s "$TORCH_LIB_PATH"/libATen*.so aten/build/src/ATen
-  aten/tools/run_tests.sh aten/build
+  ln -s "$TORCH_LIB_PATH"/libcaffe2* build/bin
+  ls build/bin
+  aten/tools/run_tests.sh build/bin
 fi
 
 rm -rf ninja

.travis.aten.yml

@@ -16,7 +16,7 @@ script:
     - cd aten
     - mkdir build install
     - cd build
-    - cmake .. -DNO_CUDA=1 -DCMAKE_INSTALL_PREFIX=../install
+    - cmake .. -DUSE_CUDA=OFF -DCMAKE_INSTALL_PREFIX=../install
     - make install
     - ../tools/run_tests.sh .
     - cd ..

CMakeLists.txt

@@ -44,10 +44,11 @@ option(USE_ACL "Use ARM Compute Library" OFF)
 option(USE_ASAN "Use Address Sanitizer" OFF)
 option(USE_ATEN "Use ATen" OFF)
 option(USE_CUDA "Use CUDA" ON)
+option(USE_ROCM "Use ROCm" OFF)
 option(CAFFE2_STATIC_LINK_CUDA "Statically link CUDA libraries" OFF)
 cmake_dependent_option(
     USE_CUDNN "Use cuDNN" ON
-    "USE_CUDA" OFF)  # New option
+    "USE_CUDA" OFF)
 option(USE_FFMPEG "Use ffmpeg" OFF)
 cmake_dependent_option(
     USE_GFLAGS "Use GFLAGS" ON
@@ -58,7 +59,7 @@ cmake_dependent_option(
 cmake_dependent_option(
     USE_GLOO "Use Gloo" ON
     "BUILD_CAFFE2" OFF)
-option(USE_GLOO_IBVERBS "Use Gloo IB verbs for distributed support" OFF)  # New option
+option(USE_GLOO_IBVERBS "Use Gloo IB verbs for distributed support" OFF)
 cmake_dependent_option(
     USE_LEVELDB "Use LEVELDB" ON
     "BUILD_CAFFE2" OFF)
@@ -77,7 +78,7 @@ cmake_dependent_option(
     "BUILD_CAFFE2" OFF)
 option(USE_NATIVE_ARCH "Use -march=native" OFF)
 option(USE_NCCL "Use NCCL" ON)
-option(USE_SYSTEM_NCCL "Use system-wide NCCL" OFF)  # New option
+option(USE_SYSTEM_NCCL "Use system-wide NCCL" OFF)
 option(USE_NERVANA_GPU "Use Nervana GPU backend" OFF)
 option(USE_NNAPI "Use NNAPI" OFF)
 option(USE_NNPACK "Use NNPACK" ON)
@@ -104,13 +105,16 @@ cmake_dependent_option(
 cmake_dependent_option(
     USE_MKLML "Use MKLML interface in MKL BLAS" ON
     "BUILD_CAFFE2" OFF)
-option(USE_DISTRIBUTED "Use THD (distributed)" OFF)  # New option
-option(USE_DISTRIBUTED_MW "Use THD (distributed) master worker" OFF)  # New option
+option(USE_DISTRIBUTED "Use THD (distributed)" OFF)
+option(USE_DISTRIBUTED_MW "Use THD (distributed) master worker" OFF)
 
 # Legacy options, which we will eventually remove
 cmake_dependent_option(
     WITH_CUDA "Legacy CUDA" ON
     "USE_CUDA" OFF)
+cmake_dependent_option(
+    WITH_ROCM "Legacy ROCm" ON
+    "USE_ROCM" OFF)
 cmake_dependent_option(
     NO_CUDA "Legacy no CUDA" OFF
     "USE_CUDA" ON)
@@ -158,9 +162,18 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
 
 enable_testing()
 
+# ---[ Build variables set within the cmake tree
+include(cmake/BuildVariables.cmake)
+set(CAFFE2_WHITELIST "" CACHE STRING "A whitelist file of files that one should build.")
+
+# Set default build type
+if(NOT CMAKE_BUILD_TYPE)
+    message(STATUS "Build type not set - defaulting to Release")
+    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build from: Debug Release RelWithDebInfo MinSizeRel Coverage." FORCE)
+endif()
+
 # ---[ Misc checks to cope with various compiler modes
 include(cmake/MiscCheck.cmake)
-include(cmake/BuildVariables.cmake)
 
 # External projects
 include(ExternalProject)
@@ -170,14 +183,6 @@ include(ExternalProject)
 include(cmake/Utils.cmake)
 include(cmake/public/utils.cmake)
 
-set(CAFFE2_WHITELIST "" CACHE STRING "A whitelist file of files that one should build.")
-
-# Set default build type
-if(NOT CMAKE_BUILD_TYPE)
-    message(STATUS "Build type not set - defaulting to Release")
-    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build from: Debug Release RelWithDebInfo MinSizeRel Coverage." FORCE)
-endif()
-
 # ---[ Dependencies
 include(cmake/Dependencies.cmake)
 
@@ -249,7 +254,9 @@ include_directories(BEFORE ${PROJECT_SOURCE_DIR})
 include_directories(BEFORE ${PROJECT_BINARY_DIR})
 
 # ---[ Old caffe protobuf
-add_subdirectory(caffe/proto)
+if(BUILD_CAFFE2)
+  add_subdirectory(caffe/proto)
+endif()
 
 # ---[ Main build
 add_subdirectory(caffe2)
@@ -313,7 +320,7 @@ if ((NOT USE_GLOG) OR (NOT USE_GFLAGS) OR BUILD_CUSTOM_PROTOBUF)
       "generate files that are not well tested.")
 endif()
 
-if (USE_CUDA)
+if (USE_CUDA OR USE_ROCM)
   # TODO: check if we should include other cuda dependency libraries
   # to the interface as well.