[CI] Refactors directory layout (#80)

.github/workflows/cpu-legacy-nightly.yaml

@@ -38,7 +38,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/
     - name: Publish

.github/workflows/cpu-legacy.yaml

@@ -37,7 +37,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/
     - name: Publish

.github/workflows/cpu-nightly.yaml

@@ -38,7 +38,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         kubectl cp ${PODNAME}:nosetests.xml ./nosetests.xml --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/

.github/workflows/cpu.yaml

@@ -37,7 +37,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/
     - name: Publish

.github/workflows/gpu-nightly.yaml

@@ -38,7 +38,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         kubectl cp ${PODNAME}:nosetests.xml ./nosetests.xml --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/

.github/workflows/gpu.yaml

@@ -37,7 +37,7 @@ jobs:
     - name: Download
       run: |-
         kubectl exec -it ${PODNAME} -- \
-        sh -c 'tar -czf hybridbackend.tgz -C env/build/ .' && \
+        sh -c 'tar -czf hybridbackend.tgz -C build/auditwheel/ .' && \
         kubectl cp ${PODNAME}:hybridbackend.tgz ./hybridbackend.tgz --retries=3 && \
         mkdir -p dist && tar -xzf ./hybridbackend.tgz -C dist/
     - name: Publish

.gitignore

@@ -4,8 +4,13 @@
 *.o
 *.d
 *.log
-build/
-dist/
+build/lib.*
+build/bdist.*
+build/temp.*
+build/reports/
+build/doc/
+build/wheel/
+build/auditwheel/
 *.egg-info/
 __pycache__/
 .pylint.d/

BUILD.md

@@ -7,7 +7,7 @@
 [Docker is the future](https://docs.docker.com/engine/install/)
 
 ```bash
-env/run make -j$(nproc)
+build/run make -j$(nproc)
 ```
 
 ### 1.2 Use customized container image for developers
@@ -32,7 +32,7 @@ Build HybridBackend on customized developer image, or build and push customized
 image from counterpart developer image:
 
 ```bash
-env/run make -j$(nproc)
+build/run make -j$(nproc)
 ```
 
 Or
@@ -77,7 +77,7 @@ Requirements:
 Build & install arrow:
 
 ```bash
-cd env/arrow/
+cd build/arrow/
 ARROW_USE_CXX11_ABI=${HYBRIDBACKEND_USE_CXX11_ABI} \
 ARROW_HDFS=ON \
 ARROW_S3=ON \
@@ -87,12 +87,12 @@ ARROW_S3=ON \
 Build & install sparsehash:
 
 ```bash
-cd env/sparsehash
+cd build/sparsehash
 ./build.sh
 ```
 
 Install TensorFlow and other requirements, see
-[Dockerfiles](env/dockerfiles/) for more detail.
+[Dockerfiles](build/dockerfiles/) for more detail.
 
 Configure & build:
 
@@ -122,7 +122,7 @@ Requirements:
 Build & install arrow:
 
 ```bash
-cd env/arrow/
+cd build/arrow/
 ARROW_USE_CXX11_ABI=${HYBRIDBACKEND_USE_CXX11_ABI} \
 ARROW_HDFS=ON \
 ARROW_S3=ON \
@@ -132,7 +132,7 @@ ARROW_S3=ON \
 Build & install sparsehash:
 
 ```bash
-cd env/sparsehash
+cd build/sparsehash
 ./build.sh
 ```
 

CONTRIBUTING.md

@@ -12,8 +12,8 @@ series [How to Contribute to an Open Source Project on GitHub](https://kcd.im/pu
 Before any commits, please use below tools to format and check code style:
 
 ```bash
-env/run tools/format
-env/run tools/lint
+build/run tools/format
+build/run tools/lint
 ```
 
 Commit message style should follow below format:
@@ -29,8 +29,8 @@ Commit message style should follow below format:
 Test your commit using default developer docker:
 
 ```bash
-env/run make -j8
-env/run make test
+build/run make -j8
+build/run make test
 ```
 
 Also, CI builds would be triggered if a commit is pushed.

Makefile

@@ -125,7 +125,7 @@ endif
 
 ifneq ($(OS),Darwin)
 D_FILES := $(shell \
-    find \( -path ./env -o -path ./build -o -path ./dist \) \
+    find \( -path ./build \) \
 	-prune -false -o -type f -name '*.d' \
 	-exec realpath {} --relative-to . \;)
 
@@ -134,7 +134,7 @@ endif
 
 THIRDPARTY_DEPS :=
 ifeq ($(HYBRIDBACKEND_WITH_ARROW),ON)
-ARROW_HOME ?= env/arrow/dist
+ARROW_HOME ?= build/arrow/dist
 ARROW_API_H := $(ARROW_HOME)/include/arrow/api.h
 THIRDPARTY_DEPS := $(THIRDPARTY_DEPS) $(ARROW_API_H)
 CFLAGS := $(CFLAGS) \
@@ -195,7 +195,7 @@ COMMON_LDFLAGS := $(COMMON_LDFLAGS) \
 endif
 
 ifeq ($(HYBRIDBACKEND_WITH_SPARSEHASH),ON)
-SPARSEHASH_HOME ?= env/sparsehash/dist
+SPARSEHASH_HOME ?= build/sparsehash/dist
 SPARSEHASH_DENSE_HASH_MAP := $(SPARSEHASH_HOME)/include/sparsehash/dense_hash_map
 THIRDPARTY_DEPS := $(THIRDPARTY_DEPS) $(SPARSEHASH_DENSE_HASH_MAP)
 CFLAGS := $(CFLAGS) \
@@ -213,8 +213,7 @@ ifeq ($(HYBRIDBACKEND_WITH_TENSORFLOW),ON)
 TENSORFLOW_LIB := $(LIBNAME)/tensorflow/lib$(LIBNAME)_tensorflow.so
 -include $(LIBNAME)/tensorflow/Makefile
 CORE_DEPS := $(CORE_DEPS) $(TENSORFLOW_LIB)
-CFLAGS := $(CFLAGS) \
-	-DHYBRIDBACKEND_TENSORFLOW="$(TENSORFLOW_DISTRO)"
+CFLAGS := $(CFLAGS) -DHYBRIDBACKEND_TENSORFLOW=1
 ifeq ($(HYBRIDBACKEND_WITH_TENSORFLOW_HALF),ON)
 CFLAGS := $(CFLAGS) -DHYBRIDBACKEND_TENSORFLOW_HALF=1
 endif
@@ -226,13 +225,13 @@ build: $(CORE_DEPS)
 	WHEEL_BUILD="$(HYBRIDBACKEND_WHEEL_BUILD)" \
 	WHEEL_REQUIRES="$(HYBRIDBACKEND_WHEEL_REQUIRES)" \
 	WHEEL_DEBUG="$(HYBRIDBACKEND_WHEEL_DEBUG)" \
-	$(PYTHON) setup.py bdist_wheel -d env/dist
-	@ls env/dist/*.whl
+	$(PYTHON) setup.py bdist_wheel -d build/wheel
+	@ls build/wheel/*.whl
 
 .PHONY: doc
 doc:
-	mkdir -p env/dist
-	sphinx-build -M html docs/ env/dist/doc
+	mkdir -p build/doc
+	sphinx-build -M html docs/ build/doc
 
 TESTS := $(shell find hybridbackend/ -type f -name "*_test.py")
 
@@ -246,8 +245,13 @@ test:
 
 .PHONY: clean
 clean:
-	rm -fr env/dist/
-	rm -fr build/
+	rm -fr build/doc/
+	rm -fr build/reports/
+	rm -fr build/wheel/
+	rm -fr build/auditwheel/
+	rm -fr build/lib.*
+	rm -fr build/bdist.*
+	rm -fr build/temp.*
 	rm -fr *.egg-info/
 	rm -rf .pylint.d/
 	find -name *.c -exec rm -fr {} \;

README.md

@@ -95,4 +95,4 @@ registered adopters in advance.
 If you would like to share your experiences with others, you are welcome to
 contact us in DingTalk:
 
-[<img src="https://github.com/alibaba/HybridBackend/raw/main/images/dingtalk.png" alt="dingtalk" width="200"/>](https://h5.dingtalk.com/circle/healthCheckin.html?dtaction=os&corpId=ding14f3e2ea4b79994cadf6428847a62d4a&51951ad=a84b419&cbdbhh=qwertyuiop)
+[<img src="https://github.com/alibaba/HybridBackend/raw/main/docs/images/dingtalk.png" alt="dingtalk" width="200"/>](https://h5.dingtalk.com/circle/healthCheckin.html?dtaction=os&corpId=ding14f3e2ea4b79994cadf6428847a62d4a&51951ad=a84b419&cbdbhh=qwertyuiop)

env/run → build/run

@@ -46,8 +46,8 @@ else
   fi
 fi
 
-if [[ ! -f "env/dockerfiles/Dockerfile.developer-${TAG}" ]]; then
-  echo >&2 "env/dockerfiles/Dockerfile.developer-${TAG} not found"
+if [[ ! -f "build/dockerfiles/Dockerfile.developer-${TAG}" ]]; then
+  echo >&2 "build/dockerfiles/Dockerfile.developer-${TAG} not found"
   exit 1
 fi
 

tools/build-container

@@ -23,8 +23,8 @@ if [[ -z "$TAG" ]]; then
 fi
 DOCKERIMAGE="${REPO}:developer-${TAG}"
 DOCKERFILE="dockerfiles/Dockerfile.developer-${TAG}"
-if [[ ! -f "env/${DOCKERFILE}" ]]; then
-  echo >&2 "env/${DOCKERFILE} not found"
+if [[ ! -f "build/${DOCKERFILE}" ]]; then
+  echo >&2 "build/${DOCKERFILE} not found"
   exit 1
 fi
 
@@ -34,22 +34,22 @@ fi
 
 set -eo pipefail
 
-rm -rf env/dist env/build
+rm -rf build/wheel build/auditwheel
 
 export COMMIT=$(git rev-parse --short HEAD 2>/dev/null)
 export COMMIT_TIME=$(git show -s --format=%ct HEAD 2>/dev/null)
 
 if [[ -z "$RELEASE" ]]; then
-  env/run tools/distbuild HYBRIDBACKEND_WHEEL_BUILD=".dev${COMMIT_TIME}"
+  build/run tools/distbuild HYBRIDBACKEND_WHEEL_BUILD=".dev${COMMIT_TIME}"
 else
-  env/run tools/distbuild
+  build/run tools/distbuild
 fi
 
 if [[ -z "$NOTESTS" ]]; then
-  env/run tools/distcheck
+  build/run tools/distcheck
 fi
 
-cd env/
+cd build/
 mkdir -p build
 
 sed "s|{{baseimage}}|${DOCKERIMAGE}|g" dockerfiles/Dockerfile.jinja2 \

tools/build-developer-container

@@ -23,14 +23,14 @@ if [[ -z "$TAG" ]]; then
 fi
 DOCKERIMAGE="${REPO}:developer-${TAG}"
 DOCKERFILE="dockerfiles/Dockerfile.developer-${TAG}"
-if [[ ! -f "env/${DOCKERFILE}" ]]; then
-  echo >&2 "env/${DOCKERFILE} not found"
+if [[ ! -f "build/${DOCKERFILE}" ]]; then
+  echo >&2 "build/${DOCKERFILE} not found"
   exit 1
 fi
 
 set -eo pipefail
 
-cd env/
+cd build/
 
 DOCKER_BUILDKIT=1 docker build \
 --build-arg BUILDKIT_INLINE_CACHE=1 \

tools/distbuild

@@ -24,4 +24,4 @@ HYBRIDBACKEND_WITH_BUILDINFO=ON \
 HYBRIDBACKEND_CHECK_INSTANCE=ON \
 $@
 
-tools/repair env/dist env/build
+tools/repair build/wheel build/auditwheel

tools/distcheck

@@ -18,7 +18,7 @@ set -eo pipefail
 
 unset PYTHONPATH
 
-pip install -U env/build/*.whl
+pip install -U build/auditwheel/*.whl
 
 make test HB_TEST_LOGDIR=build/reports
 tar -czf build/reports.tar.gz -C build/ reports

tools/format

@@ -22,8 +22,8 @@ isort --resolve-all-configs .
 
 echo "Format C/C++ files..."
 
-HEADERS=$(find \( -path ./env -o -path ./build -o -path ./dist \) -prune -false -o -type f -name '*.h')
-SOURCES=$(find \( -path ./env -o -path ./build -o -path ./dist \) -prune -false -o -type f -name '*.cc')
+HEADERS=$(find \( -path ./build \) -prune -false -o -type f -name '*.h')
+SOURCES=$(find \( -path ./build \) -prune -false -o -type f -name '*.cc')
 
 if [[ ! -z ${HEADERS} ]] || [[ ! -z ${SOURCES} ]]; then
     clang-format -i ${HEADERS} ${SOURCES}

tools/lint

@@ -24,7 +24,7 @@ echo "Lint Python files..."
 
 PYSOURCES="$@"
 if [[ -z "$PYSOURCES" ]]; then
-    PYSOURCES=$(find \( -path ./env -o -path ./build -o -path ./dist \) -prune -false -o -type f -name '*.py')
+    PYSOURCES=$(find \( -path ./build \) -prune -false -o -type f -name '*.py')
 fi
 if [[ ! -z ${PYSOURCES} ]]; then
     PYLINTHOME=$(pwd)/.pylint.d \
@@ -44,8 +44,8 @@ fi
 
 echo "Lint C/C++ files..."
 
-HEADERS=$(find \( -path ./env -o -path ./build -o -path ./dist \) -prune -false -o -type f -name '*.h')
-SOURCES=$(find \( -path ./env -o -path ./build -o -path ./dist \) -prune -false -o -type f -name '*.cc')
+HEADERS=$(find \( -path ./build \) -prune -false -o -type f -name '*.h')
+SOURCES=$(find \( -path ./build \) -prune -false -o -type f -name '*.cc')
 
 if [[ ! -z ${HEADERS} ]] || [[ ! -z ${SOURCES} ]]; then
     replacements=$(clang-format -style=file -output-replacements-xml ${HEADERS} ${SOURCES})