Refactor Docker workflows and remove obsolete tester image script (#2385)

small-turtle-1 · web-flow · commit 98839106844a · 2024-12-18T09:25:18.000+08:00
### What problem does this PR solve?

Remove tester docker container. Use builder container.

### Type of change

- [x] Refactoring
diff --git a/.github/workflows/slow_test.yml b/.github/workflows/slow_test.yml
@@ -60,22 +60,10 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
-
-      - name: Create Test image & Start tester container
-        if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
-        run: |
-          TESTER_IMAGE_NAME=infiniflow/infinity_tester:centos7_clang18
-          sudo python3 scripts/build_tester_image.py --image_name=${TESTER_IMAGE_NAME} --bin_path=${RUNNER_WORKSPACE_PREFIX}/build_tester_image
-          TESTER_CONTAINER=infinity_tester_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-          CPUS=${CPUS:-$(nproc)}
-          echo "TESTER_CONTAINER=${TESTER_CONTAINER}" >> $GITHUB_ENV
-          echo "CPUS=${CPUS}" >> $GITHUB_ENV
-          TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
           INF_DIRECTORY=$PWD
           echo "${INF_DIRECTORY}" | sudo tee .tester_env
-          sudo docker rm -f -v ${TESTER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${TESTER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} ${TESTER_IMAGE_NAME}
-          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY}"
+          sudo docker pull infiniflow/infinity_builder:centos7_clang18
+          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
 
       - name: Build release version
         if: ${{ !cancelled() && !failure() }}
@@ -122,11 +110,10 @@ jobs:
         if: ${{ !cancelled() && !failure() && steps.choose_test_type.outputs.RUN_TSAN_TEST == 'true' }}
         id: run_cluster_test
         run: |
-          sudo docker exec ${TESTER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
           # Fix sanitizer: https://github.com/ClickHouse/ClickHouse/issues/64086
           old_value=$(sudo sysctl -n vm.mmap_rnd_bits)
           sudo sysctl -w vm.mmap_rnd_bits=28
-          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
           sudo sysctl -w vm.mmap_rnd_bits=$old_value
 
       - name: Collect thread sanitizer output in cluster test
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -54,26 +54,14 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
-    
-      - name: Create Test image & Start tester container
-        if: ${{ !cancelled() && !failure() }}
-        run: |
-            TESTER_IMAGE_NAME=infiniflow/infinity_tester:centos7_clang18
-            sudo python3 scripts/build_tester_image.py --image_name=${TESTER_IMAGE_NAME} --bin_path=${RUNNER_WORKSPACE_PREFIX}/build_tester_image
-            TESTER_CONTAINER=infinity_tester_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ')
-            CPUS=${CPUS:-$(nproc)}
-            echo "TESTER_CONTAINER=${TESTER_CONTAINER}" >> $GITHUB_ENV
-            echo "CPUS=${CPUS}" >> $GITHUB_ENV
-            TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
-            INF_DIRECTORY=$PWD
-            echo "${INF_DIRECTORY}" | sudo tee .tester_env
-            sudo docker rm -f -v ${TESTER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${TESTER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} ${TESTER_IMAGE_NAME}
-            sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY}"
+          INF_DIRECTORY=$PWD
+          echo "${INF_DIRECTORY}" | sudo tee .tester_env
+          sudo docker pull infiniflow/infinity_builder:centos7_clang18
+          sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --net=host --privileged --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot -v /var/run/docker.sock:/var/run/docker.sock --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
 
       - name: Build debug version
         if: ${{ !cancelled() && !failure() }}
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug --target infinity test_main"
+        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug --target infinity"
 
       - name: Install pysdk
         if: ${{ !cancelled() && !failure() }}
@@ -83,21 +71,13 @@ jobs:
         if: ${{ !cancelled() && !failure() }}
         id: run_cluster_test
         run: |
-          sudo docker exec ${TESTER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
-          sudo docker exec ${TESTER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+          sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
 
       - name: Collect cluster test output
         if: ${{ !cancelled() }}
         run: |
           failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
           sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-debug/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
-
-      - name: Remove tester container
-        if: always()  # always run this step even if previous steps failed
-        run: |
-          if [ -n "${TESTER_CONTAINER}" ]; then
-            sudo docker rm -f -v ${TESTER_CONTAINER}
-          fi
     
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}
@@ -189,6 +169,10 @@ jobs:
           failure="${{ steps.run_tests_debug.outcome == 'failure' || steps.stop_tests_debug.outcome == 'failure' }}"
           sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=debug.log --stderror_path=debug_error.log --executable_path=cmake-build-debug/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
+      - name: Build unit test
+        if: ${{ !cancelled() && !failure() }}
+        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && cmake --build /infinity/cmake-build-debug --target test_main"
+
       - name: Unit test debug version
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "mkdir -p /var/infinity && cd /infinity/ && ASAN_OPTIONS=detect_leaks=0 cmake-build-debug/src/test_main > unittest_debug.log 2>&1"
@@ -249,15 +233,28 @@ jobs:
           echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> $GITHUB_ENV
           echo "CPUS=${CPUS}" >> $GITHUB_ENV
           TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')}
+          sudo docker pull infiniflow/infinity_builder:centos7_clang18
           sudo docker rm -f -v ${BUILDER_CONTAINER} && sudo docker run --cap-add=NET_ADMIN -d --name ${BUILDER_CONTAINER} -e TZ=$TZ -e CMAKE_BUILD_PARALLEL_LEVEL=${CPUS} -v $PWD:/infinity -v /boot:/boot --cpus ${CPUS} infiniflow/infinity_builder:centos7_clang18
 
       - name: Build release version
         if: ${{ !cancelled() && !failure() }}
-        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity test_main knn_import_benchmark knn_query_benchmark"
+        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -fr cmake-build-release && mkdir -p cmake-build-release && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_JOB_POOLS:STRING=link=8 -S /infinity -B /infinity/cmake-build-release && cmake --build /infinity/cmake-build-release --target infinity"
 
       - name: Install pysdk for Python 3.10
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "rm -rf /root/.config/pip/pip.conf && cd /infinity/ && pip3 uninstall -y infinity-sdk infinity-embedded-sdk && pip3 install . -v --config-settings=cmake.build-type='RelWithDebInfo' --config-settings=build-dir='cmake-build-release' -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd python/infinity_sdk/ && pip3 install . -i https://pypi.tuna.tsinghua.edu.cn/simple --trusted-host tuna.tsinghua.edu.cn && cd ../.."
+
+    #   - name: Run cluster test
+    #     if: ${{ !cancelled() && !failure() }}
+    #     id: run_cluster_test
+    #     run: |
+    #       sudo docker exec ${BUILDER_CONTAINER} bash -c "cd /infinity/ && rm -fr /var/infinity && export INF_DIRECTORY=`cat .tester_env` && echo INF_DIRECTORY=${INF_DIRECTORY} && python3 tools/run_cluster_test.py --infinity_path=cmake-build-release/src/infinity --infinity_dir=${INF_DIRECTORY} --minio_port=9005 --minio_console_port=9006"
+
+    #   - name: Collect cluster test output
+    #     if: ${{ !cancelled() }}
+    #     run: |
+    #       failure="${{ steps.run_cluster_test.outcome == 'failure'}}"
+    #       sudo python3 scripts/collect_cluster_log.py --executable_path=cmake-build-release/src/infinity --log_dir=/var/infinity/ --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
     
       - name: Prepare restart test data
         if: ${{ !cancelled() && !failure() }}
@@ -360,6 +357,10 @@ jobs:
           failure="${{ steps.run_tests_release.outcome == 'failure' || steps.stop_tests_release.outcome == 'failure' }}"
           sudo python3 scripts/collect_log.py --log_path=/var/infinity/log/infinity.log --stdout_path=release.log --stderror_path=release_error.log --executable_path=cmake-build-release/src/infinity --output_dir=${RUNNER_WORKSPACE_PREFIX}/log --failure=${failure}
 
+      - name: Build unit test
+        if: ${{ !cancelled() && !failure() }}
+        run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && cmake --build /infinity/cmake-build-release --target test_main knn_import_benchmark knn_query_benchmark"
+
       - name: Unit test release version
         if: ${{ !cancelled() && !failure() }}
         run: sudo docker exec ${BUILDER_CONTAINER} bash -c "mkdir -p /var/infinity && cd /infinity/ && cmake-build-release/src/test_main > unittest_release.log 2>&1"
diff --git a/.tester_env b/.tester_env
@@ -1 +1 @@
-/home/huikong/Code/work/infinity2
+/home/huikong/Code/work/infinity
diff --git a/python/test_cluster/infinity_cluster.py b/python/test_cluster/infinity_cluster.py
@@ -251,6 +251,7 @@ def add_minio(self, minio_params: MinioParams):
 
         try:
             self.minio_container = docker_client.containers.get(container_name)
+            print(self.minio_container)
             self.minio_container.start()
             self.logger.debug(f"Minio container {container_name} already exists.")
         except docker.errors.NotFound:
diff --git a/scripts/Dockerfile_infinity_builder_centos7 b/scripts/Dockerfile_infinity_builder_centos7
@@ -240,4 +240,29 @@ RUN git clone --single-branch --depth=1 https://github.com/VectorCamp/vectorscan
     && make -j 12 install \
     && rm -rf ../../vectorscan
 
+# Install iproute2
+RUN --mount=type=bind,source=iproute2-6.9.0.tar.gz,target=/root/iproute2-6.9.0.tar.gz \
+cd /root \
+&& tar -zxf iproute2-6.9.0.tar.gz && cd iproute2-6.9.0 \
+&& ./configure --prefix=/usr/local/iproute2 \
+&& make && make install \
+&& ip -V
+
+# test requirements
+RUN --mount=type=bind,source=python/test_cluster/requirements.txt,target=/root/requirements1.txt \
+    cd /root \
+    && /usr/local/bin/python3.10 -m venv /usr/local/venv310 && source /usr/local/venv310/bin/activate \
+    && pip3 install -r requirements1.txt
+
+RUN yum install -y sudo bridge-utils iptables
+
+# Install Docker
+# https://docs.docker.com/engine/install/centos/#install-using-the-convenience-script
+# RUN --mount=type=bind,source=get-docker.sh,target=/root/get-docker.sh \
+#     cd /root && sh get-docker.sh
+RUN --mount=type=bind,source=docker-27.3.1.tgz,target=/root/docker-27.3.1.tgz \
+    cd /root \
+    && tar zxf docker-27.3.1.tgz \
+    && sudo cp docker/* /usr/bin
+
 ENTRYPOINT [ "bash", "-c", "while true; do sleep 60; done"]
diff --git a/scripts/Dockerfile_infinity_tester_centos7 b/scripts/Dockerfile_infinity_tester_centos7
diff --git a/scripts/build_tester_image.py b/scripts/build_tester_image.py
diff --git a/scripts/download_deps_infinity_builder_centos7.sh b/scripts/download_deps_infinity_builder_centos7.sh
@@ -36,6 +36,7 @@ https://www.python.org/ftp/python/3.12.7/Python-3.12.7.tar.xz
 https://www.python.org/ftp/python/3.13.0/Python-3.13.0.tar.xz
 https://github.com/risinglightdb/sqllogictest-rs/releases/download/v0.20.2/sqllogictest-bin-v0.20.2-x86_64-unknown-linux-musl.tar.gz
 https://mirrors.edge.kernel.org/pub/linux/utils/net/iproute2/iproute2-6.9.0.tar.gz"
+https://download.docker.com/linux/static/stable/x86_64/docker-27.3.1.tgz
 
 SAVEIFS=$IFS   # Save current IFS (Internal Field Separator)
 IFS=$'\n'      # Change IFS to newline char
diff --git a/src/storage/meta/entry/table_index_entry.cpp b/src/storage/meta/entry/table_index_entry.cpp
@@ -482,22 +482,20 @@ void TableIndexEntry::PickCleanup(CleanupScanner *scanner) {
     Vector<SegmentID> segment_ids;
     {
         std::shared_lock r_lock(rw_locker_);
-        for (auto iter = index_by_segment_.begin(); iter != index_by_segment_.end();) {
-            auto &[segment_id, segment_index_entry] = *iter;
+        for (auto &[segment_id, segment_index_entry] : index_by_segment_) {
             if (segment_index_entry->CheckDeprecate(visible_ts)) {
-                scanner->AddEntry(std::move(segment_index_entry));
                 segment_ids.push_back(segment_id);
-                ++iter;
             } else {
                 segment_index_entry->PickCleanup(scanner);
-                ++iter;
             }
         }
     }
     if (!segment_ids.empty()) {
         std::unique_lock w_lock(rw_locker_);
         for (auto segment_id : segment_ids) {
-            index_by_segment_.erase(segment_id);
+            auto iter = index_by_segment_.find(segment_id);
+            scanner->AddEntry(std::move(iter->second));
+            index_by_segment_.erase(iter);
         }
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-/home/huikong/Code/work/infinity2`
	`1`	`+/home/huikong/Code/work/infinity`
Original file line number	Diff line number	Diff line change
`@@ -482,22 +482,20 @@ void TableIndexEntry::PickCleanup(CleanupScanner *scanner) {`
`482`	`482`	`Vector<SegmentID> segment_ids;`
`483`	`483`	`{`
`484`	`484`	`std::shared_lock r_lock(rw_locker_);`
`485`		`- for (auto iter = index_by_segment_.begin(); iter != index_by_segment_.end();) {`
`486`		`- auto &[segment_id, segment_index_entry] = *iter;`
	`485`	`+ for (auto &[segment_id, segment_index_entry] : index_by_segment_) {`
`487`	`486`	`if (segment_index_entry->CheckDeprecate(visible_ts)) {`
`488`		`- scanner->AddEntry(std::move(segment_index_entry));`
`489`	`487`	`segment_ids.push_back(segment_id);`
`490`		`- ++iter;`
`491`	`488`	`} else {`
`492`	`489`	`segment_index_entry->PickCleanup(scanner);`
`493`		`- ++iter;`
`494`	`490`	`}`
`495`	`491`	`}`
`496`	`492`	`}`
`497`	`493`	`if (!segment_ids.empty()) {`
`498`	`494`	`std::unique_lock w_lock(rw_locker_);`
`499`	`495`	`for (auto segment_id : segment_ids) {`
`500`		`- index_by_segment_.erase(segment_id);`
	`496`	`+ auto iter = index_by_segment_.find(segment_id);`
	`497`	`+ scanner->AddEntry(std::move(iter->second));`
	`498`	`+ index_by_segment_.erase(iter);`
`501`	`499`	`}`
`502`	`500`	`}`
`503`	`501`	`}`