OpenPCDet LiDar Object Detection Node

.github/templates/docker_context/docker_context.sh

@@ -39,6 +39,7 @@ while read -r module; do
         # Temporarily skip perception services that have too large image size
         if  [[ "$service_out" == "lane_detection" ]] || \
             [[ "$service_out" == "camera_object_detection" ]] || \
+            [[ "$service_out" == "lidar_object_detection" ]] || \
             [[ "$service_out" == "semantic_segmentation" ]]; then
             continue
         fi

docker/perception/lidar_object_detection/lidar_object_detection.Dockerfile

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:humble-ubuntu22.04
+ARG BASE_IMAGE=ghcr.io/watonomous/wato_monorepo/base:cuda11.7-humble-ubuntu22.04-devel
 
 ################################ Source ################################
 FROM ${BASE_IMAGE} as source
@@ -7,20 +7,70 @@ WORKDIR ${AMENT_WS}/src
 
 # Copy in source code 
 COPY src/perception/lidar_object_detection lidar_object_detection
-COPY src/wato_msgs/sample_msgs sample_msgs
 
 # Scan for rosdeps
 RUN apt-get -qq update && rosdep update && \
     rosdep install --from-paths . --ignore-src -r -s \
-        | grep 'apt-get install' \
-        | awk '{print $3}' \
-        | sort  > /tmp/colcon_install_list
+    | grep 'apt-get install' \
+    | awk '{print $3}' \
+    | sort  > /tmp/colcon_install_list
+
 
 ################################# Dependencies ################################
 FROM ${BASE_IMAGE} as dependencies
 
+RUN apt-get update && apt-get install -y \
+    git zip unzip libssl-dev libcairo2-dev lsb-release libgoogle-glog-dev libgflags-dev libatlas-base-dev libeigen3-dev software-properties-common \
+    build-essential cmake pkg-config libapr1-dev autoconf automake libtool curl libc6 libboost-all-dev debconf libomp5 libstdc++6 \
+    libqt5core5a libqt5xml5 libqt5gui5 libqt5widgets5 libqt5concurrent5 libqt5opengl5 libcap2 libusb-1.0-0 libatk-adaptor neovim \
+    python3-pip python3-setuptools \
+    && apt-get -qq autoremove -y && apt-get -qq autoclean && apt-get -qq clean && \
+    rm -rf /var/lib/apt/lists/* /root/* /root/.ros /tmp/* /usr/share/doc
+
+################################ INSTALL OpenCV with CUDA Support ##############
+WORKDIR /opt
+RUN git clone https://github.com/opencv/opencv.git && \
+    cd opencv && git checkout 4.5.5
+RUN git clone https://github.com/opencv/opencv_contrib.git && \
+    cd opencv_contrib && git checkout 4.5.5
+WORKDIR /opt/opencv/build
+RUN cmake -D CMAKE_BUILD_TYPE=RELEASE \
+    -D CMAKE_INSTALL_PREFIX=/usr/local \
+    -D OPENCV_GENERATE_PKGCONFIG=ON \
+    -D BUILD_EXAMPLES=OFF \
+    -D INSTALL_PYTHON_EXAMPLES=OFF \
+    -D INSTALL_C_EXAMPLES=OFF \
+    -D PYTHON_EXECUTABLE=$(which python3) \
+    -D PYTHON3_INCLUDE_DIR=$(python3 -c "from distutils.sysconfig import get_python_inc; print(get_python_inc())") \
+    -D PYTHON3_PACKAGES_PATH=$(python3 -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") \
+    -D BUILD_opencv_python3=ON \
+    -D OPENCV_EXTRA_MODULES_PATH=../../opencv_contrib/modules/ \
+    -D WITH_GSTREAMER=ON \
+    -D WITH_CUDA=ON \
+    -D ENABLE_PRECOMPILED_HEADERS=OFF \
+    .. && make -j$(nproc) && make install && ldconfig
+RUN rm -rf /opt/opencv /opt/opencv_contrib
+# Set environment variables
+ENV CUDA_HOME /usr/local/cuda
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:${LD_LIBRARY_PATH}
+ENV PATH /usr/local/cuda/bin:${PATH}
+ENV OpenCV_DIR=/usr/share/OpenCV
+# Install Python dependencies
+RUN pip3 install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
+RUN pip3 install torch-scatter -f https://data.pyg.org/whl/torch-1.13.1+cu116.html
+ENV TORCH_CUDA_ARCH_LIST="3.5;5.0;6.0;6.1;7.0;7.5;8.0;8.6+PTX"
+RUN pip3 install spconv-cu116 pyquaternion numpy==1.23 pillow==8.4 mayavi open3d av2
+WORKDIR /home/bolty
+RUN git clone https://github.com/WATonomous/OpenPCDet.git
+RUN cd OpenPCDet && pip3 install -r requirements.txt
+RUN pip3 install kornia==0.6.8
+RUN pip3 install nuscenes-devkit==1.0.5
+WORKDIR /home/bolty/OpenPCDet/
+RUN python3 setup.py develop
+
 # Install Rosdep requirements
 COPY --from=source /tmp/colcon_install_list /tmp/colcon_install_list
+RUN apt-get update && apt-fast update
 RUN apt-fast install -qq -y --no-install-recommends $(cat /tmp/colcon_install_list)
 
 # Copy in source code from source stage
@@ -32,20 +82,20 @@ WORKDIR /
 RUN apt-get -qq autoremove -y && apt-get -qq autoclean && apt-get -qq clean && \
     rm -rf /root/* /root/.ros /tmp/* /var/lib/apt/lists/* /usr/share/doc/*
 
-################################ Build ################################
+################################ Source #######################################
 FROM dependencies as build
 
 # Build ROS2 packages
 WORKDIR ${AMENT_WS}
 RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
     colcon build \
-        --cmake-args -DCMAKE_BUILD_TYPE=Release
+    --cmake-args -DCMAKE_BUILD_TYPE=Release
 
 # Entrypoint will run before any CMD on launch. Sources ~/opt/<ROS_DISTRO>/setup.bash and ~/ament_ws/install/setup.bash
 COPY docker/wato_ros_entrypoint.sh ${AMENT_WS}/wato_ros_entrypoint.sh
 ENTRYPOINT ["./wato_ros_entrypoint.sh"]
 
-################################ Prod ################################
+################################ Prod #########################################
 FROM build as deploy
 
 # Source Cleanup and Security Setup

modules/docker-compose.perception.yaml

@@ -44,8 +44,23 @@ services:
         - "${PERCEPTION_LIDAR_OBJECT_DETECTION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_LIDAR_OBJECT_DETECTION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch lidar_object_detection lidar_object_detection.launch.py"
-
+    command: /bin/bash -c "ros2 launch lidar_object_detection nuscenes_launch.py"
+    volumes:
+      - /mnt/wato-drive2/perception_models/transfusion_trained_model.pth:/home/bolty/OpenPCDet/models/transfusion_trained_model.pth
+      - /mnt/wato-drive2/perception_models/cbgs_transfusion_lidar.pth:/home/bolty/OpenPCDet/models/cbgs_transfusion_lidar.pth
+      - /mnt/wato-drive2/perception_models/pv_rcnn_8369.pth:/home/bolty/OpenPCDet/models/pv_rcnn_8369.pth
+      - /mnt/wato-drive2/perception_models/nuscenes_trained_voxelNext.pth:/home/bolty/OpenPCDet/models/nuscenes_trained_voxelNext.pth
+      # New Models to Try
+      - /mnt/wato-drive/perception/bevfusion_pretrain/bevfusion-det.pth:/home/bolty/OpenPCDet/models/bevfusion-det.pth
+      - /mnt/wato-drive/perception/bevfusion_pretrain/lidar-only-det.pth:/home/bolty/OpenPCDet/models/lidar-only-det.pth
+      - /mnt/wato-drive/perception/bevfusion_pretrain/lidar-only-seg.pth:/home/bolty/OpenPCDet/models/lidar-only-seg.pth
+      - /mnt/wato-drive/perception/bevfusion_pretrain/lidar-only-seg.pth:/home/bolty/OpenPCDet/models/swint-nuimages-pretrained.pth
+      - /mnt/wato-drive/perception/pcds/pvrcnn_ros_ckpt.pth:/home/bolty/OpenPCDet/models/pvrcnn_ros_ckpt.pth
+      - /mnt/wato-drive/perception/pcds/pvrcnn_base_ckpt.pth:/home/bolty/OpenPCDet/models/pvrcnn_base_ckpt.pth
+      # Data sets
+      - /mnt/wato-drive2/nuscenes-1.0-mini:/home/bolty/OpenPCDet/data/nuscenes/v1.0-mini
+      - /mnt/wato-drive2/wato-datasets/nuscenes/full-dataset:/home/bolty/OpenPCDet/data/nuscenes/v1.0-trainval
+
   semantic_segmentation:
     build:
       context: ..
@@ -88,4 +103,4 @@ services:
         - "${PERCEPTION_DEPTH_ESTIMATION_IMAGE}:build_main"
       target: deploy
     image: "${PERCEPTION_DEPTH_ESTIMATION_IMAGE}:${TAG}"
-    command: /bin/bash -c "ros2 launch depth_estimation eve.launch.py"
+    command: /bin/bash -c "ros2 launch depth_estimation eve.launch.py"

src/perception/lidar_object_detection/lidar_object_detection/config/eve_config.yaml

@@ -0,0 +1,5 @@
+lidar_object_detection_node:
+  ros__parameters:
+    lidar_topic: /velodyne_points
+    model_config_path: /home/bolty/OpenPCDet/tools/cfgs/nuscenes_models/transfusion_lidar.yaml
+    model_path: /home/bolty/OpenPCDet/models/transfusion_trained_model.pth

src/perception/lidar_object_detection/lidar_object_detection/config/nuscenes_config.yaml

@@ -0,0 +1,5 @@
+lidar_object_detection_node:
+  ros__parameters:
+    lidar_topic: /LIDAR_TOP
+    model_config_path: /home/bolty/OpenPCDet/tools/cfgs/nuscenes_models/transfusion_lidar.yaml
+    model_path: /home/bolty/OpenPCDet/models/cbgs_transfusion_lidar.pth

src/perception/lidar_object_detection/lidar_object_detection/launch/eve_launch.py

@@ -0,0 +1,27 @@
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from ament_index_python.packages import get_package_share_directory
+import os
+
+
+def generate_launch_description():
+    ld = LaunchDescription()
+    config = os.path.join(
+        get_package_share_directory('lidar_object_detection'),
+        'config',
+        'eve_config.yaml'
+    )
+
+    # nodes
+    lidar_object_detection = Node(
+        package='lidar_object_detection',
+        executable='lidar_object_detection_node',
+        name='lidar_object_detection_node',
+        parameters=[config],
+        arguments=['--ros-args', '--log-level', 'info']
+    )
+
+    # finalize
+    ld.add_action(lidar_object_detection)
+
+    return ld

src/perception/lidar_object_detection/lidar_object_detection/launch/nuscenes_launch.py

@@ -0,0 +1,27 @@
+from launch import LaunchDescription
+from launch_ros.actions import Node
+from ament_index_python.packages import get_package_share_directory
+import os
+
+
+def generate_launch_description():
+    ld = LaunchDescription()
+    config = os.path.join(
+        get_package_share_directory('lidar_object_detection'),
+        'config',
+        'nuscenes_config.yaml'
+    )
+
+    # nodes
+    lidar_object_detection = Node(
+        package='lidar_object_detection',
+        executable='lidar_object_detection_node',
+        name='lidar_object_detection_node',
+        parameters=[config],
+        arguments=['--ros-args', '--log-level', 'info']
+    )
+
+    # finalize
+    ld.add_action(lidar_object_detection)
+
+    return ld