support yolox-pose sdk

.github/workflows/lint.yml

@@ -25,6 +25,8 @@ jobs:
           style: file
       - name: Check index.rst
         run: |
+          which clang-format
+          clang-format --version
           python .github/scripts/check_index_rst.py docs/en/index.rst
           python .github/scripts/check_index_rst.py docs/zh_cn/index.rst
       - name: Check markdown link

configs/mmpose/pose-detection_yolox-pose_sdk_dynamic.py

@@ -0,0 +1,9 @@
+_base_ = ['./pose-detection_static.py', '../_base_/backends/sdk.py']
+
+codebase_config = dict(model_type='sdk_yoloxpose')
+
+backend_config = dict(pipeline=[
+    dict(type='LoadImageFromFile'),
+    dict(type='PoseToDetConverter'),
+    dict(type='PackDetPoseInputs')
+])

csrc/mmdeploy/apis/c/mmdeploy/pose_detector.cpp

@@ -69,6 +69,8 @@ void mmdeploy_pose_detector_release_result(mmdeploy_pose_detection_t* results, i
   for (int i = 0; i < count; ++i) {
     delete[] results[i].point;
     delete[] results[i].score;
+    delete[] results[i].bboxes;
+    delete[] results[i].bbox_score;
   }
   delete[] results;
 }
@@ -156,16 +158,27 @@ int mmdeploy_pose_detector_get_result(mmdeploy_value_t output,
     for (const auto& bbox_result : detections) {
       auto& res = _results[result_idx++];
       auto size = bbox_result.key_points.size();
+      auto num_bbox = bbox_result.detections.size();
 
       res.point = new mmdeploy_point_t[size];
       res.score = new float[size];
       res.length = static_cast<int>(size);
+      res.bboxes = new mmdeploy_rect_t[num_bbox];
+      res.bbox_score = new float[num_bbox];
+      res.num_bbox = static_cast<int>(num_bbox);
 
       for (int k = 0; k < size; k++) {
         res.point[k].x = bbox_result.key_points[k].bbox[0];
         res.point[k].y = bbox_result.key_points[k].bbox[1];
         res.score[k] = bbox_result.key_points[k].score;
       }
+      for (int k = 0; k < num_bbox; k++) {
+        res.bboxes[k].left = bbox_result.detections[k].boundingbox[0];
+        res.bboxes[k].top = bbox_result.detections[k].boundingbox[1];
+        res.bboxes[k].right = bbox_result.detections[k].boundingbox[2];
+        res.bboxes[k].bottom = bbox_result.detections[k].boundingbox[3];
+        res.bbox_score[k] = bbox_result.detections[k].score;
+      }
     }
 
     *results = _results.release();

csrc/mmdeploy/apis/c/mmdeploy/pose_detector.h

@@ -19,7 +19,10 @@ extern "C" {
 typedef struct mmdeploy_pose_detection_t {
   mmdeploy_point_t* point;  ///< keypoint
   float* score;             ///< keypoint score
+  mmdeploy_rect_t* bboxes;  ///< bboxes
+  float* bbox_score;        ///< bboxes score
   int length;               ///< number of keypoint
+  int num_bbox;             ///< number of bboxes
 } mmdeploy_pose_detection_t;
 
 typedef struct mmdeploy_pose_detector* mmdeploy_pose_detector_t;
@@ -76,6 +79,7 @@ MMDEPLOY_API int mmdeploy_pose_detector_apply(mmdeploy_pose_detector_t detector,
  * bboxes, must be release by \ref mmdeploy_pose_detector_release_result
  * @return status code of the operation
  */
+
 MMDEPLOY_API int mmdeploy_pose_detector_apply_bbox(mmdeploy_pose_detector_t detector,
                                                    const mmdeploy_mat_t* mats, int mat_count,
                                                    const mmdeploy_rect_t* bboxes,

csrc/mmdeploy/apis/csharp/MMDeploy/APIs/PoseDetector.cs

@@ -8,7 +8,10 @@ internal unsafe struct CPoseDetect
     {
         public Pointf* Point;
         public float* Score;
+        public Rect* BBox;
+        public float* BBoxScore;
         public int Length;
+        public int BBoxNum;
     }
 #pragma warning restore 0649
 
@@ -28,6 +31,16 @@ public struct PoseDetect
         /// </summary>
         public List<float> Scores;
 
+        /// <summary>
+        /// BBox.
+        /// </summary>
+        public Rect BBox;
+
+        /// <summary>
+        /// BBoxScore.
+        /// </summary>
+        public float BBoxScore;
+
         /// <summary>
         /// Init points and scores if empty.
         /// </summary>
@@ -40,6 +53,17 @@ private void Init()
             }
         }
 
+        /// <summary>
+        /// Set bounding box and score.
+        /// </summary>
+        /// <param name="bbox">BBox.</param>
+        /// <param name="score">BBox score.</param>
+        public void SetBBox(Rect bbox, float score)
+        {
+            BBox = bbox;
+            BBoxScore = score;
+        }
+
         /// <summary>
         /// Add single keypoint to list.
         /// </summary>
@@ -170,13 +194,26 @@ private unsafe void FormatResult(int matCount, int* bboxCount, CPoseDetect* resu
                 PoseDetectorOutput outi = default;
                 for (int j = 0; j < bboxCount[i]; j++)
                 {
-                    PoseDetect boxRes = default;
-                    for (int k = 0; k < results->Length; k++)
+                    int bbox_num = results->BBoxNum;
+                    int num_point_each_bbox = results->Length / results->BBoxNum;
+                    for (int box_id = 0; box_id < bbox_num; box_id++)
                     {
-                        boxRes.Add(results->Point[k], results->Score[k]);
+                        PoseDetect boxRes = default;
+                        Rect bbox = default;
+                        float score = results->BBoxScore[box_id];
+                        bbox.Left = results->BBox[box_id].Left;
+                        bbox.Top = results->BBox[box_id].Top;
+                        bbox.Right = results->BBox[box_id].Right;
+                        bbox.Bottom = results->BBox[box_id].Bottom;
+                        boxRes.SetBBox(bbox, score);
+                        for (int kp_id = 0; kp_id < num_point_each_bbox; kp_id++)
+                        {
+                            boxRes.Add(results->Point[(box_id * num_point_each_bbox) + kp_id], results->Score[(box_id * num_point_each_bbox) + kp_id]);
+                        }
+
+                        outi.Add(boxRes);
                     }
 
-                    outi.Add(boxRes);
                     results++;
                     total++;
                 }

csrc/mmdeploy/apis/java/mmdeploy/PoseDetector.java

@@ -17,13 +17,21 @@ public static class Result {
         /** Scores of points */
         public float[] score;
 
+        /** BBox */
+        public Rect [] bbox;
+
+        /** BBox score */
+        public float[] bboxScore;
+
         /** Initializes a new instance of the Result class.
          * @param point: points.
          * @param score: scores of points.
         */
-        public Result(PointF[] point, float [] score) {
+        public Result(PointF[] point, float[] score, Rect[] bbox, float[] bboxScore) {
             this.point = point;
             this.score = score;
+            this.bbox = bbox;
+            this.bboxScore = bboxScore;
         }
     }
 

csrc/mmdeploy/apis/java/native/mmdeploy_PoseDetector.cpp

@@ -37,10 +37,15 @@ jobjectArray Java_mmdeploy_PoseDetector_apply(JNIEnv *env, jobject thiz, jlong h
       return NULL;
     }
     auto result_cls = env->FindClass("mmdeploy/PoseDetector$Result");
-    auto result_ctor = env->GetMethodID(result_cls, "<init>", "([Lmmdeploy/PointF;[F)V");
+    auto result_ctor =
+        env->GetMethodID(result_cls, "<init>", "([Lmmdeploy/PointF;[F[Lmmdeploy/Rect;[F)V");
+    // auto result_ctor =
+    //     env->GetMethodID(result_cls, "<init>", "([Lmmdeploy/PointF;[F;[Lmmdeploy/Rect;[F)V");
     auto array = env->NewObjectArray(size, result_cls, nullptr);
     auto pointf_cls = env->FindClass("mmdeploy/PointF");
     auto pointf_ctor = env->GetMethodID(pointf_cls, "<init>", "(FF)V");
+    auto rect_cls = env->FindClass("mmdeploy/Rect");
+    auto rect_ctor = env->GetMethodID(rect_cls, "<init>", "(FFFF)V");
 
     for (int i = 0; i < size; ++i) {
       auto keypoint_array = env->NewObjectArray(results[i].length, pointf_cls, nullptr);
@@ -51,7 +56,19 @@ jobjectArray Java_mmdeploy_PoseDetector_apply(JNIEnv *env, jobject thiz, jlong h
       }
       auto score_array = env->NewFloatArray(results[i].length);
       env->SetFloatArrayRegion(score_array, 0, results[i].length, (jfloat *)results[i].score);
-      auto res = env->NewObject(result_cls, result_ctor, keypoint_array, score_array);
+      auto bbox_array = env->NewObjectArray(results[i].num_bbox, rect_cls, nullptr);
+      for (int j = 0; j < results[i].num_bbox; j++) {
+        auto bboxj =
+            env->NewObject(rect_cls, rect_ctor, (jfloat)results[i].bboxes[j].left,
+                           (jfloat)results[i].bboxes[j].top, (jfloat)results[i].bboxes[j].right,
+                           (jfloat)results[i].bboxes[j].bottom);
+        env->SetObjectArrayElement(bbox_array, j, bboxj);
+      }
+      auto bbox_score_array = env->NewFloatArray(results[i].num_bbox);
+      env->SetFloatArrayRegion(bbox_score_array, 0, results[i].num_bbox,
+                               (jfloat *)results[i].bbox_score);
+      auto res = env->NewObject(result_cls, result_ctor, keypoint_array, score_array, bbox_array,
+                                bbox_score_array);
       env->SetObjectArrayElement(array, i, res);
     }
     mmdeploy_pose_detector_release_result(results, size);

csrc/mmdeploy/apis/python/pose_detector.cpp

@@ -65,20 +65,38 @@ class PyPoseDetector {
 
     auto output = py::list{};
     auto result = detection;
+
     for (int i = 0; i < mats.size(); i++) {
-      int n_point = result->length;
-      auto pred = py::array_t<float>({bbox_count[i], n_point, 3});
-      auto dst = pred.mutable_data();
-      for (int j = 0; j < bbox_count[i]; j++) {
+      int n_point_total = result->length;
+      int n_bbox = result->num_bbox;
+      int n_point = n_bbox > 0 ? n_point_total / n_bbox : 0;
+      int pts_ind = 0;
+      auto pred_pts = py::array_t<float>({n_bbox * n_point, 3});
+      auto pred_bbox = py::array_t<float>({n_bbox, 5});
+      auto dst_pts = pred_pts.mutable_data();
+      auto dst_bbox = pred_bbox.mutable_data();
+
+      // printf("num_bbox %d num_pts %d\n", result->num_bbox, result->length);
+      for (int j = 0; j < n_bbox; j++) {
         for (int k = 0; k < n_point; k++) {
-          dst[0] = result->point[k].x;
-          dst[1] = result->point[k].y;
-          dst[2] = result->score[k];
-          dst += 3;
+          pts_ind = j * n_point + k;
+          dst_pts[0] = result->point[pts_ind].x;
+          dst_pts[1] = result->point[pts_ind].y;
+          dst_pts[2] = result->score[pts_ind];
+          dst_pts += 3;
+          // printf("pts %f %f %f\n", dst_pts[0], dst_pts[1], dst_pts[2]);
         }
-        result++;
+        dst_bbox[0] = result->bboxes[j].left;
+        dst_bbox[1] = result->bboxes[j].top;
+        dst_bbox[2] = result->bboxes[j].right;
+        dst_bbox[3] = result->bboxes[j].bottom;
+        dst_bbox[4] = result->bbox_score[j];
+        // printf("box %f %f %f %f %f\n", dst_bbox[0], dst_bbox[1], dst_bbox[2], dst_bbox[3],
+        // dst_bbox[4]);
+        dst_bbox += 5;
       }
-      output.append(std::move(pred));
+      result++;
+      output.append(py::make_tuple(std::move(pred_bbox), std::move(pred_pts)));
     }
 
     int total = std::accumulate(bbox_count.begin(), bbox_count.end(), 0);
@@ -101,12 +119,12 @@ static PythonBindingRegisterer register_pose_detector{[](py::module& m) {
            }),
            py::arg("model_path"), py::arg("device_name"), py::arg("device_id") = 0)
       .def("__call__",
-           [](PyPoseDetector* self, const PyImage& img) -> py::array {
+           [](PyPoseDetector* self, const PyImage& img) -> py::tuple {
              return self->Apply({img}, {})[0];
            })
       .def(
           "__call__",
-          [](PyPoseDetector* self, const PyImage& img, const Rect& box) -> py::array {
+          [](PyPoseDetector* self, const PyImage& img, const Rect& box) -> py::tuple {
             std::vector<std::vector<Rect>> bboxes;
             bboxes.push_back({box});
             return self->Apply({img}, bboxes)[0];
@@ -115,7 +133,7 @@ static PythonBindingRegisterer register_pose_detector{[](py::module& m) {
       .def(
           "__call__",
           [](PyPoseDetector* self, const PyImage& img,
-             const std::vector<Rect>& bboxes) -> py::array {
+             const std::vector<Rect>& bboxes) -> py::tuple {
             std::vector<std::vector<Rect>> _bboxes;
             _bboxes.push_back(bboxes);
             return self->Apply({img}, _bboxes)[0];

csrc/mmdeploy/codebase/mmpose/keypoints_from_heatmap.cpp

@@ -55,6 +55,9 @@ class TopdownHeatmapBaseHeadDecode : public MMPose {
     }
 
     auto& img_metas = _data["img_metas"];
+    if (img_metas.contains("bbox")) {
+      from_value(img_metas["bbox"], bbox_);
+    }
 
     vector<float> center;
     vector<float> scale;
@@ -78,6 +81,7 @@ class TopdownHeatmapBaseHeadDecode : public MMPose {
       output.key_points.push_back({{x, y}, s});
       data += 3;
     }
+    output.detections.push_back({{bbox_[0], bbox_[1], bbox_[2], bbox_[3]}, bbox_[4]});
     return to_value(std::move(output));
   }
 
@@ -354,6 +358,7 @@ class TopdownHeatmapBaseHeadDecode : public MMPose {
   float valid_radius_factor_{0.0546875f};
   bool use_udp_{false};
   string target_type_{"GaussianHeatmap"};
+  vector<float> bbox_{0, 0, 1, 1, 1};
 };
 
 MMDEPLOY_REGISTER_CODEBASE_COMPONENT(MMPose, TopdownHeatmapBaseHeadDecode);

csrc/mmdeploy/codebase/mmpose/keypoints_from_regression.cpp

@@ -36,7 +36,9 @@ class DeepposeRegressionHeadDecode : public MMPose {
     }
 
     auto& img_metas = _data["img_metas"];
-
+    if (img_metas.contains("bbox")) {
+      from_value(img_metas["bbox"], bbox_);
+    }
     vector<float> center;
     vector<float> scale;
     from_value(img_metas["center"], center);
@@ -60,6 +62,7 @@ class DeepposeRegressionHeadDecode : public MMPose {
       output.key_points.push_back({{x, y}, s});
       data += 3;
     }
+    output.detections.push_back({{bbox_[0], bbox_[1], bbox_[2], bbox_[3]}, bbox_[4]});
     return to_value(std::move(output));
   }
 
@@ -106,6 +109,9 @@ class DeepposeRegressionHeadDecode : public MMPose {
     *(data + 0) = *(data + 0) * scale_x + center[0] - scale[0] * 0.5;
     *(data + 1) = *(data + 1) * scale_y + center[1] - scale[1] * 0.5;
   }
+
+ private:
+  vector<float> bbox_{0, 0, 1, 1, 1};
 };
 
 MMDEPLOY_REGISTER_CODEBASE_COMPONENT(MMPose, DeepposeRegressionHeadDecode);

csrc/mmdeploy/codebase/mmpose/mmpose.h

@@ -17,8 +17,14 @@ struct PoseDetectorOutput {
     float score;
     MMDEPLOY_ARCHIVE_MEMBERS(bbox, score);
   };
+  struct BBox {
+    std::array<float, 4> boundingbox;  // x1,y1,x2,y2
+    float score;
+    MMDEPLOY_ARCHIVE_MEMBERS(boundingbox, score);
+  };
   std::vector<KeyPoint> key_points;
-  MMDEPLOY_ARCHIVE_MEMBERS(key_points);
+  std::vector<BBox> detections;
+  MMDEPLOY_ARCHIVE_MEMBERS(key_points, detections);
 };
 
 MMDEPLOY_DECLARE_CODEBASE(MMPose, mmpose);

csrc/mmdeploy/codebase/mmpose/simcc_label.cpp

@@ -49,7 +49,9 @@ class SimCCLabelDecode : public MMPose {
     }
 
     auto& img_metas = _data["img_metas"];
-
+    if (img_metas.contains("bbox")) {
+      from_value(img_metas["bbox"], bbox_);
+    }
     Tensor keypoints({Device{"cpu"}, DataType::kFLOAT, {simcc_x.shape(0), simcc_x.shape(1), 2}});
     Tensor scores({Device{"cpu"}, DataType::kFLOAT, {simcc_x.shape(0), simcc_x.shape(1), 1}});
     get_simcc_maximum(simcc_x, simcc_y, keypoints, scores);
@@ -73,6 +75,7 @@ class SimCCLabelDecode : public MMPose {
       keypoints_data += 2;
       scores_data += 1;
     }
+    output.detections.push_back({{bbox_[0], bbox_[1], bbox_[2], bbox_[3]}, bbox_[4]});
     return to_value(output);
   }
 
@@ -103,6 +106,7 @@ class SimCCLabelDecode : public MMPose {
   }
 
  private:
+  vector<float> bbox_{0, 0, 1, 1, 1};
   bool flip_test_{false};
   bool shift_heatmap_{false};
   float simcc_split_ratio_{2.0};