opencv · asmorkalov · Jul 2, 2024 · Jun 20, 2024 · Jun 20, 2024 · Jun 20, 2024
diff --git a/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown b/doc/tutorials/dnn/dnn_yolo/dnn_yolo.markdown
@@ -24,7 +24,9 @@ model, but the methodology applies to other supported models.
 
 @note Currently, OpenCV supports the following YOLO models:
 - [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX/blob/main),
-- [YoloNas](https://github.com/Deci-AI/super-gradients/tree/master),
+- [YOLONas](https://github.com/Deci-AI/super-gradients/tree/master),
+- [YOLOv10](https://github.com/THU-MIG/yolov10/tree/main),
+- [YOLOv9](https://github.com/WongKinYiu/yolov9),
 - [YOLOv8](https://github.com/ultralytics/ultralytics/tree/main),
 - [YOLOv7](https://github.com/WongKinYiu/yolov7/tree/main),
 - [YOLOv6](https://github.com/meituan/YOLOv6/blob/main),
@@ -79,7 +81,7 @@ the ONNX graph, a process that we will detail further in the subsequent sections
 
 Now that we know know the parameters of the pre-precessing we can go on and export the model from
 Pytorch to ONNX graph. Since in this tutorial we are using YOLOX as our sample model, lets use its
-export for demonstration purposes (the process is  identical for the rest of the YOLO detectors).
+export for demonstration purposes (the process is  identical for the rest of the YOLO detectors except `YOLOv10` model, see details on how to export it later in the post).
 To exporting YOLOX we can just use [export script](https://github.com/Megvii-BaseDetection/YOLOX/blob/ac58e0a5e68e57454b7b9ac822aced493b553c53/tools/export_onnx.py). Particularly we need following commands:
 
 @code{.bash}
@@ -125,6 +127,20 @@ than YOLOX) in case it is needed. However, usually each YOLO repository has pred
     onnx.save(model_simp, args.output_name)
 @endcode
 
+#### Exporting YOLOv10 model
+
+In oder to run YOLOv10 one needs to cut off postporcessing with dynamic shapes from torch and then convert it to ONNX. If someone is looking for on how to cut off the postprocessing, there is this [forked branch](https://github.com/Abdurrahheem/yolov10/tree/ash/opencv-export) from official YOLOv10. The forked branch cuts of the postprocessing by [returning output](https://github.com/Abdurrahheem/yolov10/blob/4fdaafd912c8891642bfbe85751ea66ec20f05ad/ultralytics/nn/modules/head.py#L522) of the model before postprocessing procedure itself. To convert torch model to ONNX follow this proceduce.
+
+@code{.bash}
+git clone [email protected]:Abdurrahheem/yolov10.git
+conda create -n yolov10 python=3.9
+conda activate yolov10
+pip install -r requirements.txt
+python export_opencv.py --model=<model-name> --imgsz=<input-img-size>
+@endcode
+
+By default `--model="yolov10s"` and `--imgsz=(480,640)`. This will generate file `yolov10s.onnx`, which can be use for inference in OpenCV
+
 ### Running Yolo ONNX detector with OpenCV Sample
 
 Once we have our ONNX graph of the model, we just simply can run with OpenCV's sample. To that we need to make sure:
@@ -144,24 +160,25 @@ Once we have our ONNX graph of the model, we just simply can run with OpenCV's s
                                 --padvalue=<padding_value> \
                                 --paddingmode=<padding_mode> \
                                 --backend=<computation_backend> \
-                                --target=<target_computation_device>
+                                --target=<target_computation_device> \
+                                --width=<model_input_width> \
+                                --height=<model_input_height> \
 @endcode
 
-VIDEO DEMO:
-@youtube{NHtRlndE2cg}
-
 - --input: File path to your input image or video. If omitted, it will capture frames from a camera.
 - --classes: File path to a text file containing class names for object detection.
 - --thr: Confidence threshold for detection (e.g., 0.5).
 - --nms: Non-maximum suppression threshold (e.g., 0.4).
 - --mean: Mean normalization value (e.g., 0.0 for no mean normalization).
-- --scale: Scale factor for input normalization (e.g., 1.0).
+- --scale: Scale factor for input normalization (e.g., 1.0, 1/255.0, etc).
 - --yolo: YOLO model version (e.g., YOLOv3, YOLOv4, etc.).
 - --padvalue: Padding value used in pre-processing (e.g., 114.0).
 - --paddingmode: Method for handling image resizing and padding. Options: 0 (resize without extra processing), 1 (crop after resize), 2 (resize with aspect ratio preservation).
 - --backend: Selection of computation backend (0 for automatic, 1 for Halide, 2 for OpenVINO, etc.).
 - --target: Selection of target computation device (0 for CPU, 1 for OpenCL, etc.).
 - --device: Camera device number (0 for default camera). If `--input` is not provided camera with index 0 will used by default.
+- --width: Model input width. Not to be confused with the image width. (e.g., 416, 480, 640, 1280, etc).
+- --height: Model input height. Not to be confused with the image height. (e.g., 416, 480, 640, 1280, etc).
 
 Here `mean`, `scale`, `padvalue`, `paddingmode` should exactly match those that we discussed
 in pre-processing section in order for the model to match result in PyTorch
@@ -183,7 +200,8 @@ cd <build directory of OpenCV>
 ./bin/example_dnn_yolo_detector
 @endcode
 
-This will execute the YOLOX detector with your camera. For YOLOv8 (for instance), follow these additional steps:
+This will execute the YOLOX detector with your camera.
+For YOLOv8 (for instance), follow these additional steps:
 
 @code{.sh}
 cd opencv_extra/testdata/dnn
@@ -195,6 +213,23 @@ cd <build directory of OpenCV>
 ./bin/example_dnn_yolo_detector --model=onnx/models/yolov8n.onnx --yolo=yolov8 --mean=0.0 --scale=0.003921568627 --paddingmode=2 --padvalue=144.0 --thr=0.5 --nms=0.4 --rgb=0
 @endcode
 
+For YOLOv10, follow these steps:
+
+@code{.sh}
+cd opencv_extra/testdata/dnn
+python download_models.py yolov10
+cd ..
+export OPENCV_TEST_DATA_PATH=$(pwd)
+cd <build directory of OpenCV>
+
+./bin/example_dnn_yolo_detector --model=onnx/models/yolov8n.onnx --yolo=yolov10 --width=640 --height=480  --scale=0.003921568627 --padvalue=114
+@endcode
+
+This will run `YOLOv10` detector on first camera found on your system. If you want to run it on a image/video file, you can use `--input` option to specify the path to the file.
+
+
+VIDEO DEMO:
+@youtube{NHtRlndE2cg}
 
 ### Building a Custom Pipeline
 

diff --git a/modules/dnn/test/test_onnx_importer.cpp b/modules/dnn/test/test_onnx_importer.cpp
@@ -19,7 +19,8 @@ void yoloPostProcessing(
     std::vector<Rect2d>& keep_boxes,
     float conf_threshold,
     float iou_threshold,
-    const std::string& test_name);
+    const std::string& model_name,
+    const int nc=80);
 
 template<typename TString>
 static std::string _tf(TString filename, bool required = true)
@@ -2670,19 +2671,22 @@ void yoloPostProcessing(
     std::vector<Rect2d>& keep_boxes,
     float conf_threshold,
     float iou_threshold,
-    const std::string& test_name
+    const std::string& model_name,
+    const int nc
 ){
 
     // Retrieve
     std::vector<int> classIds;
     std::vector<float> confidences;
     std::vector<Rect2d> boxes;
 
-    if (test_name == "yolov8"){
+    if (model_name == "yolov8" || model_name == "yolov10" ||
+        model_name == "yolov9")
+    {
         cv::transposeND(outs[0], {0, 2, 1}, outs[0]);
     }
 
-    if (test_name == "yolonas"){
+    if (model_name == "yolonas"){
         // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84]
         Mat concat_out;
         // squeeze the first dimension
@@ -2696,22 +2700,27 @@ void yoloPostProcessing(
         outs[0] = outs[0].reshape(0, std::vector<int>{1, 8400, 84});
     }
 
+    // assert if last dim is 85 or 84
+    CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]");
+    CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: ");
+
     for (auto preds : outs){
 
         preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85]
         for (int i = 0; i < preds.rows; ++i)
         {
             // filter out non object
-            float obj_conf = (test_name == "yolov8" || test_name == "yolonas") ? 1.0f : preds.at<float>(i, 4) ;
+            float obj_conf = (model_name == "yolov8" || model_name == "yolonas" ||
+                              model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at<float>(i, 4) ;
             if (obj_conf < conf_threshold)
                 continue;
 
-            Mat scores = preds.row(i).colRange((test_name == "yolov8" || test_name == "yolonas") ? 4 : 5, preds.cols);
+            Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols);
             double conf;
             Point maxLoc;
             minMaxLoc(scores, 0, &conf, 0, &maxLoc);
 
-            conf = (test_name == "yolov8" || test_name == "yolonas") ? conf : conf * obj_conf;
+            conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf;
             if (conf < conf_threshold)
                 continue;
 
@@ -2722,15 +2731,14 @@ void yoloPostProcessing(
             double w = det[2];
             double h = det[3];
 
-            // std::cout << "cx: " << cx << " cy: " << cy << " w: " << w << " h: " << h << " conf: " << conf << " idx: " << maxLoc.x << std::endl;
             // [x1, y1, x2, y2]
-            if (test_name == "yolonas"){
+            if (model_name == "yolonas" || model_name == "yolov10"){
                 boxes.push_back(Rect2d(cx, cy, w, h));
             } else {
                 boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h,
                                         cx + 0.5 * w, cy + 0.5 * h));
             }
-           classIds.push_back(maxLoc.x);
+            classIds.push_back(maxLoc.x);
             confidences.push_back(conf);
         }
     }
@@ -2747,7 +2755,75 @@ void yoloPostProcessing(
     }
 }
 
+TEST_P(Test_ONNX_nets, YOLOv10)
+{
+
+    std::string weightPath = _tf("models/yolov10s.onnx", false);
+
+    Size targetSize{640, 480};
+    float conf_threshold = 0.50;
+    float iou_threshold = 0.50;
+
+    std::vector<int> refClassIds{1, 16, 7};
+    std::vector<float> refScores{0.9510f, 0.9454f, 0.8404f};
+
+    std::vector<Rect2d> refBoxes{
+        Rect2d(105.5014, 112.8838, 472.9274, 350.0603),
+        Rect2d(109.8231, 185.7994, 258.5916, 452.9302),
+        Rect2d(388.5018,  62.1034, 576.6399, 143.3986)
+        };
+
+    Image2BlobParams imgParams(
+        Scalar::all(1 / 255.0),
+        targetSize,
+        Scalar::all(0),
+        true,
+        CV_32F,
+        DNN_LAYOUT_NCHW,
+        DNN_PMODE_LETTERBOX,
+        Scalar::all(114)
+        );
 
+    testYOLO(
+        weightPath, refClassIds, refScores, refBoxes,
+        imgParams, conf_threshold, iou_threshold,
+        1.0e-4, 1.0e-4, "yolov10");
+}
+
+TEST_P(Test_ONNX_nets, YOLOv9)
+{
+
+    std::string weightPath = _tf("models/yolov9t.onnx", false);
+
+    Size targetSize{640, 480};
+    float conf_threshold = 0.50;
+    float iou_threshold = 0.50;
+
+    std::vector<int> refClassIds{1, 16, 2}; // wrong class mapping for yolov9
+    std::vector<float> refScores{0.959274f, 0.901125, 0.559396f};
+
+    std::vector<Rect2d> refBoxes{
+        Rect2d(106.255, 107.927, 472.497, 350.309),
+        Rect2d(108.633, 185.256, 259.287, 450.672),
+        Rect2d(390.701, 62.1454, 576.928, 141.795)
+        };
+
+    Image2BlobParams imgParams(
+        Scalar::all(1 / 255.0),
+        targetSize,
+        Scalar::all(0),
+        true,
+        CV_32F,
+        DNN_LAYOUT_NCHW,
+        DNN_PMODE_LETTERBOX,
+        Scalar::all(114)
+        );
+
+    testYOLO(
+        weightPath, refClassIds, refScores, refBoxes,
+        imgParams, conf_threshold, iou_threshold,
+        1.0e-4, 1.0e-4, "yolov9");
+}
 TEST_P(Test_ONNX_nets, YOLOX)
 {
     applyTestTag(CV_TEST_TAG_DEBUG_VERYLONG);

diff --git a/samples/dnn/yolo_detector.cpp b/samples/dnn/yolo_detector.cpp
@@ -27,7 +27,8 @@ void yoloPostProcessing(
     std::vector<Rect2d>& keep_boxes,
     float conf_threshold,
     float iou_threshold,
-    const std::string& test_name
+    const std::string& model_name,
+    const int nc
 );
 
 std::vector<std::string> classes;
@@ -40,6 +41,7 @@ std::string keys =
     "{ yolo        | yolox | yolo model version. }"
     "{ input i     | | Path to input image or video file. Skip this argument to capture frames from a camera. }"
     "{ classes     | | Optional path to a text file with names of classes to label detected objects. }"
+    "{ nc          | 80 | Number of classes. Default is 80 (coming from COCO dataset). }"
     "{ thr         | .5 | Confidence threshold. }"
     "{ nms         | .4 | Non-maximum suppression threshold. }"
     "{ mean        | 0.0 | Normalization constant. }"
@@ -107,19 +109,21 @@ void yoloPostProcessing(
     std::vector<Rect2d>& keep_boxes,
     float conf_threshold,
     float iou_threshold,
-    const std::string& test_name)
+    const std::string& model_name,
+    const int nc=80)
 {
     // Retrieve
     std::vector<int> classIds;
     std::vector<float> confidences;
     std::vector<Rect2d> boxes;
 
-    if (test_name == "yolov8")
+    if (model_name == "yolov8" || model_name == "yolov10" ||
+        model_name == "yolov9")
     {
         cv::transposeND(outs[0], {0, 2, 1}, outs[0]);
     }
 
-    if (test_name == "yolonas")
+    if (model_name == "yolonas")
     {
         // outs contains 2 elemets of shape [1, 8400, 80] and [1, 8400, 4]. Concat them to get [1, 8400, 84]
         Mat concat_out;
@@ -131,25 +135,30 @@ void yoloPostProcessing(
         // remove the second element
         outs.pop_back();
         // unsqueeze the first dimension
-        outs[0] = outs[0].reshape(0, std::vector<int>{1, 8400, 84});
+        outs[0] = outs[0].reshape(0, std::vector<int>{1, 8400, nc + 4});
     }
 
+    // assert if last dim is 85 or 84
+    CV_CheckEQ(outs[0].dims, 3, "Invalid output shape. The shape should be [1, #anchors, 85 or 84]");
+    CV_CheckEQ((outs[0].size[2] == nc + 5 || outs[0].size[2] == 80 + 4), true, "Invalid output shape: ");
+
     for (auto preds : outs)
     {
         preds = preds.reshape(1, preds.size[1]); // [1, 8400, 85] -> [8400, 85]
         for (int i = 0; i < preds.rows; ++i)
         {
             // filter out non object
-            float obj_conf = (test_name == "yolov8" || test_name == "yolonas") ? 1.0f : preds.at<float>(i, 4) ;
+            float obj_conf = (model_name == "yolov8" || model_name == "yolonas" ||
+                              model_name == "yolov9" || model_name == "yolov10") ? 1.0f : preds.at<float>(i, 4) ;
             if (obj_conf < conf_threshold)
                 continue;
 
-            Mat scores = preds.row(i).colRange((test_name == "yolov8" || test_name == "yolonas") ? 4 : 5, preds.cols);
+            Mat scores = preds.row(i).colRange((model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? 4 : 5, preds.cols);
             double conf;
             Point maxLoc;
             minMaxLoc(scores, 0, &conf, 0, &maxLoc);
 
-            conf = (test_name == "yolov8" || test_name == "yolonas") ? conf : conf * obj_conf;
+            conf = (model_name == "yolov8" || model_name == "yolonas" || model_name == "yolov9" || model_name == "yolov10") ? conf : conf * obj_conf;
             if (conf < conf_threshold)
                 continue;
 
@@ -161,7 +170,7 @@ void yoloPostProcessing(
             double h = det[3];
 
             // [x1, y1, x2, y2]
-            if (test_name == "yolonas"){
+            if (model_name == "yolonas" || model_name == "yolov10"){
                 boxes.push_back(Rect2d(cx, cy, w, h));
             } else {
                 boxes.push_back(Rect2d(cx - 0.5 * w, cy - 0.5 * h,
@@ -203,6 +212,7 @@ int main(int argc, char** argv)
     // if model is default, use findFile to get the full path otherwise use the given path
     std::string weightPath = findFile(parser.get<String>("model"));
     std::string yolo_model = parser.get<String>("yolo");
+    int nc = parser.get<int>("nc");
 
     float confThreshold = parser.get<float>("thr");
     float nmsThreshold = parser.get<float>("nms");
@@ -219,6 +229,7 @@ int main(int argc, char** argv)
     // check if yolo model is valid
     if (yolo_model != "yolov5" && yolo_model != "yolov6"
         && yolo_model != "yolov7" && yolo_model != "yolov8"
+        && yolo_model != "yolov10" && yolo_model !="yolov9"
         && yolo_model != "yolox" && yolo_model != "yolonas")
         CV_Error(Error::StsError, "Invalid yolo model: " + yolo_model);
 
@@ -331,7 +342,8 @@ int main(int argc, char** argv)
         yoloPostProcessing(
             outs, keep_classIds, keep_confidences, keep_boxes,
             confThreshold, nmsThreshold,
-            yolo_model);
+            yolo_model,
+            nc);
         //![postprocess]
 
         // covert Rect2d to Rect