feat: add tensorrt yolo package

perception/tensorrt_yolo/.gitignore

@@ -0,0 +1,2 @@
+data/
+calib_image/

perception/tensorrt_yolo/CMakeLists.txt

@@ -0,0 +1,272 @@
+cmake_minimum_required(VERSION 3.5)
+project(tensorrt_yolo)
+
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic -Wno-deprecated-declarations -Werror)
+endif()
+
+option(CUDA_VERBOSE "Verbose output of CUDA modules" OFF)
+
+find_package(ament_cmake_auto REQUIRED)
+ament_auto_find_build_dependencies()
+
+find_package(OpenCV REQUIRED)
+
+# set flags for CUDA availability
+option(CUDA_AVAIL "CUDA available" OFF)
+find_package(CUDA)
+if(CUDA_FOUND)
+  find_library(CUBLAS_LIBRARIES cublas HINTS
+    ${CUDA_TOOLKIT_ROOT_DIR}/lib64
+    ${CUDA_TOOLKIT_ROOT_DIR}/lib
+  )
+  if(CUDA_VERBOSE)
+    message(STATUS "CUDA is available!")
+    message(STATUS "CUDA Libs: ${CUDA_LIBRARIES}")
+    message(STATUS "CUDA Headers: ${CUDA_INCLUDE_DIRS}")
+  endif()
+  set(CUDA_AVAIL ON)
+else()
+  message("CUDA NOT FOUND")
+  set(CUDA_AVAIL OFF)
+endif()
+
+# set flags for TensorRT availability
+option(TRT_AVAIL "TensorRT available" OFF)
+# try to find the tensorRT modules
+find_library(NVINFER NAMES nvinfer)
+find_library(NVONNXPARSER nvonnxparser)
+find_library(NVINFER_PLUGIN NAMES nvinfer_plugin)
+if(NVINFER AND NVONNXPARSER AND NVINFER_PLUGIN)
+  if(CUDA_VERBOSE)
+    message(STATUS "TensorRT is available!")
+    message(STATUS "NVINFER: ${NVINFER}")
+    message(STATUS "NVPARSERS: ${NVPARSERS}")
+    message(STATUS "NVINFER_PLUGIN: ${NVINFER_PLUGIN}")
+    message(STATUS "NVONNXPARSER: ${NVONNXPARSER}")
+  endif()
+  set(TRT_AVAIL ON)
+else()
+  message("TensorRT is NOT Available")
+  set(TRT_AVAIL OFF)
+endif()
+
+# set flags for CUDNN availability
+option(CUDNN_AVAIL "CUDNN available" OFF)
+# try to find the CUDNN module
+find_library(CUDNN_LIBRARY
+NAMES libcudnn.so${__cudnn_ver_suffix} libcudnn${__cudnn_ver_suffix}.dylib ${__cudnn_lib_win_name}
+PATHS $ENV{LD_LIBRARY_PATH} ${__libpath_cudart} ${CUDNN_ROOT_DIR} ${PC_CUDNN_LIBRARY_DIRS} ${CMAKE_INSTALL_PREFIX}
+PATH_SUFFIXES lib lib64 bin
+DOC "CUDNN library."
+)
+if(CUDNN_LIBRARY)
+  if(CUDA_VERBOSE)
+    message(STATUS "CUDNN is available!")
+    message(STATUS "CUDNN_LIBRARY: ${CUDNN_LIBRARY}")
+  endif()
+  set(CUDNN_AVAIL ON)
+else()
+  message("CUDNN is NOT Available")
+  set(CUDNN_AVAIL OFF)
+endif()
+
+# Download onnx
+find_program(GDOWN_AVAIL "gdown")
+if(NOT GDOWN_AVAIL)
+  message("gdown: command not found. External files could not be downloaded.")
+endif()
+set(PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
+if(NOT EXISTS "${PATH}")
+  execute_process(COMMAND mkdir -p ${PATH})
+endif()
+# yolov3
+set(FILE "${PATH}/yolov3.onnx")
+message(STATUS "Checking and downloading yolov3.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com/uc?id=1ZYoBqVynmO5kKntyN56GEbELRpuXG8Ym" -O ${PATH}/yolov3.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov4
+set(FILE "${PATH}/yolov4.onnx")
+message(STATUS "Checking and downloading yolov4.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com//uc?id=1vkNmSwcIpTkJ_-BrKhxtit0PBJeJYTVX" -O ${PATH}/yolov4.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov4-tiny
+set(FILE "${PATH}/yolov4-tiny.onnx")
+message(STATUS "Checking and downloading yolov4.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com/uc?id=1rUHjV_JfkmlVFgb20XXrOMWo_HZAjrFh" -O ${PATH}/yolov4-tiny.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov5s
+set(FILE "${PATH}/yolov5s.onnx")
+message(STATUS "Checking and downloading yolov5s.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com//uc?id=1CF21nQWigwCPTr5psQZXg6cBQIOYKbad" -O ${PATH}/yolov5s.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov5m
+set(FILE "${PATH}/yolov5m.onnx")
+message(STATUS "Checking and downloading yolov5m.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com//uc?id=1a1h50KJH6slwmjKZpPlS-errukF-BrgG" -O ${PATH}/yolov5m.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov5l
+set(FILE "${PATH}/yolov5l.onnx")
+message(STATUS "Checking and downloading yolov5l.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com/uc?id=1xO8S92Cq7qrmx93UHHyA7Cd7-dJsBDP8" -O ${PATH}/yolov5l.onnx
+    ERROR_QUIET
+  )
+endif()
+
+# yolov5x
+set(FILE "${PATH}/yolov5x.onnx")
+message(STATUS "Checking and downloading yolov5x.onnx")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com/uc?id=1kAHuNJUCxpD-yWrS6t95H3zbAPfClLxI" -O ${PATH}/yolov5x.onnx
+    ERROR_QUIET
+  )
+endif()
+
+set(FILE "${PATH}/coco.names")
+message(STATUS "Checking and downloading coco.names")
+if(NOT EXISTS "${FILE}")
+  message(STATUS "... file does not exist. Downloading now ...")
+  execute_process(
+    COMMAND gdown "https://drive.google.com/uc?id=19wXD23PE16kJDkZ7j2W3Ijvx5I1kO9kJ" -O ${PATH}/coco.names
+    ERROR_QUIET
+  )
+endif()
+
+# create calib image directory for int8 calibration
+set(CALIB_IMAGE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/calib_image")
+if(NOT EXISTS "${CALIB_IMAGE_PATH}")
+  execute_process(COMMAND mkdir -p ${CALIB_IMAGE_PATH})
+endif()
+
+if(TRT_AVAIL AND CUDA_AVAIL AND CUDNN_AVAIL)
+  include_directories(
+    include
+    lib/include
+    ${OpenCV_INCLUDE_DIRS}
+    ${CUDA_INCLUDE_DIRS}
+  )
+
+
+  ### yolo ###
+  cuda_add_library(mish_plugin SHARED
+    lib/src/plugins/mish.cu
+    lib/src/plugins/mish_plugin.cpp
+  )
+
+  cuda_add_library(yolo_layer_plugin SHARED
+    lib/src/plugins/yolo_layer.cu
+    lib/src/plugins/yolo_layer_plugin.cpp
+  )
+
+  cuda_add_library(nms_plugin SHARED
+    lib/src/plugins/nms.cu
+    lib/src/plugins/nms_plugin.cpp
+  )
+
+  ament_auto_add_library(yolo SHARED
+    lib/src/trt_yolo.cpp
+  )
+
+  target_include_directories(yolo PUBLIC
+    lib/include
+  )
+
+  target_link_libraries(yolo
+    ${NVINFER}
+    ${NVONNXPARSER}
+    ${NVINFER_PLUGIN}
+    ${CUDA_LIBRARIES}
+    ${CUBLAS_LIBRARIES}
+    ${CUDA_curand_LIBRARY}
+    ${CUDNN_LIBRARY}
+    mish_plugin
+    yolo_layer_plugin
+    nms_plugin
+  )
+
+  ament_auto_add_library(tensorrt_yolo_nodelet SHARED
+    src/nodelet.cpp
+  )
+
+  target_link_libraries(tensorrt_yolo_nodelet
+    ${OpenCV_LIBS}
+    yolo
+    mish_plugin
+    yolo_layer_plugin
+    nms_plugin
+  )
+
+  rclcpp_components_register_node(tensorrt_yolo_nodelet
+    PLUGIN "object_recognition::TensorrtYoloNodelet"
+    EXECUTABLE tensorrt_yolo_node
+  )
+
+  if(BUILD_TESTING)
+    find_package(ament_lint_auto REQUIRED)
+    ament_lint_auto_find_test_dependencies()
+  endif()
+
+  ament_auto_package(INSTALL_TO_SHARE
+    config
+    data
+    launch
+  )
+
+  install(
+    TARGETS
+      mish_plugin
+      yolo_layer_plugin
+      nms_plugin
+    LIBRARY DESTINATION lib
+    ARCHIVE DESTINATION lib
+    RUNTIME DESTINATION bin
+  )
+
+else()
+  message("TensorrtYolo won't be built, CUDA and/or TensorRT were not found.")
+  ament_auto_package(INSTALL_TO_SHARE
+    config
+    data
+    launch
+  )
+endif()

perception/tensorrt_yolo/README.md

@@ -0,0 +1,99 @@
+# tensorrt_yolo
+
+## Purpose
+
+This package detects 2D bounding boxes for target objects e.g., cars, trucks, bicycles, and pedestrians on a image based on YOLO(You only look once) model.
+
+## Inner-workings / Algorithms
+
+### Cite
+
+yolov3
+
+Redmon, J., & Farhadi, A. (2018). Yolov3: An incremental improvement. arXiv preprint arXiv:1804.02767.
+
+yolov4
+
+Bochkovskiy, A., Wang, C. Y., & Liao, H. Y. M. (2020). Yolov4: Optimal speed and accuracy of object detection. arXiv preprint arXiv:2004.10934.
+
+yolov5
+
+Jocher, G., et al. (2021). ultralytics/yolov5: v6.0 - YOLOv5n 'Nano' models, Roboflow integration, TensorFlow export, OpenCV DNN support (v6.0). Zenodo. <https://doi.org/10.5281/zenodo.5563715>
+
+## Inputs / Outputs
+
+### Input
+
+| Name       | Type                | Description     |
+| ---------- | ------------------- | --------------- |
+| `in/image` | `sensor_msgs/Image` | The input image |
+
+### Output
+
+| Name          | Type                                                  | Description                                        |
+| ------------- | ----------------------------------------------------- | -------------------------------------------------- |
+| `out/objects` | `autoware_perception_msgs/DetectedObjectsWithFeature` | The detected objects with 2D bounding boxes        |
+| `out/image`   | `sensor_msgs/Image`                                   | The image with 2D bounding boxes for visualization |
+
+## Parameters
+
+### Core Parameters
+
+| Name                | Type         | Default Value                                                                                                      | Description                                                                           |
+| ------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------- |
+| `anchors`           | double array | [10.0, 13.0, 16.0, 30.0, 33.0, 23.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 116.0, 90.0, 156.0, 198.0, 373.0, 326.0] | The anchors to create bounding box candidates                                         |
+| `scale_x_y`         | double array | [1.0, 1.0, 1.0]                                                                                                    | The scale parameter to eliminate grid sensitivity                                     |
+| `score_thresh`      | double       | 0.1                                                                                                                | If the objectness score is less than this value, the object is ignored in yolo layer. |
+| `iou_thresh`        | double       | 0.45                                                                                                               | The iou threshold for NMS method                                                      |
+| `detections_per_im` | int          | 100                                                                                                                | The maximum detection number for one frame                                            |
+| `use_darknet_layer` | bool         | true                                                                                                               | The flag to use yolo layer in darknet                                                 |
+| `ignore_thresh`     | double       | 0.5                                                                                                                | If the output score is less than this value, ths object is ignored.                   |
+
+### Node Parameters
+
+| Name                    | Type   | Default Value | Description                                                        |
+| ----------------------- | ------ | ------------- | ------------------------------------------------------------------ |
+| `onnx_file`             | string | ""            | The onnx file name for yolo model                                  |
+| `engine_file`           | string | ""            | The tensorrt engine file name for yolo model                       |
+| `label_file`            | string | ""            | The label file with label names for detected objects written on it |
+| `calib_image_directory` | string | ""            | The directory name including calibration images for int8 inference |
+| `calib_cache_file`      | string | ""            | The calibration cache file for int8 inference                      |
+| `mode`                  | string | "FP32"        | The inference mode: "FP32", "FP16", "INT8"                         |
+
+## Assumptions / Known limits
+
+This package includes multiple licenses.
+
+## Onnx model
+
+### YOLOv3
+
+[YOLOv3](https://drive.google.com/uc?id=1ZYoBqVynmO5kKntyN56GEbELRpuXG8Ym "YOLOv3"): Converted from darknet [weight file](https://pjreddie.com/media/files/yolov3.weights "weight file") and [conf file](https://github.com/pjreddie/darknet/blob/master/cfg/yolov3.cfg "conf file").
+
+### YOLOv4
+
+[YOLOv4](https://drive.google.com//uc?id=1vkNmSwcIpTkJ_-BrKhxtit0PBJeJYTVX "YOLOv4"): Converted from darknet [weight file](https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights "weight file") and [conf file](https://github.com/AlexeyAB/darknet/blob/master/cfg/yolov4.cfg "conf file").
+
+### YOLOv5
+
+Refer to [this guide](https://github.com/ultralytics/yolov5/issues/251 "guide")
+
+- [YOLOv5s](https://drive.google.com//uc?id=1CF21nQWigwCPTr5psQZXg6cBQIOYKbad "YOLOv5s")
+
+- [YOLOv5m](https://drive.google.com//uc?id=1a1h50KJH6slwmjKZpPlS-errukF-BrgG "YOLOv5m")
+
+- [YOLOv5l](https://drive.google.com/uc?id=1xO8S92Cq7qrmx93UHHyA7Cd7-dJsBDP8 "YOLOv5l")
+
+- [YOLOv5x](https://drive.google.com/uc?id=1kAHuNJUCxpD-yWrS6t95H3zbAPfClLxI "YOLOv5x")
+
+## Reference repositories
+
+- <https://github.com/pjreddie/darknet>
+
+- <https://github.com/AlexeyAB/darknet>
+
+- <https://github.com/ultralytics/yolov5>
+
+- <https://github.com/wang-xinyu/tensorrtx>
+
+- <https://github.com/NVIDIA/retinanet-examples>

perception/tensorrt_yolo/config/yolov3.param.yaml

@@ -0,0 +1,10 @@
+/**:
+  ros__parameters:
+    num_anchors: 3
+    anchors: [116.0, 90.0, 156.0, 198.0, 373.0, 326.0, 30.0, 61.0, 62.0, 45.0, 59.0, 119.0, 10.0, 13.0, 16.0, 30.0, 33.0, 23.0]
+    scale_x_y: [1.0, 1.0, 1.0]
+    score_threshold: 0.1
+    iou_thresh: 0.45
+    detections_per_im: 100
+    use_darknet_layer: true
+    ignore_thresh: 0.5

perception/tensorrt_yolo/config/yolov4-tiny.param.yaml

@@ -0,0 +1,10 @@
+/**:
+  ros__parameters:
+    num_anchors: 3
+    anchors: [81.0, 82.0, 135.0, 169.0, 344.0, 319.0, 23.0, 27.0, 37.0, 58.0, 81.0, 82.0]
+    scale_x_y: [1.05, 1.05]
+    score_threshold: 0.1
+    iou_thresh: 0.45
+    detections_per_im: 100
+    use_darknet_layer: true
+    ignore_thresh: 0.5