diff --git a/common/tvm_utility/design/tvm-utility-design.md b/common/tvm_utility/design/tvm-utility-design.md index 2e322261152c0..42f908edcdc86 100644 --- a/common/tvm_utility/design/tvm-utility-design.md +++ b/common/tvm_utility/design/tvm-utility-design.md @@ -30,6 +30,14 @@ int main() { } ``` +#### Version checking + +The `InferenceEngineTVM::version_check` function can be used to check the version of the neural network in use against the range of earliest to latest supported versions. + +The `InferenceEngineTVM` class holds the latest supported version, which needs to be updated when the targeted version changes; after having tested the effect of the version change on the packages dependent on this one. + +The earliest supported version depends on each package making use of the inference, and so should be defined (and maintained) in those packages. + #### Models Dependent packages are expected to use the `get_neural_network` cmake function from this package in order to get the compiled TVM models. diff --git a/common/tvm_utility/include/tvm_utility/pipeline.hpp b/common/tvm_utility/include/tvm_utility/pipeline.hpp index a6578ab885490..aacc2ea84059e 100644 --- a/common/tvm_utility/include/tvm_utility/pipeline.hpp +++ b/common/tvm_utility/include/tvm_utility/pipeline.hpp @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef TVM_UTILITY__PIPELINE_HPP_ +#define TVM_UTILITY__PIPELINE_HPP_ + #include +#include #include #include @@ -26,11 +30,21 @@ #include #include -#ifndef TVM_UTILITY__PIPELINE_HPP_ -#define TVM_UTILITY__PIPELINE_HPP_ +using autoware::common::types::char8_t; namespace tvm_utility { + +/** + * @brief Possible version status for a neural network. + */ +enum class Version { + OK, + Unknown, + Untested, + Unsupported, +}; + namespace pipeline { @@ -176,6 +190,7 @@ using NetworkNode = std::pair>; typedef struct { // Network info + std::array modelzoo_version; std::string network_name; std::string network_backend; @@ -294,12 +309,37 @@ class InferenceEngineTVM : public InferenceEngine return output_; } + /** + * @brief Get version information from the config structure and check if there is a mismatch + * between the supported version(s) and the actual version. + * @param[in] version_from Earliest supported model version. + * @return The version status. + */ + Version version_check(const std::array & version_from) const + { + auto x{config_.modelzoo_version[0]}; + auto y{config_.modelzoo_version[1]}; + Version ret{Version::OK}; + + if (x == 0) { + ret = Version::Unknown; + } else if (x > version_up_to[0] || (x == version_up_to[0] && y > version_up_to[1])) { + ret = Version::Untested; + } else if (x < version_from[0] || (x == version_from[0] && y < version_from[1])) { + ret = Version::Unsupported; + } + + return ret; + } + private: InferenceEngineTVMConfig config_; TVMArrayContainerVector output_; tvm::runtime::PackedFunc set_input; tvm::runtime::PackedFunc execute; tvm::runtime::PackedFunc get_output; + // Latest supported model version. + const std::array version_up_to{2, 1, 0}; }; } // namespace pipeline diff --git a/common/tvm_utility/package.xml b/common/tvm_utility/package.xml index e648a3bd47b59..ea2a6c8940f73 100644 --- a/common/tvm_utility/package.xml +++ b/common/tvm_utility/package.xml @@ -29,6 +29,7 @@ autoware_cmake ament_index_cpp + autoware_auto_common libopenblas-dev libopencv-dev tvm_vendor diff --git a/common/tvm_utility/test/yolo_v2_tiny/main.cpp b/common/tvm_utility/test/yolo_v2_tiny/main.cpp index cc96bfe8e465b..4ed77f0f97d83 100644 --- a/common/tvm_utility/test/yolo_v2_tiny/main.cpp +++ b/common/tvm_utility/test/yolo_v2_tiny/main.cpp @@ -1,4 +1,4 @@ -// Copyright 2021 Arm Limited and Contributors. +// Copyright 2021-2022 Arm Limited and Contributors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -245,6 +245,9 @@ TEST(PipelineExamples, SimplePipeline) tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP); + auto version_status = IE.version_check({2, 0, 0}); + EXPECT_NE(version_status, tvm_utility::Version::Unsupported); + // Push data input the pipeline and get the output auto output = pipeline.schedule(IMAGE_FILENAME); diff --git a/common/tvm_utility/tvm_utility-extras.cmake b/common/tvm_utility/tvm_utility-extras.cmake index f8c1583b28f8c..169e44f5f342e 100644 --- a/common/tvm_utility/tvm_utility-extras.cmake +++ b/common/tvm_utility/tvm_utility-extras.cmake @@ -14,7 +14,7 @@ # Get user-provided variables set(DOWNLOAD_ARTIFACTS OFF CACHE BOOL "enable artifacts download") -set(MODELZOO_VERSION "1.3.0-20220902" CACHE STRING "targeted ModelZoo version") +set(MODELZOO_VERSION "2.1.0-20221102" CACHE STRING "targeted ModelZoo version") # # Download the selected neural network if it is not already present on disk. diff --git a/perception/lidar_apollo_segmentation_tvm/include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp b/perception/lidar_apollo_segmentation_tvm/include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp index af8fc6eaf1b70..b1f8b55614e21 100644 --- a/perception/lidar_apollo_segmentation_tvm/include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp +++ b/perception/lidar_apollo_segmentation_tvm/include/lidar_apollo_segmentation_tvm/lidar_apollo_segmentation_tvm.hpp @@ -158,6 +158,14 @@ class LIDAR_APOLLO_SEGMENTATION_TVM_PUBLIC ApolloLidarSegmentation std::shared_ptr detectDynamicObjects( const sensor_msgs::msg::PointCloud2 & input); + /// \brief Get the name of the neural network used. + /// \return The name. + const std::string & network_name() const; + + /// \brief Check the model's version against supported versions. + /// \return The version status. + tvm_utility::Version version_check() const; + private: const int32_t range_; const float32_t score_threshold_; @@ -166,6 +174,8 @@ class LIDAR_APOLLO_SEGMENTATION_TVM_PUBLIC ApolloLidarSegmentation const int32_t min_pts_num_; const float32_t height_thresh_; const pcl::PointCloud::Ptr pcl_pointcloud_ptr_; + // Earliest supported model version. + const std::array model_version_from{2, 0, 0}; // Pipeline using PrePT = ApolloLidarSegmentationPreProcessor; diff --git a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp index 6afa123b9317e..58ceb7897ea2f 100644 --- a/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp +++ b/perception/lidar_apollo_segmentation_tvm/src/lidar_apollo_segmentation_tvm.cpp @@ -19,6 +19,7 @@ #include #include +#include #include using autoware::common::types::bool8_t; @@ -192,6 +193,13 @@ std::shared_ptr ApolloLidarSegmentation::detec return output; } + +const std::string & ApolloLidarSegmentation::network_name() const { return config.network_name; } + +tvm_utility::Version ApolloLidarSegmentation::version_check() const +{ + return IE->version_check(model_version_from); +} } // namespace lidar_apollo_segmentation_tvm } // namespace perception } // namespace autoware diff --git a/perception/lidar_apollo_segmentation_tvm/test/main.cpp b/perception/lidar_apollo_segmentation_tvm/test/main.cpp index 0306cddf0b8ed..b5ba33743c6c9 100644 --- a/perception/lidar_apollo_segmentation_tvm/test/main.cpp +++ b/perception/lidar_apollo_segmentation_tvm/test/main.cpp @@ -44,6 +44,9 @@ void test_segmentation(bool use_intensity_feature, bool use_constant_feature, bo range, score_threshold, use_intensity_feature, use_constant_feature, z_offset, min_height, max_height, objectness_thresh, min_pts_num, height_thresh); + auto version_status = segmentation.version_check(); + EXPECT_NE(version_status, tvm_utility::Version::Unsupported); + std::random_device rd; std::mt19937 gen(42); std::uniform_real_distribution dis(-50.0, 50.0); diff --git a/perception/lidar_apollo_segmentation_tvm_nodes/src/lidar_apollo_segmentation_tvm_node.cpp b/perception/lidar_apollo_segmentation_tvm_nodes/src/lidar_apollo_segmentation_tvm_node.cpp index e6b6cd0025409..d96eaf9cf084c 100644 --- a/perception/lidar_apollo_segmentation_tvm_nodes/src/lidar_apollo_segmentation_tvm_node.cpp +++ b/perception/lidar_apollo_segmentation_tvm_nodes/src/lidar_apollo_segmentation_tvm_node.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,21 @@ ApolloLidarSegmentationNode::ApolloLidarSegmentationNode(const rclcpp::NodeOptio declare_parameter("min_pts_num", rclcpp::ParameterValue{3}).get(), declare_parameter("height_thresh", rclcpp::ParameterValue{0.5}).get())} { + // Log unexpected versions of the neural network. + auto version_status = m_detector_ptr->version_check(); + if (version_status != tvm_utility::Version::OK) { + auto network_name = m_detector_ptr->network_name(); + if (version_status == tvm_utility::Version::Unknown) { + RCLCPP_INFO( + get_logger(), "The '%s' network doesn't provide a version number.", network_name.c_str()); + } else if (version_status == tvm_utility::Version::Untested) { + RCLCPP_WARN( + get_logger(), "The version of the '%s' network is untested.", network_name.c_str()); + } else if (version_status == tvm_utility::Version::Unsupported) { + RCLCPP_ERROR( + get_logger(), "The version of the '%s' network is unsupported.", network_name.c_str()); + } + } } void ApolloLidarSegmentationNode::pointCloudCallback(