[ETHOSN] Update to 20.08 version of the ethosn-driver. (apache#6606)

- Updated ethosn relay backend to account for 20.08 api changes around cascading and quantization.
   - Note: 20.05 compatibility is maintained for now to avoid compilation and test failures while the docker image still uses 20.05. The version switch can be removed along with associated compatibility measures when no longer necessary.

docker/install/ubuntu_install_ethosn_driver_stack.sh

@@ -22,7 +22,7 @@ set -o pipefail
 
 repo_url="https://github.com/Arm-software/ethos-n-driver-stack"
 repo_dir="ethosn-driver"
-repo_revision="20.05"
+repo_revision="20.08"
 install_path="/opt/arm/$repo_dir"
 
 tmpdir=$(mktemp -d)

src/relay/backend/contrib/ethosn/capabilities.h

@@ -31,6 +31,8 @@
 
 #include <vector>
 
+#include "ethosn_api_version.h"
+
 namespace tvm {
 namespace relay {
 namespace contrib {
@@ -41,6 +43,39 @@ namespace ethosn {
  * variant[1] - N57
  * variant[2] - N37
  */
+#if _ETHOSN_API_VERSION_ == 2008
+static std::vector<char> variants[3] = {
+    {
+        0x03, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00,
+        0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
+        0x10, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00,
+        0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    },
+    {
+        0x03, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00,
+        0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
+        0x10, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00,
+        0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    },
+    {
+        0x03, 0x00, 0x00, 0x00, 0x78, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x10, 0x00,
+        0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
+        0x10, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
+        0x01, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+        0x00, 0x01, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x01, 0x00,
+        0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    }};
+#else
 static std::vector<char> variants[3] = {
     {
         0x02, 0x00, 0x00, 0x00, 0x74, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
@@ -72,7 +107,7 @@ static std::vector<char> variants[3] = {
         0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x01,
         0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
     }};
-
+#endif
 }  // namespace ethosn
 }  // namespace contrib
 }  // namespace relay

src/relay/backend/contrib/ethosn/codegen.cc

@@ -578,7 +578,12 @@ sl::CompilationOptions EthosnCompiler::CreateOptions() {
   options.m_DisableWinograd = cfg.value()->disable_winograd;
   options.m_DebugInfo.m_DumpDebugFiles = cfg.value()->dump_debug_files;
   options.m_DebugInfo.m_DebugDir = cfg.value()->debug_dir;
+#if _ETHOSN_API_VERSION_ == 2008
+  options.m_CompilerAlgorithm =
+      sl::EthosNCompilerAlgorithmFromString(cfg.value()->compiler_algorithm.c_str());
+#else
   options.m_EnableCascading = cfg.value()->enable_cascading;
+#endif
   return options;
 }
 

src/relay/backend/contrib/ethosn/codegen_ethosn.h

@@ -46,6 +46,7 @@
 #include "../../../../runtime/contrib/ethosn/ethosn_runtime.h"
 #include "../codegen_c/codegen_c.h"
 #include "ethosn_api.h"
+#include "ethosn_api_version.h"
 #include "ethosn_support_library/Support.hpp"
 #include "ethosn_support_library/SupportQueries.hpp"
 
@@ -242,7 +243,11 @@ struct EthosnCompilerConfigNode : public tvm::AttrsNode<EthosnCompilerConfigNode
   bool disable_winograd;
   bool dump_debug_files;
   String debug_dir;
+#if _ETHOSN_API_VERSION_ == 2008
+  String compiler_algorithm;
+#else
   bool enable_cascading;
+#endif
 
   TVM_DECLARE_ATTRS(EthosnCompilerConfigNode, "ext.attrs.EthosnCompilerConfigNode") {
     TVM_ATTR_FIELD(variant)
@@ -264,7 +269,11 @@ struct EthosnCompilerConfigNode : public tvm::AttrsNode<EthosnCompilerConfigNode
     TVM_ATTR_FIELD(disable_winograd).set_default(false);
     TVM_ATTR_FIELD(dump_debug_files).set_default(false);
     TVM_ATTR_FIELD(debug_dir).set_default(".");
+#if _ETHOSN_API_VERSION_ == 2008
+    TVM_ATTR_FIELD(compiler_algorithm).set_default("NonCascadingOnly");
+#else
     TVM_ATTR_FIELD(enable_cascading).set_default(false);
+#endif
   }
 };
 

src/relay/backend/contrib/ethosn/ethosn_api.cc

@@ -32,6 +32,7 @@
 #include <utility>
 #include <vector>
 
+#include "ethosn_api_version.h"
 #include "ethosn_support_library/Support.hpp"
 #include "ethosn_support_library/SupportQueries.hpp"
 
@@ -71,7 +72,11 @@ EthosnError EthosnAPI::QnnConv2d(const Expr& expr, ConvolutionParams* params) {
   sl::QuantizationInfo output_q_info;
   err += Tvm2Npu(input_zero_point, input_scale, &data_q_info);
   err += Tvm2Npu(kernel_zero_point, kernel_scale, &weights_q_info);
+#if _ETHOSN_API_VERSION_ == 2008
+  err += Tvm2Npu(0, data_q_info.GetScale() * weights_q_info.GetScale(), &bias_q_info);
+#else
   err += Tvm2Npu(0, data_q_info.m_Scale * weights_q_info.m_Scale, &bias_q_info);
+#endif
   err += Tvm2Npu(output_zero_point, output_scale, &output_q_info);
 
   // Convert convolution attributes
@@ -165,7 +170,11 @@ EthosnError EthosnAPI::QnnFullyConnected(const Expr& expr, FullyConnectedParams*
   sl::QuantizationInfo output_q_info;
   err += Tvm2Npu(input_zero_point, input_scale, &data_q_info);
   err += Tvm2Npu(kernel_zero_point, kernel_scale, &weights_q_info);
+#if _ETHOSN_API_VERSION_ == 2008
+  err += Tvm2Npu(0, data_q_info.GetScale() * weights_q_info.GetScale(), &bias_q_info);
+#else
   err += Tvm2Npu(0, data_q_info.m_Scale * weights_q_info.m_Scale, &bias_q_info);
+#endif
   err += Tvm2Npu(output_zero_point, output_scale, &output_q_info);
 
   // Create fc info
@@ -724,6 +733,10 @@ TVM_REGISTER_GLOBAL("relay.ethos-n.query").set_body([](tvm::TVMArgs args, tvm::T
 #endif
 });
 
+TVM_REGISTER_GLOBAL("relay.ethos-n.api.version").set_body_typed([]() -> int {
+  return _ETHOSN_API_VERSION_;
+});
+
 }  // namespace ethosn
 }  // namespace contrib
 }  // namespace relay

src/relay/backend/contrib/ethosn/ethosn_api_version.h

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "ethosn_support_library/Support.hpp"
+
+#ifndef TVM_RELAY_BACKEND_CONTRIB_ETHOSN_ETHOSN_API_VERSION_H_
+#define TVM_RELAY_BACKEND_CONTRIB_ETHOSN_ETHOSN_API_VERSION_H_
+
+/*!
+ * \brief To be used as a temperory switch to ensure
+ * compatibility with the previous version of the api
+ * while needed e.g. by docker images. Can be removed
+ * along with associated compatibility measures when no
+ * longer necessary.
+ */
+#define _ETHOSN_API_VERSION_ 2008
+#ifndef COMPILER_ALGORITHM_MODE
+#undef _ETHOSN_API_VERSION_
+#define _ETHOSN_API_VERSION_ 2005
+#endif
+
+#endif  // TVM_RELAY_BACKEND_CONTRIB_ETHOSN_ETHOSN_API_VERSION_H_

tests/python/contrib/test_ethosn/infrastructure.py

@@ -304,3 +304,7 @@ def get_conv2d_qnn_params(input_zp, input_sc, kernel_zp, kernel_sc, kernel_h, ke
     output_sc = (output_max - output_min) / 255
     output_zp = -int(output_min / output_sc)
     return output_zp, output_sc
+
+
+def get_ethosn_api_version():
+    return tvm.get_global_func("relay.ethos-n.api.version")()

tests/python/contrib/test_ethosn/test_conv2d.py

@@ -185,6 +185,14 @@ def test_conv2d_failure():
     if not ethosn_available():
         return
 
+    _scale_error_msg = (
+        "Overall scale (of the input * weights / output) should be in the range [0, 1)"
+    )
+    if tei.get_ethosn_api_version() == 2008:
+        _scale_error_msg = (
+            "Overall scale (of the input * weights / output) should be in the range [0, 1}"
+        )
+
     trials = [
         (
             (1, 4, 4, 4),
@@ -203,7 +211,7 @@ def test_conv2d_failure():
             "uint8",
             8,
             "HWIO",
-            "Overall scale (of the input * weights / output) should be in the range [0, 1)",
+            _scale_error_msg,
         ),
         (
             (1, 4, 4, 4),

tests/python/contrib/test_ethosn/test_networks.py

@@ -122,12 +122,15 @@ def test_mobilenet_v1():
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
+    _compile_hash = {"81637c89339201a07dc96e3b5dbf836a"}
+    if tei.get_ethosn_api_version() == 2008:
+        _compile_hash = {"47e216d8ab2bf491708ccf5620bc0d02"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224_quant.tgz",
         model_sub_path="mobilenet_v1_1.0_224_quant.tflite",
         input_dict={"input": (1, 224, 224, 3)},
-        compile_hash="81637c89339201a07dc96e3b5dbf836a",
+        compile_hash=_compile_hash,
         output_count=1,
         host_ops=3,
         npu_partitions=1,
@@ -141,12 +144,15 @@ def test_inception_v3():
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
+    _compile_hash = {"de0e175af610ebd45ccb03d170dc9664"}
+    if tei.get_ethosn_api_version() == 2008:
+        _compile_hash = {"8c9d75659cd7bc9ff6dd6d490d28f9b2"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite_11_05_08/inception_v3_quant.tgz",
         model_sub_path="inception_v3_quant.tflite",
         input_dict={"input": (1, 299, 299, 3)},
-        compile_hash="de0e175af610ebd45ccb03d170dc9664",
+        compile_hash=_compile_hash,
         output_count=1,
         host_ops=0,
         npu_partitions=1,
@@ -159,12 +165,15 @@ def test_inception_v4():
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
+    _compile_hash = {"06bf6cb56344f3904bcb108e54edfe87"}
+    if tei.get_ethosn_api_version() == 2008:
+        _compile_hash = {"798292bfa596ca7c32086396b494b46c"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/inception_v4_299_quant_20181026.tgz",
         model_sub_path="inception_v4_299_quant.tflite",
         input_dict={"input": (1, 299, 299, 3)},
-        compile_hash="06bf6cb56344f3904bcb108e54edfe87",
+        compile_hash=_compile_hash,
         output_count=1,
         host_ops=3,
         npu_partitions=1,
@@ -177,12 +186,15 @@ def test_ssd_mobilenet_v1():
     # codegen, which could come about from either a change in Support Library
     # version or a change in the Ethos-N codegen. To update this requires running
     # on hardware that isn't available in CI.
+    _compile_hash = {"29aec6b184b09454b4323271aadf89b1", "6211d96103880b016baa85e638abddef"}
+    if tei.get_ethosn_api_version() == 2008:
+        _compile_hash = {"5999f26e140dee0d7866491997ef78c5", "24e3a690a7e95780052792d5626c85be"}
     _test_image_network(
         model_url="https://storage.googleapis.com/download.tensorflow.org/"
         "models/tflite/coco_ssd_mobilenet_v1_1.0_quant_2018_06_29.zip",
         model_sub_path="detect.tflite",
         input_dict={"normalized_input_image_tensor": (1, 300, 300, 3)},
-        compile_hash={"29aec6b184b09454b4323271aadf89b1", "6211d96103880b016baa85e638abddef"},
+        compile_hash=_compile_hash,
         output_count=4,
         host_ops=28,
         npu_partitions=2,