added qat-ptq workflow notebook

[tanayvarshney]

Description

Added notebook demonstrating the workflows for QAT & PTQ

Fixes # (issue)

Type of change

Please delete options that are not relevant and/or add your own.

• added new examples

Checklist:

• My code follows the style guidelines of this project (You can use the linters)
• I have performed a self-review of my own code
• I have commented my code, particularly in hard-to-understand areas and hacks

[github-actions]

There are some changes that do not conform to C++ style guidelines:

diff --git a/workspace/core/conversion/evaluators/aten.cpp b/tmp/changes.txt
index 4d8795f..c8c2c00 100644
--- a/workspace/core/conversion/evaluators/aten.cpp
+++ b/tmp/changes.txt
@@ -184,7 +184,7 @@ auto aten_registrations TORCHTRT_UNUSED =

                      int64_t start = 0;
                      auto startIVal = args.at(n->input(1)).IValue();
-                      if(!startIVal->isNone()){
+                      if (!startIVal->isNone()) {
                        start = args.at(n->input(1)).unwrapToInt();
                      }
                      int64_t end = args.at(n->input(2)).unwrapToInt();
diff --git a/workspace/core/conversion/converters/impl/unary.cpp b/tmp/changes.txt
index a1d03a3..6b0ee2b 100644
--- a/workspace/core/conversion/converters/impl/unary.cpp
+++ b/tmp/changes.txt
@@ -10,45 +10,41 @@ namespace converters {
namespace impl {
namespace {

-
auto abs_registration TORCHTRT_UNUSED = RegisterNodeConversionPatterns().pattern(
-  {"aten::abs (Tensor self) -> Tensor",
-  [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool {
-    auto in = args[0].ITensor();
-    bool unary_supported_input = in->getType() == nvinfer1::DataType::kFLOAT
-     || in->getType() == nvinfer1::DataType::kHALF
-     || in->getType() == nvinfer1::DataType::kINT8;
-    if(unary_supported_input){
-      auto unary_layer = ctx->net->addUnary(*in, nvinfer1::UnaryOperation::kABS);
-      TORCHTRT_CHECK(unary_layer, "Unable to create abs layer from node: " << *n);
-      unary_layer->setName(util::node_info(n).c_str());
-      auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], unary_layer->getOutput(0));
-      LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
-      return true;
-    }
-    else{
-      //For types not supported by kABS, use an elementwise implementation abs(x) = max(x, -1 * x)
-      at::Tensor neg_one = torch::full({1}, -1).to(util::TRTDataTypeToScalarType(in->getType()));
-      auto neg_one_const = tensor_to_const(ctx, neg_one);
-      auto neg_layer = add_elementwise(
+    {"aten::abs (Tensor self) -> Tensor", [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool {
+       auto in = args[0].ITensor();
+       bool unary_supported_input = in->getType() == nvinfer1::DataType::kFLOAT ||
+           in->getType() == nvinfer1::DataType::kHALF || in->getType() == nvinfer1::DataType::kINT8;
+       if (unary_supported_input) {
+         auto unary_layer = ctx->net->addUnary(*in, nvinfer1::UnaryOperation::kABS);
+         TORCHTRT_CHECK(unary_layer, "Unable to create abs layer from node: " << *n);
+         unary_layer->setName(util::node_info(n).c_str());
+         auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], unary_layer->getOutput(0));
+         LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
+         return true;
+       } else {
+         // For types not supported by kABS, use an elementwise implementation abs(x) = max(x, -1 * x)
+         at::Tensor neg_one = torch::full({1}, -1).to(util::TRTDataTypeToScalarType(in->getType()));
+         auto neg_one_const = tensor_to_const(ctx, neg_one);
+         auto neg_layer = add_elementwise(
             ctx,
             nvinfer1::ElementWiseOperation::kPROD,
             in,
             neg_one_const,
             util::node_info(n) + std::string("_Negation"));
-      TORCHTRT_CHECK(neg_layer, "Unable to create prod layer from node: " << *n);
-      auto max_layer = add_elementwise(
+         TORCHTRT_CHECK(neg_layer, "Unable to create prod layer from node: " << *n);
+         auto max_layer = add_elementwise(
             ctx,
             nvinfer1::ElementWiseOperation::kMAX,
             in,
             neg_layer->getOutput(0),
             util::node_info(n) + std::string("_Max"));
-      TORCHTRT_CHECK(max_layer, "Unable to create max layer from node: " << *n);
-      auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], max_layer->getOutput(0));
-      LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
-      return true;
-    }
-  }});
+         TORCHTRT_CHECK(max_layer, "Unable to create max layer from node: " << *n);
+         auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], max_layer->getOutput(0));
+         LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
+         return true;
+       }
+     }});

#define convert(unary, trt_type)                                                               \
  auto unary##_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns().pattern(       \
diff --git a/workspace/tests/core/conversion/converters/test_element_wise.cpp b/tmp/changes.txt
index 994fb25..540fa12 100644
--- a/workspace/tests/core/conversion/converters/test_element_wise.cpp
+++ b/tmp/changes.txt
@@ -27,8 +27,8 @@ void pointwise_test_helper(
  if (!singleInput) {
    torch_inputs.push_back(at::randint(1, 5, shape2, {at::kCUDA}));
  }
-  if(int_tensors){
-    for(size_t i = 0UL; i < torch_inputs.size(); ++i){
+  if (int_tensors) {
+    for (size_t i = 0UL; i < torch_inputs.size(); ++i) {
      torch_inputs[i] = torch_inputs[i].to(at::kInt);
    }
  }
diff --git a/workspace/tests/core/conversion/converters/test_unary.cpp b/tmp/changes.txt
index a7ab3bb..1d40c3c 100644
--- a/workspace/tests/core/conversion/converters/test_unary.cpp
+++ b/tmp/changes.txt
@@ -1,9 +1,9 @@
#include <string>
-#include "torch/torch.h"
#include "core/compiler.h"
#include "gtest/gtest.h"
#include "tests/util/util.h"
#include "torch/csrc/jit/ir/irparser.h"
+#include "torch/torch.h"

namespace {
std::string gen_test_graph(const std::string& unary) {
@@ -22,7 +22,7 @@ TEST(Converters, ATenAbsIntConvertsCorrectly) {

  auto in = at::tensor({-1, 1, -2, 2, -3, 3}, {at::kCUDA}).to(torch::kInt32);
  auto params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {});
-  auto jit_results = torch_tensorrt::tests::util::RunGraph(g, params, {in});  
+  auto jit_results = torch_tensorrt::tests::util::RunGraph(g, params, {in});

  in = at::clone(in);
  params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {});
diff --git a/workspace/tests/core/conversion/converters/test_select.cpp b/tmp/changes.txt
index 03b6bda..67b760a 100644
--- a/workspace/tests/core/conversion/converters/test_select.cpp
+++ b/tmp/changes.txt
@@ -376,7 +376,7 @@ TEST(Converters, ATenSliceListConvertsCorrectly) {
          %slice : Tensor[] = aten::slice(%list, %1, %2, %3)
          %out.1 : Tensor, %out.2 : Tensor = prim::ListUnpack(%slice)
          return (%out.1, %out.2))IR";
-  
+
  auto g = std::make_shared<torch::jit::Graph>();

  torch::jit::parseIR(graph, g.get());
ERROR: Some files do not conform to style guidelines

[tanayvarshney]

@narendasan

[narendasan]

The format changed? its now an ipynb.txt, is this intentional?

[tanayvarshney]

No, not intentional, thank you for flagging. Corrected it now.

[github-actions]

There are some changes that do not conform to C++ style guidelines:

diff --git a/workspace/core/conversion/evaluators/aten.cpp b/tmp/changes.txt
index 4d8795f..c8c2c00 100644
--- a/workspace/core/conversion/evaluators/aten.cpp
+++ b/tmp/changes.txt
@@ -184,7 +184,7 @@ auto aten_registrations TORCHTRT_UNUSED =

                      int64_t start = 0;
                      auto startIVal = args.at(n->input(1)).IValue();
-                      if(!startIVal->isNone()){
+                      if (!startIVal->isNone()) {
                        start = args.at(n->input(1)).unwrapToInt();
                      }
                      int64_t end = args.at(n->input(2)).unwrapToInt();
diff --git a/workspace/core/conversion/converters/impl/unary.cpp b/tmp/changes.txt
index a1d03a3..6b0ee2b 100644
--- a/workspace/core/conversion/converters/impl/unary.cpp
+++ b/tmp/changes.txt
@@ -10,45 +10,41 @@ namespace converters {
namespace impl {
namespace {

-
auto abs_registration TORCHTRT_UNUSED = RegisterNodeConversionPatterns().pattern(
-  {"aten::abs (Tensor self) -> Tensor",
-  [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool {
-    auto in = args[0].ITensor();
-    bool unary_supported_input = in->getType() == nvinfer1::DataType::kFLOAT
-     || in->getType() == nvinfer1::DataType::kHALF
-     || in->getType() == nvinfer1::DataType::kINT8;
-    if(unary_supported_input){
-      auto unary_layer = ctx->net->addUnary(*in, nvinfer1::UnaryOperation::kABS);
-      TORCHTRT_CHECK(unary_layer, "Unable to create abs layer from node: " << *n);
-      unary_layer->setName(util::node_info(n).c_str());
-      auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], unary_layer->getOutput(0));
-      LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
-      return true;
-    }
-    else{
-      //For types not supported by kABS, use an elementwise implementation abs(x) = max(x, -1 * x)
-      at::Tensor neg_one = torch::full({1}, -1).to(util::TRTDataTypeToScalarType(in->getType()));
-      auto neg_one_const = tensor_to_const(ctx, neg_one);
-      auto neg_layer = add_elementwise(
+    {"aten::abs (Tensor self) -> Tensor", [](ConversionCtx* ctx, const torch::jit::Node* n, args& args) -> bool {
+       auto in = args[0].ITensor();
+       bool unary_supported_input = in->getType() == nvinfer1::DataType::kFLOAT ||
+           in->getType() == nvinfer1::DataType::kHALF || in->getType() == nvinfer1::DataType::kINT8;
+       if (unary_supported_input) {
+         auto unary_layer = ctx->net->addUnary(*in, nvinfer1::UnaryOperation::kABS);
+         TORCHTRT_CHECK(unary_layer, "Unable to create abs layer from node: " << *n);
+         unary_layer->setName(util::node_info(n).c_str());
+         auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], unary_layer->getOutput(0));
+         LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
+         return true;
+       } else {
+         // For types not supported by kABS, use an elementwise implementation abs(x) = max(x, -1 * x)
+         at::Tensor neg_one = torch::full({1}, -1).to(util::TRTDataTypeToScalarType(in->getType()));
+         auto neg_one_const = tensor_to_const(ctx, neg_one);
+         auto neg_layer = add_elementwise(
             ctx,
             nvinfer1::ElementWiseOperation::kPROD,
             in,
             neg_one_const,
             util::node_info(n) + std::string("_Negation"));
-      TORCHTRT_CHECK(neg_layer, "Unable to create prod layer from node: " << *n);
-      auto max_layer = add_elementwise(
+         TORCHTRT_CHECK(neg_layer, "Unable to create prod layer from node: " << *n);
+         auto max_layer = add_elementwise(
             ctx,
             nvinfer1::ElementWiseOperation::kMAX,
             in,
             neg_layer->getOutput(0),
             util::node_info(n) + std::string("_Max"));
-      TORCHTRT_CHECK(max_layer, "Unable to create max layer from node: " << *n);
-      auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], max_layer->getOutput(0));
-      LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
-      return true;
-    }
-  }});
+         TORCHTRT_CHECK(max_layer, "Unable to create max layer from node: " << *n);
+         auto out_tensor = ctx->AssociateValueAndTensor(n->outputs()[0], max_layer->getOutput(0));
+         LOG_DEBUG("Output tensor shape: " << out_tensor->getDimensions());
+         return true;
+       }
+     }});

#define convert(unary, trt_type)                                                               \
  auto unary##_registrations TORCHTRT_UNUSED = RegisterNodeConversionPatterns().pattern(       \
diff --git a/workspace/tests/core/conversion/converters/test_element_wise.cpp b/tmp/changes.txt
index 994fb25..540fa12 100644
--- a/workspace/tests/core/conversion/converters/test_element_wise.cpp
+++ b/tmp/changes.txt
@@ -27,8 +27,8 @@ void pointwise_test_helper(
  if (!singleInput) {
    torch_inputs.push_back(at::randint(1, 5, shape2, {at::kCUDA}));
  }
-  if(int_tensors){
-    for(size_t i = 0UL; i < torch_inputs.size(); ++i){
+  if (int_tensors) {
+    for (size_t i = 0UL; i < torch_inputs.size(); ++i) {
      torch_inputs[i] = torch_inputs[i].to(at::kInt);
    }
  }
diff --git a/workspace/tests/core/conversion/converters/test_unary.cpp b/tmp/changes.txt
index a7ab3bb..1d40c3c 100644
--- a/workspace/tests/core/conversion/converters/test_unary.cpp
+++ b/tmp/changes.txt
@@ -1,9 +1,9 @@
#include <string>
-#include "torch/torch.h"
#include "core/compiler.h"
#include "gtest/gtest.h"
#include "tests/util/util.h"
#include "torch/csrc/jit/ir/irparser.h"
+#include "torch/torch.h"

namespace {
std::string gen_test_graph(const std::string& unary) {
@@ -22,7 +22,7 @@ TEST(Converters, ATenAbsIntConvertsCorrectly) {

  auto in = at::tensor({-1, 1, -2, 2, -3, 3}, {at::kCUDA}).to(torch::kInt32);
  auto params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {});
-  auto jit_results = torch_tensorrt::tests::util::RunGraph(g, params, {in});  
+  auto jit_results = torch_tensorrt::tests::util::RunGraph(g, params, {in});

  in = at::clone(in);
  params = torch_tensorrt::core::ir::get_static_params(g->inputs(), {});
diff --git a/workspace/tests/core/conversion/converters/test_select.cpp b/tmp/changes.txt
index 03b6bda..67b760a 100644
--- a/workspace/tests/core/conversion/converters/test_select.cpp
+++ b/tmp/changes.txt
@@ -376,7 +376,7 @@ TEST(Converters, ATenSliceListConvertsCorrectly) {
          %slice : Tensor[] = aten::slice(%list, %1, %2, %3)
          %out.1 : Tensor, %out.2 : Tensor = prim::ListUnpack(%slice)
          return (%out.1, %out.2))IR";
-  
+
  auto g = std::make_shared<torch::jit::Graph>();

  torch::jit::parseIR(graph, g.get());
ERROR: Some files do not conform to style guidelines