diff --git a/.bazelrc b/.bazelrc index bcc2a11042..f9e0b4ab07 100644 --- a/.bazelrc +++ b/.bazelrc @@ -22,7 +22,7 @@ # +------------------------------------------------------------+ # Enable colorful output of GCC build --cxxopt="-fdiagnostics-color=always" -build --cxxopt='-std=c++14' +build --cxxopt='-std=c++17' #build --linkopt="-Wl,--no-as-needed" diff --git a/.circleci/config.yml b/.circleci/config.yml index dab13dcb88..38c25a1c65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -269,10 +269,10 @@ commands: parameters: torch-build: type: string - default: "2.1.0.dev20230419+cu118" + default: "2.1.0.dev20230601+cu118" torchvision-build: type: string - default: "0.16.0.dev20230419+cu118" + default: "0.16.0.dev20230601+cu118" torch-build-index: type: string default: "https://download.pytorch.org/whl/nightly/cu118" @@ -1350,10 +1350,10 @@ parameters: # Nightly platform config torch-build: type: string - default: "2.1.0.dev20230419+cu118" + default: "2.1.0.dev20230601+cu118" torchvision-build: type: string - default: "0.16.0.dev20230419+cu118" + default: "0.16.0.dev20230601+cu118" torch-build-index: type: string default: "https://download.pytorch.org/whl/nightly/cu118" diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b103b8d86..1fdd9390e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.17) project(Torch-TensorRT LANGUAGES CXX) -# use c++14 like PyTorch -set(CMAKE_CXX_STANDARD 14) +# use c++17 like PyTorch +set(CMAKE_CXX_STANDARD 17) # Build the libraries with -fPIC set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/README.md b/README.md index f3ad4842de..ae586461e6 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,7 @@ torch.jit.save(trt_ts_module, "trt_torchscript_module.ts") # save the TRT embedd These are the following dependencies used to verify the testcases. Torch-TensorRT can work with other versions, but the tests are not guaranteed to pass. - Bazel 5.2.0 -- Libtorch 2.1.0.dev20230419 (built with CUDA 11.8) +- Libtorch 2.1.0.dev20230601 (built with CUDA 11.8) - CUDA 11.8 - cuDNN 8.8.0 - TensorRT 8.6.0 diff --git a/WORKSPACE b/WORKSPACE index 9aa75b4e17..ef5446bf2f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -51,17 +51,17 @@ new_local_repository( http_archive( name = "libtorch", build_file = "@//third_party/libtorch:BUILD", - sha256 = "1a526a9cd19c1015674d26921dbb94bcd2d632a6f9c431a21c43f4e24768d834", + sha256 = "c8407ae3462c344ae3814e82023e22ece759ebe75023f35bdf62e9c0a7e79035", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) http_archive( name = "libtorch_pre_cxx11_abi", build_file = "@//third_party/libtorch:BUILD", - sha256 = "60c5912a5085a6a7073b3804b10d41d6cc054693bbeb7a45e0247050c2837bac", + sha256 = "76f983bd6d784cc0a95c679034d297abe36911c16b2188498b13a9028177e28e", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) # Download these tarballs manually from the NVIDIA website diff --git a/examples/dynamo/dynamo_compile_advanced_usage.py b/examples/dynamo/dynamo_compile_advanced_usage.py index 4e1e67f816..fa9450f5cd 100644 --- a/examples/dynamo/dynamo_compile_advanced_usage.py +++ b/examples/dynamo/dynamo_compile_advanced_usage.py @@ -9,7 +9,6 @@ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ import torch -from torch_tensorrt.dynamo.backend import create_backend from torch_tensorrt.fx.lower_setting import LowerPrecision # %% @@ -39,9 +38,9 @@ def forward(self, x: torch.Tensor, y: torch.Tensor): # Next, we compile the model using torch.compile # For the default settings, we can simply call torch.compile -# with the backend "tensorrt", and run the model on an +# with the backend "torch_tensorrt", and run the model on an # input to cause compilation, as so: -optimized_model = torch.compile(model, backend="tensorrt") +optimized_model = torch.compile(model, backend="torch_tensorrt") optimized_model(*sample_inputs) # %% @@ -58,18 +57,23 @@ def forward(self, x: torch.Tensor, y: torch.Tensor): # %% # If we want to customize certain options in the backend, -# but still use the torch.compile call directly, we can call the -# convenience/helper function create_backend to create a custom backend -# which has been pre-populated with certain keys -custom_backend = create_backend( - lower_precision=LowerPrecision.FP16, - debug=True, - min_block_size=2, - torch_executed_ops={}, -) +# but still use the torch.compile call directly, we can provide +# custom options to the backend via the "options" keyword +# which takes in a dictionary mapping options to values. +# +# For accepted backend options, see the CompilationSettings dataclass: +# py/torch_tensorrt/dynamo/backend/_settings.py +backend_kwargs = { + "lower_precision": LowerPrecision.FP16, + "debug": True, + "min_block_size": 2, + "torch_executed_ops": {"torch.ops.aten.sub.Tensor"}, +} # Run the model on an input to cause compilation, as so: -optimized_model_custom = torch.compile(model_half, backend=custom_backend) +optimized_model_custom = torch.compile( + model_half, backend="torch_tensorrt", options=backend_kwargs +) optimized_model_custom(*sample_inputs_half) # %% diff --git a/py/requirements.txt b/py/requirements.txt index 03398014fb..9b9914930b 100644 --- a/py/requirements.txt +++ b/py/requirements.txt @@ -2,7 +2,7 @@ numpy packaging pybind11==2.6.2 --extra-index-url https://download.pytorch.org/whl/nightly/cu118 -torch==2.1.0.dev20230419+cu118 -torchvision==0.16.0.dev20230419+cu118 +torch==2.1.0.dev20230601+cu118 +torchvision==0.16.0.dev20230601+cu118 --extra-index-url https://pypi.ngc.nvidia.com tensorrt==8.6.0 diff --git a/py/torch_tensorrt/dynamo/backend/backends.py b/py/torch_tensorrt/dynamo/backend/backends.py index 9df3f1c686..81d8b72090 100644 --- a/py/torch_tensorrt/dynamo/backend/backends.py +++ b/py/torch_tensorrt/dynamo/backend/backends.py @@ -2,6 +2,7 @@ import torch import traceback from functools import partial +from dataclasses import replace, fields import torch._dynamo as td from torch_tensorrt.dynamo.backend._settings import CompilationSettings @@ -25,7 +26,15 @@ def torch_tensorrt_backend( gm: torch.fx.GraphModule, sample_inputs: Sequence[torch.Tensor], settings: CompilationSettings = CompilationSettings(), + **kwargs ): + # If the user specifies keyword args, overwrite those fields in settings + # Validate all specified kwargs to ensure they are true fields of the dataclass + if kwargs: + valid_attrs = {attr.name for attr in fields(settings)} + valid_kwargs = {k: v for k, v in kwargs.items() if k in valid_attrs} + settings = replace(settings, **valid_kwargs) + DEFAULT_BACKEND = aot_torch_tensorrt_aten_backend return DEFAULT_BACKEND(gm, sample_inputs, settings=settings) diff --git a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel index 263ed6d40b..296cda1093 100644 --- a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel +++ b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.rhel @@ -56,17 +56,17 @@ new_local_repository( http_archive( name = "libtorch", build_file = "@//third_party/libtorch:BUILD", - sha256 = "1a526a9cd19c1015674d26921dbb94bcd2d632a6f9c431a21c43f4e24768d834", + sha256 = "c8407ae3462c344ae3814e82023e22ece759ebe75023f35bdf62e9c0a7e79035", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) http_archive( name = "libtorch_pre_cxx11_abi", build_file = "@//third_party/libtorch:BUILD", - sha256 = "60c5912a5085a6a7073b3804b10d41d6cc054693bbeb7a45e0247050c2837bac", + sha256 = "76f983bd6d784cc0a95c679034d297abe36911c16b2188498b13a9028177e28e", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) #################################################################################### diff --git a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.ubuntu b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.ubuntu index 263ed6d40b..296cda1093 100644 --- a/toolchains/ci_workspaces/WORKSPACE.x86_64.release.ubuntu +++ b/toolchains/ci_workspaces/WORKSPACE.x86_64.release.ubuntu @@ -56,17 +56,17 @@ new_local_repository( http_archive( name = "libtorch", build_file = "@//third_party/libtorch:BUILD", - sha256 = "1a526a9cd19c1015674d26921dbb94bcd2d632a6f9c431a21c43f4e24768d834", + sha256 = "c8407ae3462c344ae3814e82023e22ece759ebe75023f35bdf62e9c0a7e79035", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-cxx11-abi-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) http_archive( name = "libtorch_pre_cxx11_abi", build_file = "@//third_party/libtorch:BUILD", - sha256 = "60c5912a5085a6a7073b3804b10d41d6cc054693bbeb7a45e0247050c2837bac", + sha256 = "76f983bd6d784cc0a95c679034d297abe36911c16b2188498b13a9028177e28e", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230419%2Bcu118.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu118/libtorch-shared-with-deps-2.1.0.dev20230601%2Bcu118.zip"], ) ####################################################################################