diff --git a/.github/scripts/generate-release-matrix.py b/.github/scripts/generate-release-matrix.py index e1b276184c..2a84d2bfb2 100644 --- a/.github/scripts/generate-release-matrix.py +++ b/.github/scripts/generate-release-matrix.py @@ -5,16 +5,16 @@ import sys RELEASE_CUDA_VERSION = { - "wheel": ["cu124"], - "tarball": ["cu124"], + "wheel": ["cu128"], + "tarball": ["cu128"], } RELEASE_PYTHON_VERSION = { - "wheel": ["3.8", "3.9", "3.10", "3.11", "3.12"], - "tarball": ["3.10"], + "wheel": ["3.9", "3.10", "3.11", "3.12"], + "tarball": ["3.11"], } CXX11_TARBALL_CONTAINER_IMAGE = { - "cu124": "pytorch/libtorch-cxx11-builder:cuda12.4-main", + "cu128": "pytorch/libtorch-cxx11-builder:cuda12.8-main", } diff --git a/.github/scripts/generate-tensorrt-test-matrix.py b/.github/scripts/generate-tensorrt-test-matrix.py index 7ff90c7ffd..251e4c94fd 100644 --- a/.github/scripts/generate-tensorrt-test-matrix.py +++ b/.github/scripts/generate-tensorrt-test-matrix.py @@ -11,16 +11,16 @@ # channel: nightly if the future tensorRT version test workflow is triggered from the main branch or your personal branch # channel: test if the future tensorRT version test workflow is triggered from the release branch(release/2.5 etc....) CUDA_VERSIONS_DICT = { - "nightly": ["cu126"], - "test": ["cu124", "cu126"], - "release": ["cu124", "cu126"], + "nightly": ["cu128"], + "test": ["cu118", "cu126", "cu128"], + "release": ["cu118", "cu126", "cu128"], } # please update the python version you want to test with the future tensorRT version here # channel: nightly if the future tensorRT version test workflow is triggered from the main branch or your personal branch # channel: test if the future tensorRT version test workflow is triggered from the release branch(release/2.5 etc....) PYTHON_VERSIONS_DICT = { - "nightly": ["3.9"], + "nightly": ["3.11"], "test": ["3.9", "3.10", "3.11", "3.12"], "release": ["3.9", "3.10", "3.11", "3.12"], } @@ -28,14 +28,6 @@ # please update the future tensorRT version you want to test here TENSORRT_VERSIONS_DICT = { "windows": { - "10.4.0": { - "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.4.0/zip/TensorRT-10.4.0.26.Windows.win10.cuda-12.6.zip", - "strip_prefix": "TensorRT-10.4.0.26", - }, - "10.5.0": { - "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.5.0/zip/TensorRT-10.5.0.18.Windows.win10.cuda-12.6.zip", - "strip_prefix": "TensorRT-10.5.0.18", - }, "10.6.0": { "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.6.0/zip/TensorRT-10.6.0.26.Windows.win10.cuda-12.6.zip", "strip_prefix": "TensorRT-10.6.0.26", @@ -44,16 +36,12 @@ "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip", "strip_prefix": "TensorRT-10.7.0.23", }, + "10.8.0": { + "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip", + "strip_prefix": "TensorRT-10.8.0.43", + }, }, "linux": { - "10.4.0": { - "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.4.0/tars/TensorRT-10.4.0.26.Linux.x86_64-gnu.cuda-12.6.tar.gz", - "strip_prefix": "TensorRT-10.4.0.26", - }, - "10.5.0": { - "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.5.0/tars/TensorRT-10.5.0.18.Linux.x86_64-gnu.cuda-12.6.tar.gz", - "strip_prefix": "TensorRT-10.5.0.18", - }, "10.6.0": { "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.6.0/tars/TensorRT-10.6.0.26.Linux.x86_64-gnu.cuda-12.6.tar.gz", "strip_prefix": "TensorRT-10.6.0.26", @@ -62,6 +50,10 @@ "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz", "strip_prefix": "TensorRT-10.7.0.23", }, + "10.8.0": { + "urls": "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz", + "strip_prefix": "TensorRT-10.8.0.43", + }, }, } @@ -87,7 +79,7 @@ def check_file_availability(url: str) -> bool: # calculate the next minor version minor = int(list(TENSORRT_VERSIONS_DICT["linux"].keys())[-1].split(".")[1]) + 1 trt_version = f"{major}.{minor}.0" - for patch in range(patch_from, 50): + for patch in range(patch_from, 80): for cuda_minor in range(4, 11): trt_linux_release_url_candidate = f"https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/{trt_version}/tars/TensorRT-{trt_version}.{patch}.Linux.x86_64-gnu.cuda-12.{cuda_minor}.tar.gz" if check_file_availability(trt_linux_release_url_candidate): diff --git a/.github/scripts/generate_binary_build_matrix.py b/.github/scripts/generate_binary_build_matrix.py index abbe745ad8..61a7894dd9 100644 --- a/.github/scripts/generate_binary_build_matrix.py +++ b/.github/scripts/generate_binary_build_matrix.py @@ -24,9 +24,9 @@ "release": ["3.9", "3.10", "3.11", "3.12"], } CUDA_ARCHES_DICT = { - "nightly": ["11.8", "12.4", "12.6"], - "test": ["11.8", "12.1", "12.4"], - "release": ["11.8", "12.1", "12.4"], + "nightly": ["11.8", "12.6", "12.8"], + "test": ["11.8", "12.6", "12.8"], + "release": ["11.8", "12.6", "12.8"], } ROCM_ARCHES_DICT = { "nightly": ["6.1", "6.2"], @@ -34,13 +34,6 @@ "release": ["6.1", "6.2"], } -CUDA_CUDDN_VERSIONS = { - "11.8": {"cuda": "11.8.0", "cudnn": "9"}, - "12.1": {"cuda": "12.1.1", "cudnn": "9"}, - "12.4": {"cuda": "12.4.1", "cudnn": "9"}, - "12.6": {"cuda": "12.6.2", "cudnn": "9"}, -} - PACKAGE_TYPES = ["wheel", "conda", "libtorch"] PRE_CXX11_ABI = "pre-cxx11" CXX11_ABI = "cxx11-abi" @@ -151,6 +144,7 @@ def initialize_globals(channel: str, build_python_only: bool) -> None: "12.1": "pytorch/manylinux2_28-builder:cuda12.1", "12.4": "pytorch/manylinux2_28-builder:cuda12.4", "12.6": "pytorch/manylinux2_28-builder:cuda12.6", + "12.8": "pytorch/manylinux2_28-builder:cuda12.8", **{ gpu_arch: f"pytorch/manylinux2_28-builder:rocm{gpu_arch}" for gpu_arch in ROCM_ARCHES @@ -278,7 +272,7 @@ def get_wheel_install_command( return f"{WHL_INSTALL_BASE} {PACKAGES_TO_INSTALL_WHL} --index-url {get_base_download_url_for_repo('whl', channel, gpu_arch_type, desired_cuda)}_pypi_pkg" # noqa: E501 else: raise ValueError( - "Split build is not supported for this configuration. It is only supported for CUDA 11.8, 12.4, 12.6 on Linux nightly builds." # noqa: E501 + "Split build is not supported for this configuration. It is only supported for CUDA 11.8, 12.4, 12.6, 12.8 on Linux nightly builds." # noqa: E501 ) if ( channel == RELEASE diff --git a/.github/workflows/build-test-linux.yml b/.github/workflows/build-test-linux.yml index a6aaac4132..32ff30b818 100644 --- a/.github/workflows/build-test-linux.yml +++ b/.github/workflows/build-test-linux.yml @@ -23,6 +23,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' filter-matrix: needs: [generate-matrix] @@ -32,7 +33,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt diff --git a/.github/workflows/build-test-tensorrt-linux.yml b/.github/workflows/build-test-tensorrt-linux.yml index f25e96c5d6..6ddd0e336d 100644 --- a/.github/workflows/build-test-tensorrt-linux.yml +++ b/.github/workflows/build-test-tensorrt-linux.yml @@ -20,6 +20,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' generate-tensorrt-matrix: needs: [generate-matrix] @@ -29,7 +30,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt diff --git a/.github/workflows/build-test-tensorrt-windows.yml b/.github/workflows/build-test-tensorrt-windows.yml index 6df471184c..bcb2d8865b 100644 --- a/.github/workflows/build-test-tensorrt-windows.yml +++ b/.github/workflows/build-test-tensorrt-windows.yml @@ -20,6 +20,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' generate-tensorrt-matrix: needs: [generate-matrix] @@ -29,7 +30,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt diff --git a/.github/workflows/build-test-windows.yml b/.github/workflows/build-test-windows.yml index 2ee31b4b74..95d3912f3e 100644 --- a/.github/workflows/build-test-windows.yml +++ b/.github/workflows/build-test-windows.yml @@ -23,6 +23,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' substitute-runner: needs: generate-matrix diff --git a/.github/workflows/docgen.yml b/.github/workflows/docgen.yml index 5805a563f5..b21a4ffc87 100644 --- a/.github/workflows/docgen.yml +++ b/.github/workflows/docgen.yml @@ -14,12 +14,12 @@ jobs: if: ${{ ! contains(github.actor, 'pytorchbot') }} environment: pytorchbot-env container: - image: docker.io/pytorch/manylinux2_28-builder:cuda12.6 + image: docker.io/pytorch/manylinux2_28-builder:cuda12.8 options: --gpus all env: - CUDA_HOME: /usr/local/cuda-12.6 - VERSION_SUFFIX: cu126 - CU_VERSION: cu126 + CUDA_HOME: /usr/local/cuda-12.8 + VERSION_SUFFIX: cu128 + CU_VERSION: cu128 CHANNEL: nightly CI_BUILD: 1 steps: diff --git a/.github/workflows/generate_binary_build_matrix.yml b/.github/workflows/generate_binary_build_matrix.yml index 11281ece94..fceb8cf1ee 100644 --- a/.github/workflows/generate_binary_build_matrix.yml +++ b/.github/workflows/generate_binary_build_matrix.yml @@ -72,7 +72,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - name: Checkout test-infra repository uses: actions/checkout@v4 with: diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml index 25454d27e2..c7e41a2d8f 100644 --- a/.github/workflows/release-linux.yml +++ b/.github/workflows/release-linux.yml @@ -24,6 +24,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' generate-release-tarball-matrix: needs: [generate-matrix] @@ -33,7 +34,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt @@ -83,7 +84,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt diff --git a/.github/workflows/release-wheel-linux.yml b/.github/workflows/release-wheel-linux.yml index 54732378eb..43034e6e8c 100644 --- a/.github/workflows/release-wheel-linux.yml +++ b/.github/workflows/release-wheel-linux.yml @@ -241,7 +241,7 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: ${{ inputs.repository }}/release/wheel/ - name: Upload pre-cxx11 tarball to GitHub - if: ${{ inputs.cxx11-tarball-release != 'true' && env.PYTHON_VERSION == '3.10' }} + if: ${{ inputs.cxx11-tarball-release != 'true' && env.PYTHON_VERSION == '3.11' }} continue-on-error: true uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index c6518e1e2a..52a47b38e8 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -24,6 +24,7 @@ jobs: test-infra-ref: main with-rocm: false with-cpu: false + python-versions: '["3.11"]' generate-release-matrix: needs: [generate-matrix] @@ -33,7 +34,7 @@ jobs: steps: - uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.11' - uses: actions/checkout@v4 with: repository: pytorch/tensorrt diff --git a/MODULE.bazel b/MODULE.bazel index f2b394c12c..01dc6916f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -36,13 +36,13 @@ new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local. new_local_repository( name = "cuda", build_file = "@//third_party/cuda:BUILD", - path = "/usr/local/cuda-12.6/", + path = "/usr/local/cuda-12.8/", ) new_local_repository( name = "cuda_win", build_file = "@//third_party/cuda:BUILD", - path = "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.6/", + path = "C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v12.8/", ) http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") @@ -55,21 +55,21 @@ http_archive( name = "libtorch", build_file = "@//third_party/libtorch:BUILD", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu126/libtorch-cxx11-abi-shared-with-deps-latest.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-cxx11-abi-shared-with-deps-latest.zip"], ) http_archive( name = "libtorch_pre_cxx11_abi", build_file = "@//third_party/libtorch:BUILD", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu126/libtorch-shared-with-deps-latest.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-shared-with-deps-latest.zip"], ) http_archive( name = "libtorch_win", build_file = "@//third_party/libtorch:BUILD", strip_prefix = "libtorch", - urls = ["https://download.pytorch.org/libtorch/nightly/cu126/libtorch-win-shared-with-deps-latest.zip"], + urls = ["https://download.pytorch.org/libtorch/nightly/cu128/libtorch-win-shared-with-deps-latest.zip"], ) # Download these tarballs manually from the NVIDIA website @@ -79,18 +79,18 @@ http_archive( http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", - strip_prefix = "TensorRT-10.7.0.23", + strip_prefix = "TensorRT-10.8.0.43", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz", ], ) http_archive( name = "tensorrt_win", build_file = "@//third_party/tensorrt/archive:BUILD", - strip_prefix = "TensorRT-10.7.0.23", + strip_prefix = "TensorRT-10.8.0.43", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip", ], ) diff --git a/README.md b/README.md index 7312ecbd5a..8ab55af8ca 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Torch-TensorRT [![Documentation](https://img.shields.io/badge/docs-master-brightgreen)](https://nvidia.github.io/Torch-TensorRT/) [![pytorch](https://img.shields.io/badge/PyTorch-2.4-green)](https://www.python.org/downloads/release/python-31013/) [![cuda](https://img.shields.io/badge/CUDA-12.4-green)](https://developer.nvidia.com/cuda-downloads) -[![trt](https://img.shields.io/badge/TensorRT-10.7.0-green)](https://github.com/nvidia/tensorrt-llm) +[![trt](https://img.shields.io/badge/TensorRT-10.8.0-green)](https://github.com/nvidia/tensorrt-llm) [![license](https://img.shields.io/badge/license-BSD--3--Clause-blue)](./LICENSE) [![linux_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-linux.yml) [![windows_tests](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml/badge.svg)](https://github.com/pytorch/TensorRT/actions/workflows/build-test-windows.yml) @@ -117,9 +117,9 @@ auto results = trt_mod.forward({input_tensor}); These are the following dependencies used to verify the testcases. Torch-TensorRT can work with other versions, but the tests are not guaranteed to pass. - Bazel 6.3.2 -- Libtorch 2.5.0.dev (latest nightly) (built with CUDA 12.4) -- CUDA 12.4 -- TensorRT 10.7.0.23 +- Libtorch 2.7.0.dev (latest nightly) (built with CUDA 12.8) +- CUDA 12.8 +- TensorRT 10.8.0.43 ## Deprecation Policy diff --git a/dev_dep_versions.yml b/dev_dep_versions.yml index 6ba35b5ef9..ab2abc8bcf 100644 --- a/dev_dep_versions.yml +++ b/dev_dep_versions.yml @@ -1,2 +1,2 @@ -__cuda_version__: "12.6" -__tensorrt_version__: "10.7.0.post1" +__cuda_version__: "12.8" +__tensorrt_version__: "10.8.0" diff --git a/docker/Dockerfile b/docker/Dockerfile index 3fff741f11..9efc5f16c7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,15 +1,16 @@ # syntax=docker/dockerfile:1 # Base image starts with CUDA -ARG BASE_IMG=nvidia/cuda:12.4.1-devel-ubuntu22.04 +#TODO: cuda version +ARG BASE_IMG=nvidia/cuda:12.8.0-devel-ubuntu22.04 FROM ${BASE_IMG} as base -ENV BASE_IMG=nvidia/cuda:12.4.1-devel-ubuntu22.04 +ENV BASE_IMG=nvidia/cuda:12.8.0-devel-ubuntu22.04 ARG TENSORRT_VERSION ENV TENSORRT_VERSION=${TENSORRT_VERSION} RUN test -n "$TENSORRT_VERSION" || (echo "No tensorrt version specified, please use --build-arg TENSORRT_VERSION=x.y to specify a version." && exit 1) -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.11 ENV PYTHON_VERSION=${PYTHON_VERSION} ARG USE_PRE_CXX11_ABI diff --git a/docker/README.md b/docker/README.md index 0037199990..4f89e6f435 100644 --- a/docker/README.md +++ b/docker/README.md @@ -5,9 +5,9 @@ * The `Dockerfile` currently uses Bazelisk to select the Bazel version, and uses the exact library versions of Torch and CUDA listed in dependencies. * The desired versions of TensorRT must be specified as build-args, with major and minor versions as in: `--build-arg TENSORRT_VERSION=a.b` * [**Optional**] The desired base image be changed by explicitly setting a base image, as in `--build-arg BASE_IMG=nvidia/cuda:11.8.0-devel-ubuntu22.04`, though this is optional. - * [**Optional**] Additionally, the desired Python version can be changed by explicitly setting a version, as in `--build-arg PYTHON_VERSION=3.10`, though this is optional as well. + * [**Optional**] Additionally, the desired Python version can be changed by explicitly setting a version, as in `--build-arg PYTHON_VERSION=3.11`, though this is optional as well. -* This `Dockerfile` installs `cxx11-abi` versions of Pytorch and builds Torch-TRT using `cxx11-abi` libtorch as well. As of torch 2.7, torch requires `cxx11-abi` for all CUDA 11.8, 12.4, and 12.6. +* This `Dockerfile` installs `cxx11-abi` versions of Pytorch and builds Torch-TRT using `cxx11-abi` libtorch as well. As of torch 2.7, torch requires `cxx11-abi` for all CUDA 11.8, 12.4, 12.6, and later versions. Note: By default the container uses the `cxx11-abi` version of Torch + Torch-TRT. If you are using a workflow that requires a build of PyTorch on the PRE CXX11 ABI, please add the Docker build argument: `--build-arg USE_PRE_CXX11_ABI=1` @@ -17,14 +17,14 @@ Note: By default the container uses the `cxx11-abi` version of Torch + Torch-TRT ### Instructions -- The example below uses TensorRT 10.7.0.23 +- The example below uses TensorRT 10.8.0.43 - See dependencies for a list of current default dependencies. > From root of Torch-TensorRT repo Build: ``` -DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.7.0 -f docker/Dockerfile -t torch_tensorrt:latest . +DOCKER_BUILDKIT=1 docker build --build-arg TENSORRT_VERSION=10.8.0 -f docker/Dockerfile -t torch_tensorrt:latest . ``` Run: diff --git a/examples/dynamo/auto_generate_converters.py b/examples/dynamo/auto_generate_converters.py index 59085fe2fb..bd3ce24ee6 100644 --- a/examples/dynamo/auto_generate_converters.py +++ b/examples/dynamo/auto_generate_converters.py @@ -5,7 +5,7 @@ =================================================================== We are going to demonstrate how to automatically generate a converter for a custom kernel using Torch-TensorRT using -the new Python based plugin system in TensorRT 10.7. +the new Python based plugin system in TensorRT 10.8. Torch-TensorRT supports falling back to PyTorch implementations of operations in the case that Torch-TensorRT does not know how to compile them in TensorRT. However, this comes at the cost of a graph break and will reduce the performance of the model. @@ -21,7 +21,7 @@ the performance and resource overhead from a graph break. Previously this involved a complex process in not only building a performant kernel but setting it up to run in TensorRT (see: `Using Custom Kernels within TensorRT Engines with Torch-TensorRT `_). -With TensorRT 10.7, there is a new Python native plugin system which greatly streamlines this process. This +With TensorRT 10.8, there is a new Python native plugin system which greatly streamlines this process. This plugin system also allows Torch-TensorRT to automatically generate the necessary conversion code to convert the operation in PyTorch to TensorRT. """ @@ -99,7 +99,7 @@ def _(x: torch.Tensor, y: torch.Tensor, b: float = 0.2, a: int = 2) -> torch.Ten # %% # Writing Plugins for TensorRT using the Quick Deploy Plugin system # ------------------------------------------------------------------- -# The quick deployment plugin system in TensorRT 10.7 allows for the creation of custom plugins in Python with significantly +# The quick deployment plugin system in TensorRT 10.8 allows for the creation of custom plugins in Python with significantly # less boilerplate. It uses a similar system PyTorch where you define a function that describes the shape and data type transformations # that the operator will perform and then define the code to launch the kernel given GPU memory handles. # diff --git a/examples/dynamo/auto_generate_plugin.py b/examples/dynamo/auto_generate_plugin.py index 2ea50b87f6..4290745bfd 100644 --- a/examples/dynamo/auto_generate_plugin.py +++ b/examples/dynamo/auto_generate_plugin.py @@ -21,7 +21,7 @@ the performance and resource overhead from a graph break. Previously this involved a complex process in not only building a performant kernel but setting it up to run in TensorRT (see: `Using Custom Kernels within TensorRT Engines with Torch-TensorRT `_). -With TensorRT 10.7, there is a new Python native plugin system which greatly streamlines this process. This +As of TensorRT 10.7, there is a new Python native plugin system which greatly streamlines this process. This plugin system also allows Torch-TensorRT to automatically generate the necessary conversion code to convert the operation in PyTorch to TensorRT. """ diff --git a/packaging/pre_build_script.sh b/packaging/pre_build_script.sh index 6acc19354b..84a8650d6d 100755 --- a/packaging/pre_build_script.sh +++ b/packaging/pre_build_script.sh @@ -24,7 +24,7 @@ export TORCH_INSTALL_PATH=$(python -c "import torch, os; print(os.path.dirname(t if [[ ${TENSORRT_VERSION} != "" ]]; then # this is the upgraded TensorRT version, replace current tensorrt version to the upgrade tensorRT version in the pyproject.toml # example: __tensorrt_version__: ">=10.3.0,<=10.6.0" - # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.7.0 + # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.8.0 current_version=$(cat dev_dep_versions.yml | grep __tensorrt_version__ | sed 's/__tensorrt_version__: //g' | sed 's/"//g') sed -i -e "s/tensorrt-cu12${current_version}/tensorrt-cu12==${TENSORRT_VERSION}/g" \ -e "s/tensorrt-cu12-bindings${current_version}/tensorrt-cu12-bindings==${TENSORRT_VERSION}/g" \ diff --git a/packaging/pre_build_script_windows.sh b/packaging/pre_build_script_windows.sh index 219c0f9fbb..719e97d6c1 100644 --- a/packaging/pre_build_script_windows.sh +++ b/packaging/pre_build_script_windows.sh @@ -9,7 +9,7 @@ echo TENSORRT_VERSION=${TENSORRT_VERSION} if [[ ${TENSORRT_VERSION} != "" ]]; then # this is the upgraded TensorRT version, replace current tensorrt version to the upgrade tensorRT version in the pyproject.toml # example: __tensorrt_version__: ">=10.3.0,<=10.6.0" - # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.7.0 + # replace: tensorrt-cu12>=10.3.0,<=10.6.0 to tensorrt-cu12==10.8.0 current_version=$(cat dev_dep_versions.yml | grep __tensorrt_version__ | sed 's/__tensorrt_version__: //g' | sed 's/"//g') sed -i -e "s/tensorrt-cu12${current_version}/tensorrt-cu12==${TENSORRT_VERSION}/g" \ -e "s/tensorrt-cu12-bindings${current_version}/tensorrt-cu12-bindings==${TENSORRT_VERSION}/g" \ diff --git a/packaging/smoke_test_script.sh b/packaging/smoke_test_script.sh index ef584fef51..222dc7088c 100644 --- a/packaging/smoke_test_script.sh +++ b/packaging/smoke_test_script.sh @@ -2,5 +2,5 @@ # The issue was smoke test installs the built torch_tensorrt wheel file and checks `import torch_tensorrt; print(torch_tensorrt.__version__)` # Since tensorrt cannot be pip installable in CI, the smoke test will fail. # One way we tried to handle it is manually install tensorrt wheel while by extracting from the tarball. -# However, the TensorRT-10.7.0.23/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it. +# However, the TensorRT-10.8.0.43/lib path doesn't seem to show up in LD_LIBRARY_PATH even if we explicitly set it. # TODO: Implement a custom smoke_test script to verify torch_tensorrt installation. \ No newline at end of file diff --git a/py/ci/Dockerfile.ci b/py/ci/Dockerfile.ci index 674fecf8d6..249913196e 100644 --- a/py/ci/Dockerfile.ci +++ b/py/ci/Dockerfile.ci @@ -1,15 +1,15 @@ -FROM pytorch/manylinux2_28-builder:cuda12.6 +FROM pytorch/manylinux2_28-builder:cuda12.8 RUN yum install -y ninja-build # download TensorRT tarball -RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz \ -&& gunzip TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz \ -&& tar -xvf TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar \ -&& rm TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar +RUN wget -q https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz \ +&& gunzip TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz \ +&& tar -xvf TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar \ +&& rm TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar -ENV TENSORRT_DIR=/TensorRT-10.7.0.23 -ENV TENSORRT_VERSION=10.7.0 +ENV TENSORRT_DIR=/TensorRT-10.8.0.43 +ENV TENSORRT_VERSION=10.8.0 RUN wget https://github.com/bazelbuild/bazelisk/releases/download/v1.17.0/bazelisk-linux-amd64 \ && mv bazelisk-linux-amd64 /usr/bin/bazel \ diff --git a/py/ci/build_manifest.txt b/py/ci/build_manifest.txt index 1cb0277187..d93710ccab 100644 --- a/py/ci/build_manifest.txt +++ b/py/ci/build_manifest.txt @@ -1,5 +1,5 @@ -py38 py39 py310 py311 +py312 libtorchtrt_pre_cxx11_abi diff --git a/py/ci/build_whl.sh b/py/ci/build_whl.sh index d8f91da686..43e332c044 100755 --- a/py/ci/build_whl.sh +++ b/py/ci/build_whl.sh @@ -3,15 +3,15 @@ # Example usage: docker run -it -v$(pwd):/workspace/TensorRT build_torch_tensorrt_wheel /bin/bash /workspace/TensorRT/py/ci/build_whl.sh export CXX=g++ -export CUDA_HOME=/usr/local/cuda-12.4 +export CUDA_HOME=/usr/local/cuda-12.8 export PROJECT_DIR=/workspace/TensorRT rm -rf /usr/local/cuda -if [[ $CUDA_HOME == "/usr/local/cuda-12.4" ]]; then +if [[ $CUDA_HOME == "/usr/local/cuda-12.8" ]]; then cp -r /usr/local/cuda-11.8 /usr/local/cuda cp -r /usr/local/cuda-12.0/ /usr/local/cuda/ - rsync -a /usr/local/cuda-12.4/ /usr/local/cuda/ + rsync -a /usr/local/cuda-12.8/ /usr/local/cuda/ export CUDA_HOME=/usr/local/cuda else ln -s $CUDA_HOME /usr/local/cuda @@ -32,18 +32,6 @@ patch_wheel() { LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${TENSERRT_DIR}/lib:$1/torch/lib:$1/tensorrt/:${CUDA_HOME}/lib64:${CUDA_HOME}/lib64/stubs $2/bin/python -m auditwheel repair $(cat ${PROJECT_DIR}/py/ci/soname_excludes.params) --plat manylinux_2_34_x86_64 dist/torch_tensorrt-*-$3-linux_x86_64.whl } -py38() { - cd ${PROJECT_DIR} - PY_BUILD_CODE=cp38-cp38 - PY_SINGLE_BUILD_CODE=cp38 - PY_VERSION=3.8 - PY_NAME=python${PY_VERSION} - PY_DIR=/opt/python/${PY_BUILD_CODE} - PY_PKG_DIR=${PY_DIR}/lib/${PY_NAME}/site-packages/ - build_wheel ${PY_DIR} ${PY_SINGLE_BUILD_CODE} - patch_wheel ${PY_PKG_DIR} ${PY_DIR} ${PY_BUILD_CODE} -} - py39() { cd ${PROJECT_DIR} PY_BUILD_CODE=cp39-cp39 diff --git a/py/torch_tensorrt/__init__.py b/py/torch_tensorrt/__init__.py index d7f5e7ba58..6d79f9b4f3 100644 --- a/py/torch_tensorrt/__init__.py +++ b/py/torch_tensorrt/__init__.py @@ -56,7 +56,7 @@ def _find_lib(name: str, paths: List[str]) -> str: ctypes.CDLL(_find_lib(lib, WIN_PATHS)) elif sys.platform.startswith("linux"): - LINUX_PATHS = ["/usr/local/cuda-12.4/lib64", "/usr/lib", "/usr/lib64"] + LINUX_PATHS = ["/usr/local/cuda-12.8/lib64", "/usr/lib", "/usr/lib64"] if "LD_LIBRARY_PATH" in os.environ: LINUX_PATHS += os.environ["LD_LIBRARY_PATH"].split(os.path.pathsep) diff --git a/pyproject.toml b/pyproject.toml index bc01a84038..2ee8c0bd99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "cffi>=1.15.1", "typing-extensions>=4.7.0", "future>=0.18.3", - "tensorrt-cu12>=10.7.0.post1,<10.8.0", + "tensorrt-cu12>=10.8.0,<10.9.0", "torch>=2.7.0.dev,<2.8.0", "pybind11==2.6.2", "numpy", @@ -56,10 +56,10 @@ keywords = [ ] dependencies = [ "torch>=2.7.0.dev,<2.8.0", - "tensorrt>=10.7.0.post1,<10.8.0", - "tensorrt-cu12>=10.7.0.post1,<10.8.0", - "tensorrt-cu12-bindings>=10.7.0,<10.8.0", - "tensorrt-cu12-libs>=10.7.0,<10.8.0", + "tensorrt>=10.8.0,<10.9.0", + "tensorrt-cu12>=10.8.0,<10.9.0", + "tensorrt-cu12-bindings>=10.8.0,<10.9.0", + "tensorrt-cu12-libs>=10.8.0,<10.9.0", "packaging>=23", "numpy", "typing-extensions>=4.7.0", diff --git a/toolchains/ci_workspaces/MODULE.bazel.tmpl b/toolchains/ci_workspaces/MODULE.bazel.tmpl index 66b5d448ca..c09659f450 100644 --- a/toolchains/ci_workspaces/MODULE.bazel.tmpl +++ b/toolchains/ci_workspaces/MODULE.bazel.tmpl @@ -67,18 +67,18 @@ http_archive( http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", - strip_prefix = "TensorRT-10.7.0.23", + strip_prefix = "TensorRT-10.8.0.43", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz", ], ) http_archive( name = "tensorrt_win", build_file = "@//third_party/tensorrt/archive:BUILD", - strip_prefix = "TensorRT-10.7.0.23", + strip_prefix = "TensorRT-10.8.0.43", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/zip/TensorRT-10.7.0.23.Windows.win10.cuda-12.6.zip", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/zip/TensorRT-10.8.0.43.Windows.win10.cuda-12.8.zip", ], ) diff --git a/toolchains/legacy/WORKSPACE.win.release.tmpl b/toolchains/legacy/WORKSPACE.win.release.tmpl index 5570516c6c..dfab56aad4 100644 --- a/toolchains/legacy/WORKSPACE.win.release.tmpl +++ b/toolchains/legacy/WORKSPACE.win.release.tmpl @@ -63,7 +63,7 @@ http_archive( new_local_repository( name = "tensorrt_win", - path = "C:/TensorRT-10.7.0.23", + path = "C:/TensorRT-10.8.0.43", build_file = "@//third_party/tensorrt/local:BUILD" ) diff --git a/toolchains/legacy/WORKSPACE.x86_64 b/toolchains/legacy/WORKSPACE.x86_64 index 1428ec439a..c8df299c8e 100644 --- a/toolchains/legacy/WORKSPACE.x86_64 +++ b/toolchains/legacy/WORKSPACE.x86_64 @@ -43,7 +43,7 @@ local_repository( new_local_repository( name = "cuda", build_file = "@//third_party/cuda:BUILD", - path = "/usr/local/cuda-12.4/", + path = "/usr/local/cuda-12.8/", ) new_local_repository( diff --git a/toolchains/legacy/WORKSPACE.x86_64.release.rhel.tmpl b/toolchains/legacy/WORKSPACE.x86_64.release.rhel.tmpl index fcc5e33f44..e0a2974c3d 100644 --- a/toolchains/legacy/WORKSPACE.x86_64.release.rhel.tmpl +++ b/toolchains/legacy/WORKSPACE.x86_64.release.rhel.tmpl @@ -71,9 +71,9 @@ http_archive( http_archive( name = "tensorrt", build_file = "@//third_party/tensorrt/archive:BUILD", - strip_prefix = "TensorRT-10.7.0.23", + strip_prefix = "TensorRT-10.8.0.43", urls = [ - "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.7.0/tars/TensorRT-10.7.0.23.Linux.x86_64-gnu.cuda-12.6.tar.gz", + "https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/10.8.0/tars/TensorRT-10.8.0.43.Linux.x86_64-gnu.cuda-12.8.tar.gz", ], ) diff --git a/toolchains/legacy/WORKSPACE.x86_64.release.ubuntu b/toolchains/legacy/WORKSPACE.x86_64.release.ubuntu index e7bad3d30f..2e953b5755 100644 --- a/toolchains/legacy/WORKSPACE.x86_64.release.ubuntu +++ b/toolchains/legacy/WORKSPACE.x86_64.release.ubuntu @@ -43,7 +43,7 @@ local_repository( new_local_repository( name = "cuda", build_file = "@//third_party/cuda:BUILD", - path = "/usr/local/cuda-12.4", + path = "/usr/local/cuda-12.8", ) new_local_repository(