Add the new build (triton-inference-server#1)

* Add the new build

* Add installation of python files

* Review edits

* Move resources -> src/

* Remove debug message

Author: Tabrizian

CMakeLists.txt

@@ -24,24 +24,80 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-cmake_minimum_required (VERSION 3.5)
 
+cmake_minimum_required(VERSION 3.17)
+
+project(tritonpythonbackend LANGUAGES C CXX)
+
+#
+# Options
+#
+# Must include options required for this project as well as any
+# projects included in this one by FetchContent.
+#
+# GPU support is disabled by default because python backend doesn't
+# because python backend does not need to access CUDA or GPUs
+#
+option(TRITON_ENABLE_GPU "Enable GPU support in backend" OFF)
+option(TRITON_ENABLE_STATS "Include statistics collections in backend" ON)
+
+set(TRITON_BACKEND_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/backend repo")
+set(TRITON_COMMON_REPO_TAG "main" CACHE STRING "Tag for triton-inference-server/common repo")
+
+#
+# Dependencies
 #
-# libtriton_python.so
+# FetchContent's composibility isn't very good. We must include the
+# transitive closure of all repos so that we can override the tag.
 #
-configure_file(libtriton_python.ldscript libtriton_python.ldscript COPYONLY)
+include(FetchContent)
+
+FetchContent_Declare(
+  repo-common
+  GIT_REPOSITORY git@github.com:triton-inference-server/common.git
+  GIT_TAG ${TRITON_COMMON_REPO_TAG}
+  GIT_SHALLOW ON
+)
+FetchContent_Declare(
+  repo-backend
+  GIT_REPOSITORY git@github.com:triton-inference-server/backend.git
+  GIT_TAG ${TRITON_BACKEND_REPO_TAG}
+  GIT_SHALLOW ON
+)
+FetchContent_MakeAvailable(repo-common repo-backend)
 
-find_package(gRPC CONFIG REQUIRED)
-find_package(Protobuf CONFIG REQUIRED)
+FetchContent_Declare(
+  googletest
+  GIT_REPOSITORY "https://github.com/google/googletest"
+  GIT_TAG "release-1.10.0"
+  GIT_SHALLOW ON
+)
+FetchContent_Declare(
+  grpc
+  GIT_REPOSITORY https://github.com/grpc/grpc.git
+  GIT_TAG "v1.31.1"
+  GIT_SHALLOW ON
+)
+FetchContent_MakeAvailable(googletest grpc)
+
+set(_PROTOBUF_LIBPROTOBUF libprotobuf)
+set(_REFLECTION grpc++_reflection)
+set(_PROTOBUF_PROTOC $<TARGET_FILE:protoc>)
+set(_GRPC_GRPCPP grpc++)
+set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
 
-set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:gRPC::grpc_cpp_plugin>)
-set(_PROTOBUF_PROTOC $<TARGET_FILE:protobuf::protoc>)
+get_target_property(Protobuf_INCLUDE_DIRS ${_PROTOBUF_LIBPROTOBUF} INTERFACE_INCLUDE_DIRECTORIES)
+get_target_property(gRPC_INCLUDE_DIRS ${_GRPC_GRPCPP} INTERFACE_INCLUDE_DIRECTORIES)
 
-get_filename_component(python_host_proto_abspath "python_host.proto" ABSOLUTE)
+include_directories(${Protobuf_INCLUDE_DIRS})
+include_directories(${gRPC_INCLUDE_DIRS})
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+get_filename_component(python_host_proto_abspath "src/python_host.proto" ABSOLUTE)
 get_filename_component(python_host_proto_dir "${python_host_proto_abspath}" PATH)
 
 set(GRPC_SRCS
-    "${CMAKE_CURRENT_BINARY_DIR}/python_host.grpc.pb.cc"3
+    "${CMAKE_CURRENT_BINARY_DIR}/python_host.grpc.pb.cc"
     "${CMAKE_CURRENT_BINARY_DIR}/python_host.pb.cc")
 
 set(GRPC_HDRS
@@ -60,11 +116,11 @@ add_custom_command(
     --cpp_out "${CMAKE_CURRENT_BINARY_DIR}"
     -I "${python_host_proto_dir}"
     --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}"
-    "python_host.proto"
-  DEPENDS "python_host.proto" proto-library
+    "${python_host_proto_abspath}"
+  DEPENDS "${python_host_proto_abspath}" ${_PROTOBUF_LIBPROTOBUF}
 )
 
-find_program(PYTHON "python3" REQUIRED)
+find_package(Python REQUIRED COMPONENTS Interpreter)   
 add_custom_command(
   OUTPUT ${GRPC_PY}
   COMMAND ${PYTHON}
@@ -73,65 +129,121 @@ add_custom_command(
     -I "${python_host_proto_dir}"
     --grpc_python_out "${CMAKE_CURRENT_BINARY_DIR}"
     --python_out "${CMAKE_CURRENT_BINARY_DIR}"
-    "python_host.proto"
-  DEPENDS "python_host.proto" proto-library
+    "${python_host_proto_abspath}"
+  DEPENDS "${python_host_proto_abspath}" ${_PROTOBUF_LIBPROTOBUF}
 )
 
 add_custom_target(python-grpc-py-library ALL
                   DEPENDS ${GRPC_PY})
 
 add_library(
   python-grpc-library EXCLUDE_FROM_ALL OBJECT
-  ${GRPC_SRCS} ${GRPC_HDRS}
+  ${GRPC_SRCS} ${GRPC_HDRS} 
 )
 
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
+configure_file(src/libtriton_python.ldscript libtriton_python.ldscript COPYONLY)
 
 add_library(
-  triton_python SHARED
-  python.cc
-
-  $<TARGET_OBJECTS:backend-library>
-  $<TARGET_OBJECTS:python-grpc-library>)
-
-if(${TRITON_ENABLE_GPU})
-  target_include_directories(triton_python PRIVATE ${CUDA_INCLUDE_DIRS})
-  target_link_libraries(
-    triton_python
-    PUBLIC -L/usr/local/cuda/lib64/stubs
-    PUBLIC -lnvidia-ml
-    PRIVATE ${CUDA_LIBRARIES}
-  )
-endif() # TRITON_ENABLE_GPU
+  triton-python-backend SHARED
+  src/python.cc
 
-set_target_properties(
-  triton_python
-  PROPERTIES LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_python.ldscript
+  $<TARGET_OBJECTS:python-grpc-library>
 )
 
-set_target_properties(
-  triton_python
-  PROPERTIES LINK_FLAGS "-Wl,--version-script libtriton_python.ldscript"
+
+add_library(
+  TritonPythonBackend::triton-python-backend ALIAS triton-python-backend
 )
 
 target_include_directories(
-        triton_python
-        PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+  triton-python-backend
+  PRIVATE
+    ${CMAKE_CURRENT_SOURCE_DIR}/src
+)
+
+target_compile_features(triton-python-backend PRIVATE cxx_std_11)
+target_compile_options(
+  triton-python-backend PRIVATE
+  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+    -Wall -Wextra -Wno-unused-parameter -Werror>
+)
 
 target_link_libraries(
-  triton_python
-  PRIVATE gRPC::grpc++
+  triton-python-backend
+  PRIVATE
+    triton-backend-utils  # from repo-backend
+    ${_GRPC_GRPCPP}
+)
+
+set_target_properties(
+  triton-python-backend PROPERTIES
+  POSITION_INDEPENDENT_CODE ON
+  OUTPUT_NAME triton_python
+  LINK_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libtriton_python.ldscript
+  LINK_FLAGS "-Wl,--version-script libtriton_python.ldscript"
+)
+
+#
+# Install
+#
+include(GNUInstallDirs)
+set(INSTALL_CONFIGDIR ${CMAKE_INSTALL_LIBDIR}/cmake/TritonPythonBackend)
+
+install(
+  TARGETS
+    triton-python-backend
+  EXPORT
+    triton-python-backend-targets
+  LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+  ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+)
+
+install(
+  EXPORT
+    triton-python-backend-targets
+  FILE
+    TritonPythonBackendTargets.cmake
+  NAMESPACE
+    TritonPythonBackend::
+  DESTINATION
+    ${INSTALL_CONFIGDIR}
+)
+
+install(
+  FILES
+    ${GRPC_PY}
+    src/resources/startup.py
+  DESTINATION
+    ${INSTALL_CONFIGDIR}
+)
+
+install(
+  FILES
+    src/resources/triton_python_backend_utils.py
+  DESTINATION
+    ${INSTALL_CONFIGDIR}
+)
+
+include(CMakePackageConfigHelpers)
+configure_package_config_file(
+  ${CMAKE_CURRENT_LIST_DIR}/cmake/TritonPythonBackendConfig.cmake.in
+  ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendConfig.cmake
+  INSTALL_DESTINATION ${INSTALL_CONFIGDIR}
 )
 
 install(
-  TARGETS triton_python
-  LIBRARY DESTINATION backends/python
+  FILES
+  ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendConfig.cmake
+  DESTINATION ${INSTALL_CONFIGDIR}
 )
 
-install(FILES
-          ${GRPC_PY}
-          resources/startup.py
-        DESTINATION lib/python/runtime)
+#
+# Export from build tree
+#
+export(
+  EXPORT triton-python-backend-targets
+  FILE ${CMAKE_CURRENT_BINARY_DIR}/TritonPythonBackendTargets.cmake
+  NAMESPACE TritonPythonBackend::
+)
 
-install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/resources/triton_python_backend_utils.py
-       DESTINATION lib/python/runtime)
+export(PACKAGE TritonPythonBackend)

cmake/TritonPythonBackendConfig.cmake.in

@@ -0,0 +1,40 @@
+  
+# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include(CMakeFindDependencyMacro)
+
+get_filename_component(
+  TRITONPYTHONBACKEND_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH
+)
+
+list(APPEND CMAKE_MODULE_PATH ${TRITONPYTHONBACKEND_CMAKE_DIR})
+
+if(NOT TARGET TritonPythonBackend::triton-python-backend)
+  include("${TRITONPYTHONBACKEND_CMAKE_DIR}/TritonPythonBackendTargets.cmake")
+endif()
+
+set(TRITONPYTHONBACKEND_LIBRARIES TritonPythonBackend::triton-python-backend)

libtriton_python.ldscript renamed to src/libtriton_python.ldscript

@@ -33,7 +33,6 @@
 #include <stdio.h>
 #include <sys/wait.h>
 #include <unistd.h>
-
 #include <algorithm>
 #include <chrono>
 #include <condition_variable>
@@ -53,15 +52,13 @@
 #include <vector>
 
 #include "python_host.grpc.pb.h"
-#include "src/backends/backend/examples/backend_utils.h"
-#include "src/backends/backend/tritonbackend.h"
-#include "src/core/json.h"
-#include "src/core/tritonserver.h"
 
-namespace ni = nvidia::inferenceserver;
-namespace nib = nvidia::inferenceserver::backend;
+#include "triton/backend/backend_common.h"
+#include "triton/common/triton_json.h"
+#include "triton/common/tritonbackend.h"
+#include "triton/common/tritonserver.h"
 
-namespace nvidia { namespace inferenceserver { namespace backend {
+namespace triton { namespace backend { namespace python {
 
 #define RESPOND_AND_RETURN_IF_ERROR(REQUEST, X)                         \
   do {                                                                  \
@@ -82,10 +79,6 @@ namespace nvidia { namespace inferenceserver { namespace backend {
     }                                                                   \
   } while (false)
 
-}}}  // namespace nvidia::inferenceserver::backend
-
-namespace {
-
 #define GUARDED_RESPOND_IF_ERROR(RESPONSES, IDX, X)                     \
   do {                                                                  \
     if ((RESPONSES)[IDX] != nullptr) {                                  \
@@ -130,18 +123,18 @@ class ModelInstanceState {
 
   /// Load Triton inputs to the appropriate Protobufs
   TRITONSERVER_Error* GetInputTensor(
-      const uint32_t iidx, TRITONBACKEND_Request* request,
-      ni::Tensor* input_tensor, std::vector<TRITONBACKEND_Response*>& responses,
-      size_t r, uint32_t& batch_size);
+      const uint32_t iidx, TRITONBACKEND_Request* request, Tensor* input_tensor,
+      std::vector<TRITONBACKEND_Response*>& responses, size_t r,
+      uint32_t& batch_size);
 
   // TODO: Create getter and setters
-  std::unique_ptr<ni::PythonInterpreter::Stub> stub;
+  std::unique_ptr<PythonInterpreter::Stub> stub;
 
  private:
   ModelInstanceState(
       ModelState* model_state, TRITONBACKEND_ModelInstance* model_instance,
       const char* name, const TRITONSERVER_InstanceGroupKind kind,
-      const int32_t device_id, ni::TritonJson::Value&& model_config,
+      const int32_t device_id, triton::common::TritonJson::Value&& model_config,
       TRITONBACKEND_Model* trition_model);
 
   TRITONSERVER_Error* ConnectPythonInterpreter();
@@ -156,7 +149,7 @@ class ModelInstanceState {
   bool connected_ = false;
 
  public:
-  ni::TritonJson::Value model_config;
+  triton::common::TritonJson::Value model_config;
 
  private:
   TRITONBACKEND_Model* triton_model_;
@@ -176,7 +169,7 @@ class ModelState {
   uint64_t ModelVersion() const { return model_version_; }
 
   // Get backend state
-  ::BackendState* BackendState() { return backend_state_; }
+  BackendState* StateForBackend() { return backend_state_; }
 
   // Get Model Path
   const char* ModelPath() { return model_path_; }
@@ -185,15 +178,15 @@ class ModelState {
   ModelState(
       TRITONSERVER_Server* triton_server, TRITONBACKEND_Model* triton_model,
       const char* model_name, const uint64_t model_version,
-      ni::TritonJson::Value&& model_config, ::BackendState* backend_state,
-      const char* model_path);
+      triton::common::TritonJson::Value&& model_config,
+      BackendState* backend_state, const char* model_path);
 
   TRITONSERVER_Server* triton_server_;
   TRITONBACKEND_Model* triton_model_;
   const std::string model_name_;
   const uint64_t model_version_;
-  ni::TritonJson::Value model_config_;
-  ::BackendState* backend_state_;
+  triton::common::TritonJson::Value model_config_;
+  BackendState* backend_state_;
   const char* model_path_;
 };
 
@@ -238,10 +231,10 @@ ModelInstanceState::CreatePythonInterpreter()
   if (interpreter_pid_ == 0) {
     // Use the python available in $PATH
     std::string python_interpreter_path =
-        model_state_->BackendState()->python_runtime;
+        model_state_->StateForBackend()->python_runtime;
 
     std::stringstream ss;
-    ss << model_state_->BackendState()->python_lib << "/startup.py";
+    ss << model_state_->StateForBackend()->python_lib << "/startup.py";
     std::string python_interpreter_startup = ss.str();
 
     subinterpreter_commandline[0] = python_interpreter_path.c_str();
@@ -278,10 +271,10 @@ ModelInstanceState::ConnectPythonInterpreter()
   auto grpc_channel =
       grpc::CreateChannel(domain_socket_, grpc::InsecureChannelCredentials());
 
-  stub = ni::PythonInterpreter::NewStub(grpc_channel);
+  stub = PythonInterpreter::NewStub(grpc_channel);
 
-  std::shared_ptr<ni::InitializationCommand> initialization_params(
-      new ni::InitializationCommand());
+  std::shared_ptr<InitializationCommand> initialization_params(
+      new InitializationCommand());
 
   std::vector<std::string> keys;
   LOG_IF_ERROR(model_config.Members(&keys), "can't get key names");
@@ -294,7 +287,7 @@ ModelInstanceState::ConnectPythonInterpreter()
         value_pair->set_value(val);
       };
 
-  ni::TritonJson::WriteBuffer buffer;
+  triton::common::TritonJson::WriteBuffer buffer;
   model_config.Write(&buffer);
 
   insert_model_param("model_config", std::move(buffer.MutableContents()));
@@ -312,7 +305,7 @@ ModelInstanceState::ConnectPythonInterpreter()
   constexpr uint8_t conn_attempts = 5;
   for (int i = 0; i < conn_attempts; ++i) {
     grpc::ClientContext context;
-    ni::Empty null_msg;
+    Empty null_msg;
     status = stub->Init(&context, *initialization_params, &null_msg);
     if (status.ok()) {
       LOG_MESSAGE(
@@ -334,7 +327,7 @@ ModelInstanceState::ConnectPythonInterpreter()
 ModelInstanceState::ModelInstanceState(
     ModelState* model_state, TRITONBACKEND_ModelInstance* triton_model_instance,
     const char* name, const TRITONSERVER_InstanceGroupKind kind,
-    const int32_t device_id, ni::TritonJson::Value&& model_config,
+    const int32_t device_id, triton::common::TritonJson::Value&& model_config,
     TRITONBACKEND_Model* triton_model)
     : model_state_(model_state), triton_model_instance_(triton_model_instance),
       name_(name), kind_(kind), device_id_(device_id),
@@ -372,7 +365,7 @@ ModelInstanceState::Create(
   RETURN_IF_ERROR(
       TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size));
 
-  ni::TritonJson::Value model_config;
+  triton::common::TritonJson::Value model_config;
 
   TRITONSERVER_Error* err = model_config.Parse(buffer, byte_size);
   RETURN_IF_ERROR(TRITONSERVER_MessageDelete(config_message));
@@ -392,7 +385,7 @@ ModelInstanceState::~ModelInstanceState()
   {
     // Close python interpreter.
     grpc::ClientContext context;
-    ni::Empty null_msg;
+    Empty null_msg;
 
     if (connected_) {
       auto err = stub->Fini(&context, null_msg, &null_msg);
@@ -424,9 +417,9 @@ ModelInstanceState::~ModelInstanceState()
 
 TRITONSERVER_Error*
 ModelInstanceState::GetInputTensor(
-    const uint32_t iidx, TRITONBACKEND_Request* request,
-    ni::Tensor* input_tensor, std::vector<TRITONBACKEND_Response*>& responses,
-    size_t r, uint32_t& batch_size)
+    const uint32_t iidx, TRITONBACKEND_Request* request, Tensor* input_tensor,
+    std::vector<TRITONBACKEND_Response*>& responses, size_t r,
+    uint32_t& batch_size)
 {
   const char* input_name;
   // Load iidx'th input name
@@ -495,7 +488,7 @@ ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state)
   RETURN_IF_ERROR(
       TRITONSERVER_MessageSerializeToJson(config_message, &buffer, &byte_size));
 
-  ni::TritonJson::Value model_config;
+  triton::common::TritonJson::Value model_config;
   TRITONSERVER_Error* err = model_config.Parse(buffer, byte_size);
   RETURN_IF_ERROR(TRITONSERVER_MessageDelete(config_message));
   RETURN_IF_ERROR(err);
@@ -514,7 +507,7 @@ ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state)
 
   void* bstate;
   RETURN_IF_ERROR(TRITONBACKEND_BackendState(backend, &bstate));
-  ::BackendState* backend_state = reinterpret_cast<::BackendState*>(bstate);
+  BackendState* backend_state = reinterpret_cast<BackendState*>(bstate);
 
   const char* path = nullptr;
   TRITONBACKEND_ModelArtifactType artifact_type;
@@ -539,17 +532,15 @@ ModelState::Create(TRITONBACKEND_Model* triton_model, ModelState** state)
 ModelState::ModelState(
     TRITONSERVER_Server* triton_server, TRITONBACKEND_Model* triton_model,
     const char* model_name, const uint64_t model_version,
-    ni::TritonJson::Value&& model_config, ::BackendState* backend_state,
-    const char* model_path)
+    triton::common::TritonJson::Value&& model_config,
+    BackendState* backend_state, const char* model_path)
     : triton_server_(triton_server), triton_model_(triton_model),
       model_name_(model_name), model_version_(model_version),
       model_config_(std::move(model_config)), backend_state_(backend_state),
       model_path_(model_path)
 {
 }
 
-}  // namespace
-
 extern "C" {
 
 TRITONSERVER_Error*
@@ -590,24 +581,24 @@ TRITONBACKEND_Initialize(TRITONBACKEND_Backend* backend)
       TRITONSERVER_LOG_VERBOSE,
       (std::string("backend configuration:\n") + buffer).c_str());
 
-  ni::TritonJson::Value backend_config;
+  triton::common::TritonJson::Value backend_config;
   if (byte_size != 0) {
     RETURN_IF_ERROR(backend_config.Parse(buffer, byte_size));
   }
 
   std::unique_ptr<BackendState> backend_state(new BackendState());
-  ni::TritonJson::Value cmdline;
+  triton::common::TritonJson::Value cmdline;
   bool found_py_lib_config = false;
   bool found_py_runtime_config = false;
 
   if (backend_config.Find("cmdline", &cmdline)) {
-    ni::TritonJson::Value python_lib;
+    triton::common::TritonJson::Value python_lib;
     if (cmdline.Find("python-lib", &python_lib)) {
       RETURN_IF_ERROR(python_lib.AsString(&backend_state->python_lib));
       found_py_lib_config = true;
     }
 
-    ni::TritonJson::Value python_runtime;
+    triton::common::TritonJson::Value python_runtime;
     if (cmdline.Find("python-runtime", &python_runtime)) {
       RETURN_IF_ERROR(python_runtime.AsString(&backend_state->python_runtime));
       found_py_runtime_config = true;
@@ -756,11 +747,11 @@ TRITONBACKEND_ModelInstanceExecute(
   }
 
   // Create ExecuteRequest
-  ni::ExecuteRequest execute_request;
+  ExecuteRequest execute_request;
   for (uint32_t r = 0; r < request_count; ++r) {
     TRITONBACKEND_Request* request = requests[r];
 
-    ni::InferenceRequest* inference_request = execute_request.add_requests();
+    InferenceRequest* inference_request = execute_request.add_requests();
 
     uint32_t requested_input_count = 0;
     GUARDED_RESPOND_IF_ERROR(
@@ -774,7 +765,7 @@ TRITONBACKEND_ModelInstanceExecute(
 
     uint32_t batch_size = 0;
     for (size_t iidx = 0; iidx < requested_input_count; ++iidx) {
-      ni::Tensor* input_tensor = inference_request->add_inputs();
+      Tensor* input_tensor = inference_request->add_inputs();
       GUARDED_RESPOND_IF_ERROR(
           responses, r,
           instance_state->GetInputTensor(
@@ -806,7 +797,7 @@ TRITONBACKEND_ModelInstanceExecute(
 
   // ExecuteResponse
   grpc::ClientContext context;
-  ni::ExecuteResponse execute_response;
+  ExecuteResponse execute_response;
 
   uint64_t compute_start_ns = 0;
   SET_TIMESTAMP(compute_start_ns);
@@ -860,7 +851,7 @@ TRITONBACKEND_ModelInstanceExecute(
     uint32_t requested_output_count = 0;
 
     // Get response r
-    ni::InferenceResponse inference_response = execute_response.responses(r);
+    InferenceResponse inference_response = execute_response.responses(r);
 
     if (inference_response.failed()) {
       TRITONSERVER_Error* err = TRITONSERVER_ErrorNew(
@@ -883,7 +874,7 @@ TRITONBACKEND_ModelInstanceExecute(
         TRITONBACKEND_RequestOutputCount(request, &requested_output_count));
     for (size_t j = 0; j < requested_output_count; ++j) {
       // Prepare output buffers.
-      const ni::Tensor python_output_result = inference_response.outputs(j);
+      const Tensor python_output_result = inference_response.outputs(j);
       TRITONBACKEND_Output* triton_output;
       TRITONSERVER_DataType triton_dt =
           static_cast<TRITONSERVER_DataType>(python_output_result.dtype());
@@ -901,7 +892,7 @@ TRITONBACKEND_ModelInstanceExecute(
 
       std::vector<int64_t> output_dims(
           python_output_dims.begin(), python_output_dims.end());
-      int64_t output_byte_size = nib::GetByteSize(triton_dt, output_dims);
+      int64_t output_byte_size = GetByteSize(triton_dt, output_dims);
       void* output_buffer;
 
       TRITONSERVER_MemoryType output_memory_type = TRITONSERVER_MEMORY_CPU;
@@ -932,7 +923,7 @@ TRITONBACKEND_ModelInstanceExecute(
       auto output_response_tensor = std::find_if(
           inference_response.outputs().begin(),
           inference_response.outputs().end(),
-          [&output_tensor_name](const ni::Tensor& itr) {
+          [&output_tensor_name](const Tensor& itr) {
             return itr.name() == output_tensor_name;
           });
 
@@ -1028,3 +1019,5 @@ TRITONBACKEND_ModelInstanceFinalize(TRITONBACKEND_ModelInstance* instance)
 }
 
 }  // extern "C"
+
+}}}  // namespace triton::backend::python

python.cc renamed to src/python.cc

@@ -26,7 +26,7 @@
 
 syntax = "proto3";
 
-package nvidia.inferenceserver;
+package triton.backend.python;
 
 message InitializationCommand
 {