First version with cuda 12.6, trt 10

src/Detector/tensorrt_yolo/CMakeLists.txt

@@ -58,13 +58,20 @@ file(GLOB TENSORRT_CUDA_FILES *.cu)
 cuda_add_library(${libname_rt} SHARED
     ${TENSORRT_CUDA_FILES}
     ${TENSORRT_SOURCE_FILES}
-    ${TENSORRT_HEADER_FILES}
-)
+    ${TENSORRT_HEADER_FILES})
 
 #message("TensorRT OpenCV libraries:")
 #message("${OpenCV_LIBS}")
 #message(${OpenCV_DIR})
 
+if (MSVC)
+    file(GLOB TensorRT_LIBRARIES ${TensorRT_LIBRARY})
+endif()
+
+message("TensorRT_LIBRARY: ${TensorRT_LIBRARY}")
+message("TensorRT_LIBRARIES: ${TensorRT_LIBRARIES}")
+
+
 set(TENSORRT_LIBS
     ${OpenCV_LIBS}
     #${CUDA_LIBRARIES}
@@ -74,8 +81,7 @@ set(TENSORRT_LIBS
     ${CUDA_curand_LIBRARY}
     ${CUDNN_LIBRARY}
     # ${LIB_PTHREAD}
-    ${TensorRT_LIBRARIES}
-)
+    ${TensorRT_LIBRARIES})
 
 if (CMAKE_COMPILER_IS_GNUCXX)
     set(TENSORRT_LIBS ${TENSORRT_LIBS} stdc++fs nvinfer_plugin nvonnxparser)

src/Detector/tensorrt_yolo/YoloONNX.cpp

@@ -22,14 +22,13 @@ bool YoloONNX::Init(const SampleYoloParams& params)
 
     auto GetBindings = [&]()
     {
-        auto numBindings = m_engine->getNbBindings();
+        auto numBindings = m_engine->getNbIOTensors();
 
         std::cout << "** Bindings: " << numBindings << " **" << std::endl;
         for (int32_t i = 0; i < numBindings; ++i)
         {
-            nvinfer1::Dims dim = m_engine->getBindingDimensions(i);
-
-            std::string bindName = m_engine->getBindingName(i);
+            std::string bindName = m_engine->getIOTensorName(i);
+            nvinfer1::Dims dim = m_engine->getTensorShape(bindName.c_str());
             for (const auto& outName : m_params.outputTensorNames)
             {
                 if (bindName == outName)
@@ -77,27 +76,17 @@ bool YoloONNX::Init(const SampleYoloParams& params)
         delete infer;
 #endif
 
-        sample::gLogInfo << "TRT Engine loaded from: " << m_params.engineFileName << std::endl;
-
-        GetBindings();
-
-        if (!m_engine)
+        if (m_engine)
         {
-            res = false;
+            GetBindings();
+            m_inputDims = m_engine->getTensorShape(m_engine->getIOTensorName(0));
+            res = true;
         }
         else
         {
-#if 1
-            m_inputDims = m_engine->getBindingDimensions(0);
-#else
-            m_inputDims.nbDims = 4;
-            m_inputDims.d[0] = m_params.explicitBatchSize;
-            m_inputDims.d[1] = 3;
-            m_inputDims.d[2] = m_params.width;
-            m_inputDims.d[3] = m_params.height;
-#endif
             res = true;
         }
+        sample::gLogInfo << "TRT Engine loaded from: " << m_params.engineFileName << " with res = " << res << std::endl;
     }
     else
     {
@@ -177,7 +166,7 @@ bool YoloONNX::ConstructNetwork(YoloONNXUniquePtr<nvinfer1::IBuilder>& builder,
     size_t dlaGlobalDRAMSize = config->getMemoryPoolLimit(nvinfer1::MemoryPoolType::kDLA_GLOBAL_DRAM);
 	std::cout << "workspaceSize = " << workspaceSize << ", dlaManagedSRAMSize = " << dlaManagedSRAMSize << ", dlaLocalDRAMSize = " << dlaLocalDRAMSize << ", dlaGlobalDRAMSize = " << dlaGlobalDRAMSize << std::endl;
 
-    config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, m_params.videoMemory ? m_params.videoMemory : 4096_MiB);
+    config->setMemoryPoolLimit(nvinfer1::MemoryPoolType::kWORKSPACE, m_params.videoMemory ? m_params.videoMemory : (1 << 20));
 #endif
 
     config->setFlag(nvinfer1::BuilderFlag::kGPU_FALLBACK);

src/Detector/tensorrt_yolo/common/BatchStream.h

@@ -1,11 +1,12 @@
 /*
- * Copyright (c) 1993-2022, NVIDIA CORPORATION. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -119,7 +120,7 @@ class MNISTBatchStream : public IBatchStream
         file.read(reinterpret_cast<char*>(rawData.data()), numElements * sizeof(uint8_t));
         mData.resize(numElements);
         std::transform(
-            rawData.begin(), rawData.end(), mData.begin(), [](uint8_t val) { return static_cast<float>(val) / 255.f; });
+            rawData.begin(), rawData.end(), mData.begin(), [](uint8_t val) { return static_cast<float>(val) / 255.F; });
     }
 
     void readLabelsFile(const std::string& labelsFilePath)
@@ -152,42 +153,39 @@ class MNISTBatchStream : public IBatchStream
 class BatchStream : public IBatchStream
 {
 public:
-    BatchStream(
-        int batchSize, int maxBatches, std::string prefix, std::string suffix, std::vector<std::string> directories)
+    BatchStream(int batchSize, int maxBatches, std::string const& prefix, std::string const& suffix,
+        std::vector<std::string> const& directories)
         : mBatchSize(batchSize)
         , mMaxBatches(maxBatches)
         , mPrefix(prefix)
         , mSuffix(suffix)
         , mDataDir(directories)
     {
-        FILE* file = fopen(locateFile(mPrefix + std::string("0") + mSuffix, mDataDir).c_str(), "rb");
-        ASSERT(file != nullptr);
+        std::ifstream file(locateFile(mPrefix + std::string("0") + mSuffix, mDataDir).c_str(), std::ios::binary);
+        ASSERT(file.good());
         int d[4];
-        size_t readSize = fread(d, sizeof(int), 4, file);
-        ASSERT(readSize == 4);
+        file.read(reinterpret_cast<char*>(d), 4 * sizeof(int32_t));
         mDims.nbDims = 4;  // The number of dimensions.
         mDims.d[0] = d[0]; // Batch Size
         mDims.d[1] = d[1]; // Channels
         mDims.d[2] = d[2]; // Height
         mDims.d[3] = d[3]; // Width
         ASSERT(mDims.d[0] > 0 && mDims.d[1] > 0 && mDims.d[2] > 0 && mDims.d[3] > 0);
-        fclose(file);
 
         mImageSize = mDims.d[1] * mDims.d[2] * mDims.d[3];
         mBatch.resize(mBatchSize * mImageSize, 0);
         mLabels.resize(mBatchSize, 0);
         mFileBatch.resize(mDims.d[0] * mImageSize, 0);
         mFileLabels.resize(mDims.d[0], 0);
-        reset(0);
     }
 
-    BatchStream(int batchSize, int maxBatches, std::string prefix, std::vector<std::string> directories)
+    BatchStream(int batchSize, int maxBatches, std::string const& prefix, std::vector<std::string> const& directories)
         : BatchStream(batchSize, maxBatches, prefix, ".batch", directories)
     {
     }
 
-    BatchStream(
-        int batchSize, int maxBatches, nvinfer1::Dims dims, std::string listFile, std::vector<std::string> directories)
+    BatchStream(int batchSize, int maxBatches, nvinfer1::Dims const& dims, std::string const& listFile,
+        std::vector<std::string> const& directories)
         : mBatchSize(batchSize)
         , mMaxBatches(maxBatches)
         , mDims(dims)
@@ -199,7 +197,6 @@ class BatchStream : public IBatchStream
         mLabels.resize(mBatchSize, 0);
         mFileBatch.resize(mDims.d[0] * mImageSize, 0);
         mFileLabels.resize(mDims.d[0], 0);
-        reset(0);
     }
 
     // Resets data members
@@ -219,7 +216,7 @@ class BatchStream : public IBatchStream
             return false;
         }
 
-        for (int csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize)
+        for (int64_t csize = 1, batchPos = 0; batchPos < mBatchSize; batchPos += csize, mFileBatchPos += csize)
         {
             ASSERT(mFileBatchPos > 0 && mFileBatchPos <= mDims.d[0]);
             if (mFileBatchPos == mDims.d[0] && !update())
@@ -228,7 +225,7 @@ class BatchStream : public IBatchStream
             }
 
             // copy the smaller of: elements left to fulfill the request, or elements left in the file buffer.
-            csize = std::min(mBatchSize - batchPos, mDims.d[0] - mFileBatchPos);
+            csize = std::min<int64_t>(mBatchSize - batchPos, mDims.d[0] - mFileBatchPos);
             std::copy_n(
                 getFileBatch() + mFileBatchPos * mImageSize, csize * mImageSize, getBatch() + batchPos * mImageSize);
             std::copy_n(getFileLabels() + mFileBatchPos, csize, getLabels() + batchPos);
@@ -295,22 +292,16 @@ class BatchStream : public IBatchStream
         if (mListFile.empty())
         {
             std::string inputFileName = locateFile(mPrefix + std::to_string(mFileCount++) + mSuffix, mDataDir);
-            FILE* file = fopen(inputFileName.c_str(), "rb");
+            std::ifstream file(inputFileName.c_str(), std::ios::binary);
             if (!file)
             {
                 return false;
             }
-
             int d[4];
-            size_t readSize = fread(d, sizeof(int), 4, file);
-            ASSERT(readSize == 4);
+            file.read(reinterpret_cast<char*>(d), 4 * sizeof(int32_t));
             ASSERT(mDims.d[0] == d[0] && mDims.d[1] == d[1] && mDims.d[2] == d[2] && mDims.d[3] == d[3]);
-            size_t readInputCount = fread(getFileBatch(), sizeof(float), mDims.d[0] * mImageSize, file);
-            ASSERT(readInputCount == size_t(mDims.d[0] * mImageSize));
-            size_t readLabelCount = fread(getFileLabels(), sizeof(float), mDims.d[0], file);
-            ASSERT(readLabelCount == 0 || readLabelCount == size_t(mDims.d[0]));
-
-            fclose(file);
+            file.read(reinterpret_cast<char*>(getFileBatch()), sizeof(float) * mDims.d[0] * mImageSize);
+            file.read(reinterpret_cast<char*>(getFileLabels()), sizeof(float) * mDims.d[0]);
         }
         else
         {
@@ -368,7 +359,7 @@ class BatchStream : public IBatchStream
         return true;
     }
 
-    int mBatchSize{0};
+    int64_t mBatchSize{0};
     int mMaxBatches{0};
     int mBatchCount{0};
     int mFileCount{0};

src/Detector/tensorrt_yolo/common/EntropyCalibrator.h

@@ -1,11 +1,12 @@
 /*
- * Copyright (c) 1993-2022, NVIDIA CORPORATION. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -28,8 +29,8 @@ template <typename TBatchStream>
 class EntropyCalibratorImpl
 {
 public:
-    EntropyCalibratorImpl(
-        TBatchStream stream, int firstBatch, std::string networkName, const char* inputBlobName, bool readCache = true)
+    EntropyCalibratorImpl(TBatchStream const& stream, int firstBatch, std::string const& networkName,
+        const char* inputBlobName, bool readCache = true)
         : mStream{stream}
         , mCalibrationTableName("CalibrationTable" + networkName)
         , mInputBlobName(inputBlobName)
@@ -51,11 +52,12 @@ class EntropyCalibratorImpl
         return mStream.getBatchSize();
     }
 
-    bool getBatch(void* bindings[], const char* names[], int /*nbBindings*/) noexcept
+    bool getBatch(void* bindings[], const char* names[], int nbBindings) noexcept
     {
         if (!mStream.next())
+        {
             return false;
-
+        }
         CHECK(cudaMemcpy(mDeviceInput, mStream.getBatch(), mInputCount * sizeof(float), cudaMemcpyHostToDevice));
         ASSERT(!strcmp(names[0], mInputBlobName));
         bindings[0] = mDeviceInput;
@@ -101,8 +103,8 @@ template <typename TBatchStream>
 class Int8EntropyCalibrator2 : public nvinfer1::IInt8EntropyCalibrator2
 {
 public:
-    Int8EntropyCalibrator2(
-        TBatchStream stream, int firstBatch, const char* networkName, const char* inputBlobName, bool readCache = true)
+    Int8EntropyCalibrator2(TBatchStream const& stream, int32_t firstBatch, const char* networkName,
+        const char* inputBlobName, bool readCache = true)
         : mImpl(stream, firstBatch, networkName, inputBlobName, readCache)
     {
     }

src/Detector/tensorrt_yolo/common/ErrorRecorder.h

@@ -1,11 +1,12 @@
 /*
- * Copyright (c) 1993-2022, NVIDIA CORPORATION. All rights reserved.
+ * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
- *     http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
@@ -16,7 +17,7 @@
 
 #ifndef ERROR_RECORDER_H
 #define ERROR_RECORDER_H
-#include "NvInferRuntimeCommon.h"
+#include "NvInferRuntime.h"
 #include "logger.h"
 #include <atomic>
 #include <cstdint>
@@ -44,7 +45,7 @@ class SampleErrorRecorder : public IErrorRecorder
 public:
     SampleErrorRecorder() = default;
 
-    virtual ~SampleErrorRecorder() noexcept {}
+    ~SampleErrorRecorder() noexcept override {}
     int32_t getNbErrors() const noexcept final
     {
         return mErrorStack.size();