From 3932523864433d9d799398389731f1210830c9fa Mon Sep 17 00:00:00 2001 From: gineshidalgo99 Date: Fri, 26 Oct 2018 20:21:02 -0400 Subject: [PATCH] Array adapted for Pybind11 and NMS bug fixed --- README.md | 2 +- doc/faq.md | 10 +- doc/release_notes.md | 7 +- include/openpose/core/array.hpp | 43 +++- include/openpose/pose/enumClasses.hpp | 1 + .../openpose/pose/poseParametersRender.hpp | 37 ++++ src/openpose/core/array.cpp | 187 ++++++++++++------ src/openpose/hand/handExtractorCaffe.cpp | 4 +- src/openpose/net/nmsBase.cu | 3 +- src/openpose/pose/poseParameters.cpp | 55 +++++- src/openpose/pose/poseParametersRender.cpp | 3 + src/openpose/pose/renderPose.cu | 35 ++++ src/openpose/utilities/flagsToOpenPose.cpp | 2 + 13 files changed, 305 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index c880feb16..e37d8248e 100644 --- a/README.md +++ b/README.md @@ -185,4 +185,4 @@ Please cite these papers in your publications if it helps your research (the fac ## License -OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. [Interested in a commercial license? Check this link](https://flintbox.com/public/project/47343/). For commercial queries, contact [Yaser Sheikh](http://www.cs.cmu.edu/~yaser/). +OpenPose is freely available for free non-commercial use, and may be redistributed under these conditions. Please, see the [license](LICENSE) for further details. Interested in a commercial license? Check this [FlintBox link](https://flintbox.com/public/project/47343/). For commercial queries, check the contact information provided in the [FlintBox link](https://flintbox.com/public/project/47343/) and also cc [Yaser Sheikh](http://www.cs.cmu.edu/~yaser/) in that email. diff --git a/doc/faq.md b/doc/faq.md index 04e78e4b6..09d7d7fb9 100644 --- a/doc/faq.md +++ b/doc/faq.md @@ -13,6 +13,7 @@ OpenPose - Frequently Asked Question (FAQ) 8. [Source Directory does not Contain CMakeLists.txt (Windows)](#source-directory-does-not-contain-cmakelists.txt-windows) 9. [How Should I Link my IP Camera?](#how-should-i-link-my-ip-camera) 10. [Difference between BODY_25 vs. COCO vs. MPI](#difference-between-body_25-vs.-coco-vs.-mpi) + 11. [How to Measure the Latency Time?](#how-to-measure-the-latency-time) @@ -87,7 +88,7 @@ Note: OpenPose library is not an executable, but a library. So instead clicking ### How Should I Link my IP Camera? -**Q: How Should I Link my IP Camera with http protocol?.** +**Q: How Should I Link my IP Camera with http protocol?** **A**: Usually with `http://CamIP:PORT_NO./video?x.mjpeg`. @@ -95,3 +96,10 @@ Note: OpenPose library is not an executable, but a library. So instead clicking ### Difference between BODY_25 vs. COCO vs. MPI COCO model will eventually be removed. BODY_25 model is faster, more accurate, and it includes foot keypoints. However, COCO requires less memory on GPU (being able to fit into 2GB GPUs with the default settings) and it runs faster on CPU-only mode. MPI model is only meant for people requiring the MPI-keypoint structure. It is also slower than BODY_25 and far less accurate. + + + +### How to Measure the Latency Time? +**Q: How to measure/calculate/estimate the latency/lag time?** + +**A**: [Profile](https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/installation.md#profiling-speed) the OpenPose speed. For 1-GPU or CPU-only systems (use `--disable_multi_thread` for simplicity in multi-GPU systems for latency measurement), the latency will be roughly the sum of all the reported measurements. diff --git a/doc/release_notes.md b/doc/release_notes.md index ed02c6f70..2c82d4723 100644 --- a/doc/release_notes.md +++ b/doc/release_notes.md @@ -275,13 +275,18 @@ OpenPose Library - Release Notes 8. Auxiliary classes in errorAndLog turned into namespaces (Profiler must be kept as class to allow static parameters). 9. Added flag `--frame_step` to allow the user to select the step or gap between processed frames. E.g., `--frame_step 5` would read and process frames 0, 5, 10, etc. 10. Added sanity checks to avoid `--frame_last` to be smaller than `--frame_first` or higher than the number of total frames. - 11. Added Array::getStride() to get step size of each dimension of the array. + 11. Array improvements for Pybind11 compatibility: + 1. Array::getStride() to get step size of each dimension of the array. + 2. Array::getPybindPtr() to get an editable const pointer. + 3. Array::pData as binding of spData. + 4. Array::Array that takes as input a pointer, so it does not re-allocate memory. 2. Functions or parameters renamed: 1. By default, python example `tutorial_developer/python_2_pose_from_heatmaps.py` was using 2 scales starting at -1x736, changed to 1 scale at -1x368. 2. WrapperStructPose default parameters changed to match those of the OpenPose demo binary. 3. WrapperT.configure() changed from 1 function that requries all arguments to individual functions that take 1 argument each. 3. Main bugs fixed: 1. CMake-GUI was forcing to Release mode, allowed Debug modes too. + 2. NMS returns in index 0 the number of found peaks. However, while the number of peaks was truncated to a maximum of 127, this index 0 was saving the real number instead of the truncated one. diff --git a/include/openpose/core/array.hpp b/include/openpose/core/array.hpp index 2169de749..4cb0e9705 100644 --- a/include/openpose/core/array.hpp +++ b/include/openpose/core/array.hpp @@ -55,6 +55,24 @@ namespace op */ Array(const std::vector& sizes, const T value); + /** + * Array constructor. + * Equivalent to default constructor, but it does not allocate memory, but rather use dataPtr. + * @param size Integer with the number of T element to be allocated. E.g., size = 5 is internally similar to + * `new T[5]`. + * @param dataPtr Pointer to the memory to be used by the Array. + */ + Array(const int size, T* const dataPtr); + + /** + * Array constructor. + * Equivalent to default constructor, but it does not allocate memory, but rather use dataPtr. + * @param sizes Vector with the size of each dimension. E.g., size = {3, 5, 2} is internally similar to: + * `new T[3*5*2]`. + * @param dataPtr Pointer to the memory to be used by the Array. + */ + Array(const std::vector& sizes, T* const dataPtr); + /** * Copy constructor. * It performs `fast copy`: For performance purpose, copying a Array or Datum or cv::Mat just copies the @@ -236,7 +254,7 @@ namespace op */ inline T* getPtr() { - return spData.get(); + return pData; // spData.get() } /** @@ -245,7 +263,17 @@ namespace op */ inline const T* getConstPtr() const { - return spData.get(); + return pData; // spData.get() + } + + /** + * Similar to getConstPtr(), but it allows the data to be edited. + * This function is only implemented for Pybind11 usage. + * @return A raw pointer to the data. + */ + inline T* getPybindPtr() const + { + return pData; // spData.get() } /** @@ -282,7 +310,7 @@ namespace op inline T& operator[](const int index) { #ifdef NDEBUG - return spData.get()[index]; + return pData[index]; // spData.get()[index] #else return at(index); #endif @@ -298,7 +326,7 @@ namespace op inline const T& operator[](const int index) const { #ifdef NDEBUG - return spData.get()[index]; + return pData[index]; // spData.get()[index] #else return at(index); #endif @@ -395,6 +423,7 @@ namespace op std::vector mSize; size_t mVolume; std::shared_ptr spData; + T* pData; // pData is a wrapper of spData. Used for Pybind11 binding. std::pair mCvMatData; /** @@ -422,11 +451,7 @@ namespace op */ T& commonAt(const int index) const; - /** - * Private auxiliar function that sets the cv::Mat wrapper and makes it point to the same data than - * std::shared_ptr points to. - */ - void setCvMatFromSharedPtr(); + void resetAuxiliary(const std::vector& sizes, T* const dataPtr = nullptr); }; // Static methods diff --git a/include/openpose/pose/enumClasses.hpp b/include/openpose/pose/enumClasses.hpp index 71a7b50a9..179abeabd 100644 --- a/include/openpose/pose/enumClasses.hpp +++ b/include/openpose/pose/enumClasses.hpp @@ -25,6 +25,7 @@ namespace op CAR_12, /**< Experimental. Do not use. */ BODY_25D, /**< Experimental. Do not use. */ BODY_23, /**< Experimental. Do not use. */ + CAR_22, /**< Experimental. Do not use. */ Size, }; diff --git a/include/openpose/pose/poseParametersRender.hpp b/include/openpose/pose/poseParametersRender.hpp index 2c61092a1..8e4d8a29f 100644 --- a/include/openpose/pose/poseParametersRender.hpp +++ b/include/openpose/pose/poseParametersRender.hpp @@ -322,6 +322,43 @@ namespace op 255.f, 0.f, 255.f, \ 255.f, 0.f, 255.f + // CAR_22 + #define POSE_CAR_22_PAIRS_RENDER_GPU \ + 0,1,1,3,3,2,2,0, 6,7,7,16,16,17,17,6, 12,13,13,14,14,15,15,12, 6,8,7,8,6,9,7,9,6,4,7,5, 12,11,13,10, \ + 16,18,17,18,16,19,17,19, 6,21,7,20 + #define POSE_CAR_22_SCALES_RENDER_GPU 0.625 + #define POSE_CAR_22_COLORS_RENDER_GPU \ + 255.f, 128.f, 128.f, \ + 255.f, 0.f, 0.f, \ + 64.f, 0.f, 0.f, \ + 255.f, 0.f, 0.f, \ + \ + 0.f, 255.f, 0.f, \ + 0.f, 255.f, 0.f, \ + \ + 0.f, 0.f, 64.f, \ + 128.f, 128.f, 255.f, \ + \ + 0.f, 255.f, 0.f, \ + 0.f, 255.f, 0.f, \ + \ + 0.f, 255.f, 0.f, \ + 0.f, 255.f, 0.f, \ + \ + 64.f, 0.f, 0.f, \ + 255.f, 128.f, 128.f, \ + 255.f, 0.f, 0.f, \ + 255.f, 0.f, 0.f, \ + \ + 0.f, 0.f, 255.f, \ + 0.f, 0.f, 255.f, \ + \ + 0.f, 255.f, 0.f, \ + 0.f, 255.f, 0.f, \ + \ + 0.f, 0.f, 255.f, \ + 0.f, 0.f, 64.f + // Rendering functions OP_API const std::vector& getPoseScales(const PoseModel poseModel); OP_API const std::vector& getPoseColors(const PoseModel poseModel); diff --git a/src/openpose/core/array.cpp b/src/openpose/core/array.cpp index d1a42b1ec..0f90da7fb 100644 --- a/src/openpose/core/array.cpp +++ b/src/openpose/core/array.cpp @@ -16,6 +16,66 @@ namespace op { + /** + * Private auxiliar function that sets the cv::Mat wrapper and makes it point to the same data than + * std::shared_ptr points to. + */ + template + void setCvMatFromPtr(std::pair& cvMatData, T* const dataPtr, const std::vector& sizes) + { + try + { + cvMatData.first = true; + cvMatData.second = cv::Mat(); + // BGR image + if (sizes.size() == 3 && sizes[2] == 3) + { + // Prepare cv::Mat + auto cvFormat = CV_32FC3; + if (typeid(T) == typeid(float)) + cvFormat = CV_32FC3; + else if (typeid(T) == typeid(double)) + cvFormat = CV_64FC3; + else if (typeid(T) == typeid(unsigned char)) + cvFormat = CV_8UC3; + else if (typeid(T) == typeid(signed char)) + cvFormat = CV_8SC3; + else if (typeid(T) == typeid(int)) + cvFormat = CV_32SC3; + else + cvMatData.first = false; + + if (cvMatData.first) + cvMatData.second = cv::Mat(sizes[0], sizes[1], cvFormat, dataPtr); + } + // Any other type + else + { + // Prepare cv::Mat + auto cvFormat = CV_32FC1; + if (typeid(T) == typeid(float)) + cvFormat = CV_32FC1; + else if (typeid(T) == typeid(double)) + cvFormat = CV_64FC1; + else if (typeid(T) == typeid(unsigned char)) + cvFormat = CV_8UC1; + else if (typeid(T) == typeid(signed char)) + cvFormat = CV_8SC1; + else if (typeid(T) == typeid(int)) + cvFormat = CV_32SC1; + else + cvMatData.first = false; + + if (cvMatData.first) + cvMatData.second = cv::Mat((int)sizes.size(), sizes.data(), cvFormat, dataPtr); + } + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + template Array::Array(const int size) { @@ -68,11 +128,44 @@ namespace op } } + template + Array::Array(const int size, T* const dataPtr) + { + try + { + if (size > 0) + resetAuxiliary(std::vector{size}, dataPtr); + else + error("Size cannot be less than 1.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + + template + Array::Array(const std::vector& sizes, T* const dataPtr) + { + try + { + if (!sizes.empty()) + resetAuxiliary(sizes, dataPtr); + else + error("Size cannot be empty or less than 1.", __LINE__, __FUNCTION__, __FILE__); + } + catch (const std::exception& e) + { + error(e.what(), __LINE__, __FUNCTION__, __FILE__); + } + } + template Array::Array(const Array& array) : mSize{array.mSize}, mVolume{array.mVolume}, spData{array.spData}, + pData{array.pData}, mCvMatData{array.mCvMatData} { } @@ -85,6 +178,7 @@ namespace op mSize = array.mSize; mVolume = array.mVolume; spData = array.spData; + pData = array.pData; mCvMatData = array.mCvMatData; // Return return *this; @@ -104,6 +198,7 @@ namespace op try { std::swap(spData, array.spData); + std::swap(pData, array.pData); std::swap(mCvMatData, array.mCvMatData); } catch (const std::exception& e) @@ -120,6 +215,7 @@ namespace op mSize = array.mSize; mVolume = array.mVolume; std::swap(spData, array.spData); + std::swap(pData, array.pData); std::swap(mCvMatData, array.mCvMatData); // Return return *this; @@ -139,7 +235,8 @@ namespace op // Constructor Array array{mSize}; // Clone data - std::copy(spData.get(), spData.get() + mVolume, array.spData.get()); + // Equivalent: std::copy(spData.get(), spData.get() + mVolume, array.spData.get()); + std::copy(pData, pData + mVolume, array.pData); // Return return std::move(array); } @@ -171,23 +268,7 @@ namespace op { try { - if (!sizes.empty()) - { - // New size & volume - mSize = sizes; - mVolume = {std::accumulate(sizes.begin(), sizes.end(), 1ul, std::multiplies())}; - // Prepare shared_ptr - spData.reset(new T[mVolume], std::default_delete()); - setCvMatFromSharedPtr(); - } - else - { - mSize = {}; - mVolume = 0ul; - spData.reset(); - // cv::Mat available but empty - mCvMatData = std::make_pair(true, cv::Mat()); - } + resetAuxiliary(sizes); } catch (const std::exception& e) { @@ -428,11 +509,10 @@ namespace op // Initial value std::string string{"Array::toString():\n"}; // Add each element - const auto* dataPtr = spData.get(); for (auto i = 0u ; i < mVolume ; i++) { // Adding element separated by a space - string += std::to_string(dataPtr[i]) + " "; + string += std::to_string(pData[i]) + " "; // Introduce an enter for each dimension change // If comented, all values will be printed in the same line auto multiplier = 1; @@ -496,68 +576,51 @@ namespace op try { if (0 <= index && (size_t)index < mVolume) - return spData.get()[index]; + return pData[index]; // spData.get()[index] else { error("Index out of bounds: 0 <= index && index < mVolume", __LINE__, __FUNCTION__, __FILE__); - return spData.get()[0]; + return pData[0]; // spData.get()[0] } } catch (const std::exception& e) { error(e.what(), __LINE__, __FUNCTION__, __FILE__); - return spData.get()[0]; + return pData[0]; // spData.get()[0] } } template - void Array::setCvMatFromSharedPtr() + void Array::resetAuxiliary(const std::vector& sizes, T* const dataPtr) { try { - mCvMatData.first = true; - mCvMatData.second = cv::Mat(); - // BGR image - if (mSize.size() == 3 && mSize[2] == 3) + if (!sizes.empty()) { - // Prepare cv::Mat - auto cvFormat = CV_32FC3; - if (typeid(T) == typeid(float)) - cvFormat = CV_32FC3; - else if (typeid(T) == typeid(double)) - cvFormat = CV_64FC3; - else if (typeid(T) == typeid(unsigned char)) - cvFormat = CV_8UC3; - else if (typeid(T) == typeid(signed char)) - cvFormat = CV_8SC3; - else if (typeid(T) == typeid(int)) - cvFormat = CV_32SC3; + // New size & volume + mSize = sizes; + mVolume = {std::accumulate(sizes.begin(), sizes.end(), 1ul, std::multiplies())}; + // Prepare shared_ptr + if (dataPtr == nullptr) + { + spData.reset(new T[mVolume], std::default_delete()); + pData = spData.get(); + } else - mCvMatData.first = false; - - if (mCvMatData.first) - mCvMatData.second = cv::Mat(mSize[0], mSize[1], cvFormat, spData.get()); + { + spData.reset(); + pData = dataPtr; + } + setCvMatFromPtr(mCvMatData, pData, mSize); // spData.get() } - // Any other type else { - // Prepare cv::Mat - auto cvFormat = CV_32FC1; - if (typeid(T) == typeid(float)) - cvFormat = CV_32FC1; - else if (typeid(T) == typeid(double)) - cvFormat = CV_64FC1; - else if (typeid(T) == typeid(unsigned char)) - cvFormat = CV_8UC1; - else if (typeid(T) == typeid(signed char)) - cvFormat = CV_8SC1; - else if (typeid(T) == typeid(int)) - cvFormat = CV_32SC1; - else - mCvMatData.first = false; - - if (mCvMatData.first) - mCvMatData.second = cv::Mat((int)mSize.size(), mSize.data(), cvFormat, spData.get()); + mSize = {}; + mVolume = 0ul; + spData.reset(); + pData = nullptr; + // cv::Mat available but empty + mCvMatData = std::make_pair(true, cv::Mat()); } } catch (const std::exception& e) diff --git a/src/openpose/hand/handExtractorCaffe.cpp b/src/openpose/hand/handExtractorCaffe.cpp index b56476992..d091dbc95 100644 --- a/src/openpose/hand/handExtractorCaffe.cpp +++ b/src/openpose/hand/handExtractorCaffe.cpp @@ -334,8 +334,8 @@ namespace op const auto scale = initScale + mMultiScaleNumberAndRange.second * i / (numberScales-1.f); // Process hand - Array handEstimated({1, handCurrent.getSize(1), - handCurrent.getSize(2)}, 0); + Array handEstimated( + {1, handCurrent.getSize(1), handCurrent.getSize(2)}, 0.f); const auto handRectangleScale = recenter( handRectangle, (float)(intRound(handRectangle.width * scale) / 2 * 2), diff --git a/src/openpose/net/nmsBase.cu b/src/openpose/net/nmsBase.cu index c1c7c58a6..bab2cbcc6 100644 --- a/src/openpose/net/nmsBase.cu +++ b/src/openpose/net/nmsBase.cu @@ -112,8 +112,9 @@ namespace op } } } + // If index 0 --> Assign number of peaks (truncated to the maximum possible number of peaks) else - output[0] = kernelPtr[globalIdx]; //number of peaks + output[0] = (kernelPtr[globalIdx] < maxPeaks ? kernelPtr[globalIdx] : maxPeaks); } } diff --git a/src/openpose/pose/poseParameters.cpp b/src/openpose/pose/poseParameters.cpp index 20ec0e4fc..9586a2025 100644 --- a/src/openpose/pose/poseParameters.cpp +++ b/src/openpose/pose/poseParameters.cpp @@ -223,6 +223,31 @@ namespace op {11, "BLTop"}, {12, "Background"}, }; + const std::map POSE_CAR_22_PARTS { + {0, "FLWheel"}, + {1, "BLWheel"}, + {2, "FRWheel"}, + {3, "BRWheel"}, + {4, "FRFogLight"}, + {5, "FLFogLight"}, + {6, "FRLight"}, + {7, "FLLight"}, + {8, "Grilles"}, + {9, "FBumper"}, + {10, "LMirror"}, + {11, "RMirror"}, + {12, "FRTop"}, + {13, "FLTop"}, + {14, "BLTop"}, + {15, "BRTop"}, + {16, "BLLight"}, + {17, "BRLight"}, + {18, "Trunk"}, + {19, "BBumper"}, + {20, "BLCorner"}, + {21, "BRCorner"}, + {22, "Background"}, + }; const std::array, (int)PoseModel::Size> POSE_MAP_INDEX{ // BODY_25 std::vector{ @@ -294,13 +319,18 @@ namespace op // Redundant ones 44,45, 46,47, 48,49, 50,51, 52,53, 54,55, 56,57, 58,59, 60,61, 62,63, 64,65, 66,67, 68,69, 70,71, 72,73 }, + // CAR_22 + std::vector{ + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37, + 38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63 + }, }; // POSE_BODY_PART_MAPPING on HPP crashes on Windows at dynamic initialization if it's on hpp const std::array, (int)PoseModel::Size> POSE_BODY_PART_MAPPING{ POSE_BODY_25_BODY_PARTS,POSE_COCO_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_MPI_BODY_PARTS, POSE_BODY_19_BODY_PARTS,POSE_BODY_19_BODY_PARTS,POSE_BODY_59_BODY_PARTS,POSE_BODY_19_BODY_PARTS, POSE_BODY_25_BODY_PARTS,POSE_BODY_25_BODY_PARTS,POSE_BODY_65_BODY_PARTS,POSE_CAR_12_PARTS, - POSE_BODY_25_BODY_PARTS,POSE_BODY_23_BODY_PARTS + POSE_BODY_25_BODY_PARTS,POSE_BODY_23_BODY_PARTS,POSE_CAR_22_PARTS }; const std::array POSE_PROTOTXT{ @@ -318,6 +348,7 @@ namespace op "car/car_12/pose_deploy.prototxt", "pose/body_25d/pose_deploy.prototxt", "pose/body_23/pose_deploy.prototxt", + "car/car_22/pose_deploy.prototxt", }; const std::array POSE_TRAINED_MODEL{ "pose/body_25/pose_iter_584000.caffemodel", @@ -334,12 +365,13 @@ namespace op "car/car_12/pose_iter_XXXXXX.caffemodel", "pose/body_25d/pose_iter_XXXXXX.caffemodel", "pose/body_23/pose_iter_XXXXXX.caffemodel", + "car/car_22/pose_iter_XXXXXX.caffemodel", }; // Constant Array Parameters // POSE_NUMBER_BODY_PARTS equivalent to size of std::map POSE_BODY_XX_BODY_PARTS - 1 (removing background) const std::array POSE_NUMBER_BODY_PARTS{ - 25, 18, 15, 15, 19, 19, 59, 19, 25, 25, 65, 12, 25, 23 + 25, 18, 15, 15, 19, 19, 59, 19, 25, 25, 65, 12, 25, 23, 22 }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS{ // BODY_25 @@ -408,6 +440,13 @@ namespace op // Ears-shoulders, ears, shoulders-wrists, hips-ankles, wrists, ankles, wrists-hips, small toes-ankles, hips) 1,15, 4,16, 15,16, 1,3, 4,6, 7,9, 10,12, 3,6, 9,12, 3,7, 6,10, 9,21, 12,18, 7,10 }, + // CAR_22 + std::vector{ + // Wheels Lights Top Front Mirrors + 0,1,1,3,3,2,2,0, 6,7,7,16,16,17,17,6, 12,13,13,14,14,15,15,12, 6,8,7,8,6,9,7,9,6,4,7,5, 12,11,13,10, + // Back Vertical Back-light replacement + 16,18,17,18,16,19,17,19, 0,7,3,17,6,12,16,14, 6,21,7,20,3,21,20,14 + }, }; const std::array POSE_MAX_PEAKS{ POSE_MAX_PEOPLE, // BODY_25 @@ -424,6 +463,7 @@ namespace op POSE_MAX_PEOPLE, // CAR_12 POSE_MAX_PEOPLE, // BODY_25D POSE_MAX_PEOPLE, // BODY_23 + POSE_MAX_PEOPLE, // CAR_22 }; const std::array POSE_CCN_DECREASE_FACTOR{ 8.f, // BODY_25 @@ -440,30 +480,31 @@ namespace op 8.f, // CAR_12 8.f, // BODY_25D 8.f, // BODY_23 + 8.f, // CAR_22 }; // Default Model Parameters // They might be modified on running time const auto nmsT = (COCO_CHALLENGE ? 0.02f : 0.05f); const std::array POSE_DEFAULT_NMS_THRESHOLD{ - nmsT, nmsT, 0.6f, 0.3f, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT + nmsT, nmsT, 0.6f, 0.3f, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT, nmsT }; const auto minAT = (COCO_CHALLENGE ? 0.75f : 0.95f); // Matlab version: 0.85f const std::array POSE_DEFAULT_CONNECT_INTER_MIN_ABOVE_THRESHOLD{ - minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT + minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT, minAT // 0.85f, 0.85f, 0.85f, 0.85f, 0.85f, 0.85f // Matlab version }; const auto conIT = (COCO_CHALLENGE ? 0.01f : 0.05f); const std::array POSE_DEFAULT_CONNECT_INTER_THRESHOLD{ - conIT, conIT, 0.01f, 0.01f, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT + conIT, conIT, 0.01f, 0.01f, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT, conIT }; const auto minSC = (COCO_CHALLENGE ? 2 : 3); const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_CNT{ - minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC + minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC, minSC }; const auto minSS = (COCO_CHALLENGE ? 0.05f : 0.4f); const std::array POSE_DEFAULT_CONNECT_MIN_SUBSET_SCORE{ - minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS + minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS, minSS // 0.2f, 0.4f, 0.4f, 0.4f, 0.4f, 0.4f // Matlab version }; diff --git a/src/openpose/pose/poseParametersRender.cpp b/src/openpose/pose/poseParametersRender.cpp index 1e33537e7..06020d13a 100644 --- a/src/openpose/pose/poseParametersRender.cpp +++ b/src/openpose/pose/poseParametersRender.cpp @@ -18,6 +18,7 @@ namespace op std::vector{POSE_CAR_12_SCALES_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_SCALES_RENDER_GPU}, // BODY_25D std::vector{POSE_BODY_23_SCALES_RENDER_GPU}, // BODY_23 + std::vector{POSE_CAR_22_SCALES_RENDER_GPU}, // CAR_22 }; const std::array, (int)PoseModel::Size> POSE_COLORS{ std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25 @@ -34,6 +35,7 @@ namespace op std::vector{POSE_CAR_12_COLORS_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_COLORS_RENDER_GPU}, // BODY_25D std::vector{POSE_BODY_23_COLORS_RENDER_GPU}, // BODY_23 + std::vector{POSE_CAR_22_COLORS_RENDER_GPU}, // CAR_22 }; const std::array, (int)PoseModel::Size> POSE_BODY_PART_PAIRS_RENDER{ std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25 @@ -50,6 +52,7 @@ namespace op std::vector{POSE_CAR_12_PAIRS_RENDER_GPU}, // CAR_12 std::vector{POSE_BODY_25_PAIRS_RENDER_GPU}, // BODY_25D std::vector{POSE_BODY_23_PAIRS_RENDER_GPU}, // BODY_23 + std::vector{POSE_CAR_22_PAIRS_RENDER_GPU}, // CAR_22 }; // Rendering functions diff --git a/src/openpose/pose/renderPose.cu b/src/openpose/pose/renderPose.cu index 29f7fdd9a..cd2bac702 100644 --- a/src/openpose/pose/renderPose.cu +++ b/src/openpose/pose/renderPose.cu @@ -17,6 +17,7 @@ namespace op __constant__ const unsigned int BODY_65_PAIRS_GPU[] = {POSE_BODY_65_PAIRS_RENDER_GPU}; __constant__ const unsigned int MPI_PAIRS_GPU[] = {POSE_MPI_PAIRS_RENDER_GPU}; __constant__ const unsigned int CAR_12_PAIRS_GPU[] = {POSE_CAR_12_PAIRS_RENDER_GPU}; + __constant__ const unsigned int CAR_22_PAIRS_GPU[] = {POSE_CAR_22_PAIRS_RENDER_GPU}; // Keypoint scales __constant__ const float BODY_25_SCALES[] = {POSE_BODY_25_SCALES_RENDER_GPU}; __constant__ const float COCO_SCALES[] = {POSE_COCO_SCALES_RENDER_GPU}; @@ -26,6 +27,7 @@ namespace op __constant__ const float BODY_65_SCALES[] = {POSE_BODY_65_SCALES_RENDER_GPU}; __constant__ const float MPI_SCALES[] = {POSE_MPI_SCALES_RENDER_GPU}; __constant__ const float CAR_12_SCALES[] = {POSE_CAR_12_SCALES_RENDER_GPU}; + __constant__ const float CAR_22_SCALES[] = {POSE_CAR_22_SCALES_RENDER_GPU}; // RGB colors __constant__ const float BODY_25_COLORS[] = {POSE_BODY_25_COLORS_RENDER_GPU}; __constant__ const float COCO_COLORS[] = {POSE_COCO_COLORS_RENDER_GPU}; @@ -35,6 +37,7 @@ namespace op __constant__ const float BODY_65_COLORS[] = {POSE_BODY_65_COLORS_RENDER_GPU}; __constant__ const float MPI_COLORS[] = {POSE_MPI_COLORS_RENDER_GPU}; __constant__ const float CAR_12_COLORS[] = {POSE_CAR_12_COLORS_RENDER_GPU}; + __constant__ const float CAR_22_COLORS[] = {POSE_CAR_22_COLORS_RENDER_GPU}; @@ -330,6 +333,33 @@ namespace op blendOriginalFrame, (googlyEyes ? 4 : -1), (googlyEyes ? 5 : -1)); } + __global__ void renderPoseCar22(float* targetPtr, const int targetWidth, const int targetHeight, + const float* const posePtr, const int numberPeople, const float threshold, + const bool googlyEyes, const bool blendOriginalFrame, const float alphaColorToAdd) + { + const auto x = (blockIdx.x * blockDim.x) + threadIdx.x; + const auto y = (blockIdx.y * blockDim.y) + threadIdx.y; + const auto globalIdx = threadIdx.y * blockDim.x + threadIdx.x; + + // Shared parameters + __shared__ float2 sharedMins[POSE_MAX_PEOPLE]; + __shared__ float2 sharedMaxs[POSE_MAX_PEOPLE]; + __shared__ float sharedScaleF[POSE_MAX_PEOPLE]; + + // Other parameters + const auto numberPartPairs = sizeof(CAR_22_PAIRS_GPU) / (2*sizeof(CAR_22_PAIRS_GPU[0])); + const auto numberScales = sizeof(CAR_22_SCALES) / sizeof(CAR_22_SCALES[0]); + const auto numberColors = sizeof(CAR_22_COLORS) / (3*sizeof(CAR_22_COLORS[0])); + const auto radius = fastMin(targetWidth, targetHeight) / 100.f; + const auto lineWidth = fastMin(targetWidth, targetHeight) / 120.f; + + // Render key points + renderKeypoints(targetPtr, sharedMaxs, sharedMins, sharedScaleF, globalIdx, x, y, targetWidth, targetHeight, + posePtr, CAR_22_PAIRS_GPU, numberPeople, 22, numberPartPairs, CAR_22_COLORS, numberColors, + radius, lineWidth, CAR_22_SCALES, numberScales, threshold, alphaColorToAdd, + blendOriginalFrame, (googlyEyes ? 6 : -1), (googlyEyes ? 7 : -1)); + } + __global__ void renderBodyPartHeatMaps(float* targetPtr, const int targetWidth, const int targetHeight, const float* const heatMapPtr, const int widthHeatMap, const int heightHeatMap, const float scaleToKeepRatio, @@ -585,6 +615,11 @@ namespace op framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, blendOriginalFrame, alphaBlending ); + else if (poseModel == PoseModel::CAR_22) + renderPoseCar22<<>>( + framePtr, frameSize.x, frameSize.y, posePtr, numberPeople, renderThreshold, googlyEyes, + blendOriginalFrame, alphaBlending + ); // Unknown else error("Invalid Model.", __LINE__, __FUNCTION__, __FILE__); diff --git a/src/openpose/utilities/flagsToOpenPose.cpp b/src/openpose/utilities/flagsToOpenPose.cpp index fd5388334..f9830bf82 100644 --- a/src/openpose/utilities/flagsToOpenPose.cpp +++ b/src/openpose/utilities/flagsToOpenPose.cpp @@ -44,6 +44,8 @@ namespace op // Car pose else if (poseModeString == "CAR_12") return PoseModel::CAR_12; + else if (poseModeString == "CAR_22") + return PoseModel::CAR_22; // else error("String (`" + poseModeString + "`) does not correspond to any model (BODY_25, COCO, MPI," " MPI_4_layers).", __LINE__, __FUNCTION__, __FILE__);