Skip to content

Commit

Permalink
Add support from an orphan ReadValue
Browse files Browse the repository at this point in the history
  • Loading branch information
@maxnick
maxnick committed Nov 8, 2023
1 parent 03e7bde commit fb60406
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 31 deletions.
10 changes: 1 addition & 9 deletions src/plugins/intel_cpu/src/infer_request.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,16 +70,8 @@ void SyncInferRequest::create_infer_request() {
if (!memoryNode) {
OPENVINO_THROW("Cannot cast ", node->getName(), " to MemoryInput");
}
auto state_name = memoryNode->getId();

// Remove suffix with pair ID. Internal information.
auto suffix_idx = state_name.find("/id=");
if (suffix_idx != std::string::npos) {
state_name = state_name.substr(0, suffix_idx);
}

m_memory_states.emplace_back(
std::make_shared<VariableStateDoubleBuffer>(state_name, memoryNode->memoryBuilder(), memoryNode->getMemoryPtr()));
m_memory_states.emplace_back(memoryNode->makeState());
}
}
}
Expand Down
56 changes: 45 additions & 11 deletions src/plugins/intel_cpu/src/memory_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
//
#include <openvino/runtime/make_tensor.hpp>

#include <nodes/common/cpu_convert.h>

#include "memory_state.h"

#include "dnnl_extension_utils.h"
Expand All @@ -16,12 +18,13 @@ namespace intel_cpu {

VariableStateDoubleBuffer::VariableStateDoubleBuffer(std::string name,
const MemBuilder& mem_build,
MemoryDescPtr external_desc,
MemoryCPtr init_val) :
IVariableState{name} {
IVariableState{name}, m_external_desc{external_desc} {
ResetPrimeMem(mem_build());
ResetSecondMem(mem_build());
m_desc = PrimeMem()->getDescPtr();
auto&& shape = m_desc->getShape();
m_internal_desc = PrimeMem()->getDescPtr();
auto&& shape = m_internal_desc->getShape();
//TODO what if by some reason we already have internal static state while the node is dynamic, is it even possible?

if (shape.isStatic()) {
Expand All @@ -32,7 +35,7 @@ VariableStateDoubleBuffer::VariableStateDoubleBuffer(std::string name,
}
} else {
//in the case of the original desc has dynamic shape we create an empty tensor
auto new_desc = ToStatic(m_desc);
auto new_desc = ToStatic(m_internal_desc);
PrimeMem()->redefineDesc(new_desc);
}
}
Expand All @@ -41,24 +44,55 @@ void VariableStateDoubleBuffer::SetState(const Blob::Ptr& newState) {
state = newState; // simply to extend the lifetime
auto&& tensor_desc = state->getTensorDesc();
if (PrimeMem()->getStaticDims() != tensor_desc.getDims()) {
auto new_desc = m_desc->cloneWithNewDims(tensor_desc.getDims());
auto new_desc = m_internal_desc->cloneWithNewDims(tensor_desc.getDims());
PrimeMem()->redefineDesc(new_desc);
}
auto blob_desc = MemoryDescUtils::convertToCpuBlockedMemoryDesc(tensor_desc);
auto src = state->buffer().as<void*>();

static const dnnl::engine eng(dnnl::engine::kind::cpu, 0);
Memory mem(eng, blob_desc, src);
Memory mem(getEngine(), blob_desc, src);
PrimeMem()->load(mem);
}

const dnnl::engine& VariableStateDoubleBuffer::getEngine() const {
static const dnnl::engine eng(dnnl::engine::kind::cpu, 0);
return eng;
}

Blob::CPtr VariableStateDoubleBuffer::GetState() const {
auto tensor = std::make_shared<Tensor>(PrimeMem());
const auto& current_dims = PrimeMem()->getStaticDims();
auto current_ext_desc = m_external_desc->cloneWithNewDims(current_dims);
auto current_internal_desc = PrimeMem()->getDescPtr();

if (current_ext_desc->isCompatible(*current_internal_desc)) {
auto tensor = std::make_shared<Tensor>(PrimeMem());
return tensor_to_blob({tensor, nullptr}); // TODO: shouldn't we provide the so ptr?
}

//test precision
{
auto internal_prc = current_internal_desc->getPrecision();
auto tmp_desc = current_ext_desc->cloneWithNewPrecision(internal_prc);
if (tmp_desc->isCompatible(*current_internal_desc)) {
auto mem = std::make_shared<Memory>(getEngine(), current_ext_desc);
size_t elements_to_convert = PrimeMem()->getDescWithType<BlockedMemoryDesc>()->getPaddedElementsCount();
auto external_prc = current_ext_desc->getPrecision();

cpu_convert(PrimeMem()->getData(), mem->getData(), internal_prc, external_prc, elements_to_convert);
auto tensor = std::make_shared<Tensor>(mem);
return tensor_to_blob({tensor, nullptr}); // TODO: shouldn't we provide the so ptr?
}
}

//reorder
auto mem = std::make_shared<Memory>(getEngine(), current_ext_desc);
mem->load(*(PrimeMem()));
auto tensor = std::make_shared<Tensor>(mem);
return tensor_to_blob({tensor, nullptr}); // TODO: shouldn't we provide the so ptr?
}

void VariableStateDoubleBuffer::Reset() {
auto new_desc = ToStatic(m_desc);
auto new_desc = ToStatic(m_internal_desc);
for (auto&& mem : m_internal_mem) {
if (mem) {
mem->redefineDesc(new_desc);
Expand Down Expand Up @@ -91,8 +125,8 @@ MemoryPtr VariableStateDoubleBuffer::OutputMem() {
return SecondMem();
}

MemoryDescPtr VariableStateDoubleBuffer::OriginalDesc() const {
return m_desc;
MemoryDescPtr VariableStateDoubleBuffer::InternalDesc() const {
return m_internal_desc;
}

} // namespace intel_cpu
Expand Down
11 changes: 8 additions & 3 deletions src/plugins/intel_cpu/src/memory_state.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ class IVariableState : public ov::IVariableState {

virtual MemoryPtr InputMem() = 0;
virtual MemoryPtr OutputMem() = 0;
virtual MemoryDescPtr OriginalDesc() const = 0;
virtual MemoryDescPtr InternalDesc() const = 0;
};

class VariableStateDoubleBuffer : public IVariableState {
Expand All @@ -35,6 +35,7 @@ class VariableStateDoubleBuffer : public IVariableState {
public:
VariableStateDoubleBuffer(std::string name,
const MemBuilder& mem_build,
MemoryDescPtr external_desc,
MemoryCPtr init_val);
//InferenceEngine::IVariableStateInternal
void Reset() override;
Expand All @@ -46,7 +47,7 @@ class VariableStateDoubleBuffer : public IVariableState {

MemoryPtr InputMem() override;
MemoryPtr OutputMem() override;
MemoryDescPtr OriginalDesc() const override;
MemoryDescPtr InternalDesc() const override;

private:
static MemoryDescPtr ToStatic(const MemoryDescPtr& desc);
Expand All @@ -67,8 +68,12 @@ class VariableStateDoubleBuffer : public IVariableState {
return m_internal_mem[buffer_num ^ 0x1];
}


const dnnl::engine& getEngine() const;

private:
MemoryDescPtr m_desc; //mem desc required by the graph internal tensor
MemoryDescPtr m_external_desc;
MemoryDescPtr m_internal_desc; //mem desc required by the graph internal tensor
std::array<MemoryPtr, 2> m_internal_mem{};
size_t buffer_num = 0;
};
Expand Down
30 changes: 24 additions & 6 deletions src/plugins/intel_cpu/src/nodes/memory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,10 +234,12 @@ MemoryInput::MemoryInput(const std::shared_ptr<ngraph::Node>& op, const GraphCon

void MemoryInput::createPrimitive() {
Input::createPrimitive();
auto parentEdge = getParentEdgeAt(0);
if (!inputShapes.empty()) {
auto parentEdge = getParentEdgeAt(0);

if (parentEdge->getParent()->isConstant()) {
Input::resetMemoryPtr(parentEdge->getMemoryPtr());
if (parentEdge->getParent()->isConstant()) {
Input::resetMemoryPtr(parentEdge->getMemoryPtr());
}
}
}

Expand Down Expand Up @@ -380,13 +382,29 @@ void MemoryInput::assignState(MemStatePtr newState) {
outMem->load(*assignedMem);
}

getOutputNode().assignExtMemory(newState->OutputMem(), newState->OriginalDesc());
getOutputNode().assignExtMemory(newState->OutputMem(), newState->InternalDesc());
}

std::function<MemoryPtr(void)> MemoryInput::memoryBuilder() const {
MemStatePtr MemoryInput::makeState() const {
// assume ov::Tensor is always dense
auto original_desc =
std::make_shared<CpuBlockedMemoryDesc>(getOriginalOutputPrecisionAtPort(0), outputShapes.at(0));

auto mem_desc = getBaseMemDescAtOutputPort(0);
const auto& eng = getEngine();
return [mem_desc, eng](){ return std::make_shared<Memory>(eng, mem_desc); };

auto state_name = getId();

// Remove suffix with pair ID. Internal information.
auto suffix_idx = state_name.find("/id=");
if (suffix_idx != std::string::npos) {
state_name = state_name.substr(0, suffix_idx);
}

return std::make_shared<VariableStateDoubleBuffer>(state_name,
[mem_desc, eng](){ return std::make_shared<Memory>(eng, mem_desc); },
original_desc,
getMemoryPtr());
}

void MemoryInput::registerOutputNode(MemoryOutput* node) {
Expand Down
4 changes: 2 additions & 2 deletions src/plugins/intel_cpu/src/nodes/memory.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class MemoryNode {
explicit MemoryNode(std::string id) : _id(id) {}
explicit MemoryNode(const std::shared_ptr<ngraph::Node>& op);
virtual ~MemoryNode() = default;
std::string getId() {
std::string getId() const {
return _id;
}
virtual void registerInputNode(MemoryInput*) = 0;
Expand Down Expand Up @@ -133,7 +133,7 @@ class MemoryInput : public Input, public MemoryNode {
void deregisterSibling(MemoryNode* node) override;

void assignState(MemStatePtr newState);
std::function<MemoryPtr(void)> memoryBuilder() const;
MemStatePtr makeState() const;

private:
MemoryOutput& getOutputNode();
Expand Down

0 comments on commit fb60406

Please sign in to comment.