Skip to content

Commit

Permalink
Refactor after the move to API2.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@maxnick
maxnick committed Nov 8, 2023
1 parent fb60406 commit 4d7a30c
Show file tree
Hide file tree
Showing 7 changed files with 177 additions and 142 deletions.
123 changes: 51 additions & 72 deletions src/plugins/intel_cpu/src/infer_request.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ void SyncInferRequest::create_infer_request() {
if (m_compiled_model->m_graphs.size() == 0) {
OPENVINO_THROW("No graph was found");
}
graph = &(m_compiled_model->get_graph()._graph);
m_graph = &(m_compiled_model->get_graph()._graph);

// Alocate memory for each tensor if static shape
for (const auto& it : m_input_ports_map) {
Expand All @@ -64,7 +64,7 @@ void SyncInferRequest::create_infer_request() {
// Save all MemoryLayer data tensors. Will use insight about mechanics
// of MemoryLayer implementation. It uses output edge of MemoryLayer
// producer as storage for tensor to keep it between infer calls.
for (auto& node : graph->GetNodes()) {
for (auto& node : m_graph->GetNodes()) {
if (node->getType() == Type::MemoryInput) {
auto memoryNode = std::dynamic_pointer_cast<node::MemoryInput>(node);
if (!memoryNode) {
Expand All @@ -81,29 +81,29 @@ SyncInferRequest::~SyncInferRequest() {
}

// state -> storage
void SyncInferRequest::AssignStates() {
for (auto &node : graph->GetNodes()) {
void SyncInferRequest::assign_states() {
for (auto &node : m_graph->GetNodes()) {
if (node->getType() == Type::MemoryInput) {
auto cur_node = std::dynamic_pointer_cast<node::MemoryInput>(node);
if (!cur_node) {
OPENVINO_THROW("Cannot cast ", node->getName(), " to MemoryInput");
}
auto cur_id = cur_node->getId();
for (const auto& state : m_memory_states) {
if (state->GetName() == cur_id) {
if (state->get_name() == cur_id) {
cur_node->assignState(state);
}
}
}
}
}

void SyncInferRequest::CommitStates() {
std::for_each(m_memory_states.begin(), m_memory_states.end(), [](const MemStatePtr& state) { state->Commit(); });
void SyncInferRequest::commit_states() {
std::for_each(m_memory_states.begin(), m_memory_states.end(), [](const MemStatePtr& state) { state->commit(); });
}

void SyncInferRequest::redefine_memory_for_input_nodes() {
const auto cpuInputNodes = graph->GetInputNodesMap();
const auto cpuInputNodes = m_graph->GetInputNodesMap();
for (const auto& port : get_inputs()) {
std::string name = get_port_name(port, m_is_legacy_api);
if (name.empty()) {
Expand All @@ -127,28 +127,9 @@ void SyncInferRequest::update_external_tensor_ptrs() {
OPENVINO_THROW("Input tensor map contains not registered during IPlugin::compile_model tensor with name ",
input_name);
}
if (external_ptr.find(input_name) != external_ptr.end()) {
if (m_external_ptr.find(input_name) != m_external_ptr.end()) {
auto tensor = get_tensor(input);
external_ptr[input_name] = tensor;
}
}
}

void SyncInferRequest::redefineMemoryForVariableNodes() {
for (auto &node : graph->GetNodes()) {
if (node->getType() == Type::MemoryInput) {
auto cur_node = dynamic_cast<node::MemoryInput*>(node.get());
if (!cur_node) {
IE_THROW() << "Cannot cast " << node->getName() << " to MemoryInput";
}
auto cur_id = cur_node->getId();
for (const auto& state : m_memory_states) {
if (state->get_name() == cur_id) {
auto cur_state_mem = cur_node->getStore();
node->redefineOutputMemory({cur_state_mem->getStaticDims()});
DEBUG_LOG(cur_state_mem->getData(), " -> ", node->getChildEdgeAt(0)->getMemory().getData());
}
}
m_external_ptr[input_name] = tensor;
}
}
}
Expand All @@ -157,7 +138,7 @@ void SyncInferRequest::infer() {
using namespace openvino::itt;
OV_ITT_SCOPED_TASK(itt::domains::intel_cpu, m_profiling_task);
auto graphLock = m_compiled_model->get_graph();
graph = &(graphLock._graph);
m_graph = &(graphLock._graph);

throw_if_canceled();
convert_batched_tensors();
Expand All @@ -166,12 +147,10 @@ void SyncInferRequest::infer() {
update_external_tensor_ptrs();
}

if (graph->hasDynamicInput()) {
if (m_graph->hasDynamicInput()) {
redefine_memory_for_input_nodes();
}

redefineMemoryForVariableNodes();

change_default_ptr();

throw_if_canceled();
Expand All @@ -180,32 +159,32 @@ void SyncInferRequest::infer() {

// state -> node
if (!m_memory_states.empty()) {
AssignStates();
assign_states();
}

graph->Infer(this);
m_graph->Infer(this);

throw_if_canceled();

// update output control blocks, if any, in order to refresh internal buffers
if (Graph::Status::ReadyDynamic == graph->getStatus()) {
for (auto&& item : outputControlBlocks) {
if (Graph::Status::ReadyDynamic == m_graph->getStatus()) {
for (auto&& item : m_outputControlBlocks) {
item.second.update();
}
}

if (!m_memory_states.empty()) {
CommitStates();
commit_states();
}

graph->PullOutputData(m_outputs);
m_graph->PullOutputData(m_outputs);
}

std::vector<ov::ProfilingInfo> SyncInferRequest::get_profiling_info() const {
if (!graph || !graph->IsReady())
if (!m_graph || !m_graph->IsReady())
OPENVINO_THROW("Graph is not ready!");
std::vector<ov::ProfilingInfo> perfMap;
graph->GetPerfData(perfMap);
m_graph->GetPerfData(perfMap);
return perfMap;
}

Expand All @@ -218,12 +197,12 @@ static inline void change_edge_ptr(const EdgePtr& edge, ov::SoPtr<ov::ITensor>&
}

void SyncInferRequest::change_default_ptr() {
const auto& inputNodesMap = graph->GetInputNodesMap();
const auto& outputNodesMap = graph->GetOutputNodesMap();
const auto& inputNodesMap = m_graph->GetInputNodesMap();
const auto& outputNodesMap = m_graph->GetOutputNodesMap();

std::unordered_set<const void*> inputPtrs;
std::function<void(const EdgePtr &edge, ov::SoPtr<ov::ITensor>& tensor)> changeInpPtr;
if (Graph::Status::ReadyDynamic == graph->getStatus()) {
if (Graph::Status::ReadyDynamic == m_graph->getStatus()) {
changeInpPtr = [&inputPtrs](const EdgePtr &edge, ov::SoPtr<ov::ITensor>& tensor) {
change_edge_ptr(edge, tensor);
inputPtrs.insert(tensor->data());
Expand All @@ -234,7 +213,7 @@ void SyncInferRequest::change_default_ptr() {
};
}

for (auto& it : external_ptr) {
for (auto& it : m_external_ptr) {
auto input = inputNodesMap.find(it.first);
if (inputNodesMap.end() == input) {
OPENVINO_ASSERT(outputNodesMap.count(it.first), "Cannot find input/output blob: ", it.first);
Expand Down Expand Up @@ -285,7 +264,7 @@ void SyncInferRequest::change_default_ptr() {
}
}

for (auto& it : external_ptr) {
for (auto& it : m_external_ptr) {
const auto& name = it.first;
auto output = outputNodesMap.find(name);
if (outputNodesMap.end() == output) {
Expand Down Expand Up @@ -323,18 +302,18 @@ void SyncInferRequest::change_default_ptr() {
change_edge_ptr(parentEdge, it.second);
}

if (Graph::Status::ReadyDynamic == graph->getStatus()) {
const auto &outMemMngrMap = graph->outputNodesMemMngrMap;
if (Graph::Status::ReadyDynamic == m_graph->getStatus()) {
const auto &outMemMngrMap = m_graph->outputNodesMemMngrMap;
for (auto&& item : outMemMngrMap) {
const auto& name = item.first;

// share intel_cpu::Tensor to Graph by injecting to corresponding ProxyMemoryMngr instance.
auto outputMemMngr = item.second;
OPENVINO_ASSERT(outputMemMngr, "proxy mem manager for output ", name, " is empty.");

auto controlBlockItr = outputControlBlocks.find(name);
auto controlBlockItr = m_outputControlBlocks.find(name);

if (controlBlockItr != outputControlBlocks.end()) {
if (controlBlockItr != m_outputControlBlocks.end()) {
auto output = outputNodesMap.find(name);
OPENVINO_ASSERT(outputNodesMap.end() != output, "Node with name: ", name, " is absent in the outputNodesMap");
auto parentEdge = output->second->getParentEdgeAt(0);
Expand All @@ -346,7 +325,7 @@ void SyncInferRequest::change_default_ptr() {
controlBlock.currentMemMngr(); // else reuse the existing buffer

outputMemMngr->setMemMngrResize(memMngr);
DEBUG_LOG("reset proxy ", outputMemMngr, ", actual ", controlBlock.currentMemMngr(), " graph ", graph, " inferrequest ", this);
DEBUG_LOG("reset proxy ", outputMemMngr, ", actual ", controlBlock.currentMemMngr(), " graph ", m_graph, " inferrequest ", this);
DEBUG_LOG(name, ", tensor ", controlBlock.tensor());
} else {
outputMemMngr->reset(); // switch to the internal memory since memory sharing is no longer possible
Expand All @@ -355,7 +334,7 @@ void SyncInferRequest::change_default_ptr() {
}
}

std::vector<ov::SoPtr<ov::IVariableState>> SyncInferRequest::query_state() {
std::vector<ov::SoPtr<ov::IVariableState>> SyncInferRequest::query_state() const {
return {m_memory_states.begin(), m_memory_states.end()};
}

Expand Down Expand Up @@ -466,7 +445,7 @@ void SyncInferRequest::set_tensor(const ov::Output<const ov::Node>& in_port, con
" are different.");
}

MemoryDescPtr actualDesc = graph->getInputNodeByName(name)->getBaseMemDescAtOutputPort(0);
MemoryDescPtr actualDesc = m_graph->getInputNodeByName(name)->getBaseMemDescAtOutputPort(0);
if (!actualDesc->isDefined()) {
// we must define desc for dynamic case
// otherwise we got incorrect check on shape compatibility inside isCompatible
Expand All @@ -478,10 +457,10 @@ void SyncInferRequest::set_tensor(const ov::Output<const ov::Node>& in_port, con
OPENVINO_SUPPRESS_DEPRECATED_END
}
if (actualDesc->isCompatible(MemoryDescUtils::convertToCpuBlockedMemoryDesc(tensor_desc)) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
external_ptr[name] = tensor;
} else if (external_ptr.find(name) != external_ptr.end()) {
external_ptr.erase(name);
m_graph->_normalizePreprocMap.find(name) == m_graph->_normalizePreprocMap.end()) {
m_external_ptr[name] = tensor;
} else if (m_external_ptr.find(name) != m_external_ptr.end()) {
m_external_ptr.erase(name);
}
} else {
const auto netOutPrc = port.get_element_type();
Expand Down Expand Up @@ -513,15 +492,15 @@ void SyncInferRequest::set_tensor(const ov::Output<const ov::Node>& in_port, con
" are different.");
}

const auto& desc = graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
const auto& desc = m_graph->getOutputNodeByName(name)->getParentEdgesAtPort(0)[0]->getMemory().getDesc();
if (!isDynamic && tensor_desc == MemoryDescUtils::convertToTensorDesc(desc)) {
external_ptr[name] = tensor;
} else if (external_ptr.find(name) != external_ptr.end()) {
external_ptr.erase(name);
m_external_ptr[name] = tensor;
} else if (m_external_ptr.find(name) != m_external_ptr.end()) {
m_external_ptr.erase(name);
}

m_outputs[name] = tensor;
outputControlBlocks.erase(name); // now the memory is under user's control
m_outputControlBlocks.erase(name); // now the memory is under user's control
}
ov::ISyncInferRequest::set_tensor(port, tensor);
}
Expand All @@ -539,13 +518,13 @@ void SyncInferRequest::set_tensors_impl(const ov::Output<const ov::Node> port, c
void SyncInferRequest::init_tensor(const std::string& name) {
OV_ITT_SCOPED_TASK(itt::domains::intel_cpu, "init_tensor");

if (!graph || !graph->IsReady())
if (!m_graph || !m_graph->IsReady())
OPENVINO_THROW("Graph is not ready!");

OPENVINO_ASSERT(!name.empty(), "Can't prepare tensor for empty name! ");

ov::SoPtr<ITensor> tensor;
const auto& inMap = graph->inputNodesMap;
const auto& inMap = m_graph->inputNodesMap;
auto input = inMap.find(name);
if (input != inMap.end()) {
auto input_port = m_input_ports_map.find(name);
Expand All @@ -572,14 +551,14 @@ void SyncInferRequest::init_tensor(const std::string& name) {
auto desc = create_tensor_desc(tensor);
if (!isDynamic &&
desc == MemoryDescUtils::convertToTensorDesc(
graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
graph->_normalizePreprocMap.find(name) == graph->_normalizePreprocMap.end()) {
external_ptr[name] = tensor;
m_graph->getInputNodeByName(name)->getChildEdgesAtPort(0)[0]->getMemory().getDesc()) &&
m_graph->_normalizePreprocMap.find(name) == m_graph->_normalizePreprocMap.end()) {
m_external_ptr[name] = tensor;
}
}
}

const auto& outMap = graph->outputNodesMap;
const auto& outMap = m_graph->outputNodesMap;
auto output = outMap.find(name);
if (output != outMap.end()) {
if (m_outputs.find(name) == m_outputs.end()) {
Expand Down Expand Up @@ -619,7 +598,7 @@ void SyncInferRequest::init_tensor(const std::string& name) {

tensor = control_block.tensor();
if (model_prec == graph_prec)
outputControlBlocks.emplace(std::make_pair(name, std::move(control_block)));
m_outputControlBlocks.emplace(std::make_pair(name, std::move(control_block)));
} else {
tensor_shape = shape.to_shape();
tensor = ov::make_tensor(port.get_element_type(), tensor_shape);
Expand Down Expand Up @@ -653,10 +632,10 @@ void SyncInferRequest::init_tensor(const std::string& name) {
}
m_outputs[name] = tensor;
auto desc = create_tensor_desc(tensor);
if (!port_shape.is_dynamic() && !external_ptr.count(name) &&
if (!port_shape.is_dynamic() && !m_external_ptr.count(name) &&
desc == MemoryDescUtils::convertToTensorDesc(
output->second->getParentEdgesAtPort(0)[0]->getMemory().getDesc())) {
external_ptr[name] = tensor;
m_external_ptr[name] = tensor;
}
// update tensors in case of multiple output ports with the same name
for (const auto& out : get_outputs()) {
Expand All @@ -681,7 +660,7 @@ void SyncInferRequest::push_input_data() {
input_name);
}
auto tensor = get_tensor(input);
graph->PushInputData(input_name, tensor);
m_graph->PushInputData(input_name, tensor);
}
}

Expand Down
24 changes: 13 additions & 11 deletions src/plugins/intel_cpu/src/infer_request.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,22 +91,26 @@ class SyncInferRequest : public ov::ISyncInferRequest {
std::array<MemMngrPtr, 2> m_buffers;
int m_buffIndx = 0;
};
std::unordered_map<std::string, OutputControlBlock> outputControlBlocks;

private:
void create_infer_request();

void init_tensor(const std::string& name);

void push_input_data();
void redefine_memory_for_input_nodes();
void assign_states();
void commit_states();
void update_external_tensor_ptrs();
void change_default_ptr();

const ov::Output<const ov::Node>& get_internal_port(const ov::Output<const ov::Node>& port) const;

Graph* graph = nullptr;
std::unordered_map<std::string, ov::SoPtr<ov::ITensor>> external_ptr;
private:
std::unordered_map<std::string, OutputControlBlock> m_outputControlBlocks;

void redefine_memory_for_input_nodes()
void AssignStates();
void CommitStates();
Graph* m_graph = nullptr;
std::unordered_map<std::string, ov::SoPtr<ov::ITensor>> m_external_ptr;

void update_external_tensor_ptrs();
const ov::Output<const ov::Node>& get_internal_port(const ov::Output<const ov::Node>& port) const;
bool m_is_legacy_api = false;

std::shared_ptr<const CompiledModel> m_compiled_model;
Expand All @@ -117,8 +121,6 @@ class SyncInferRequest : public ov::ISyncInferRequest {
std::unordered_map<std::string, ov::Output<const ov::Node>> m_input_ports_map;
std::unordered_map<std::string, ov::Output<const ov::Node>> m_output_ports_map;
std::unordered_map<std::string, ov::SoPtr<ov::ITensor>> m_outputs;

void change_default_ptr();
};

} // namespace intel_cpu
Expand Down
Loading

0 comments on commit 4d7a30c

Please sign in to comment.