diff --git a/.gitignore b/.gitignore index 97d5b8e0..5ad088d0 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,7 @@ build/ *.so *.user graph.dot + +.clang-format +.cache/ +miob/ \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fdb4c4a9..f94c6b49 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -66,7 +66,7 @@ test:cppcheck: --error-exitcode=1 --quiet --inline-suppr - --enable=warning,performance,portability,information,missingInclude + --enable=cppcoreguidelines,warning,performance,portability,information,missingInclude --std=c++11 --suppress=noValidConfiguration -I include diff --git a/.vscode/launch.json b/.vscode/launch.json index 1bce5a80..b1864afa 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -8,7 +8,7 @@ "name": "(gdb) Launch", "type": "cppdbg", "request": "launch", - "program": "${workspaceFolder}/build/src/villas-fpga-ctrl", + "program": "${workspaceFolder}/build/src/platform", "args": [ "-c", "${workspaceFolder}/etc/fpgas.json", "--connect", "\"2<->stdout\"" ], diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f5dbba0..3af4874d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/cmake) # Several CMake settings/defaults set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Werror") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra") if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) set(TOPLEVEL_PROJECT ON) diff --git a/common b/common index 0cb6cd23..3a66d8ff 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 0cb6cd23caacdd504753fd7f32e0a606d1b964f3 +Subproject commit 3a66d8ff657c0824138a03df0aca1b16c88d582a diff --git a/etc/fpgas.json b/etc/fpgas.json index cdaa9bfb..c8409e3a 100644 --- a/etc/fpgas.json +++ b/etc/fpgas.json @@ -1,6 +1,6 @@ { "fpgas": { - "vc707": { + "zcu106": { "id": "10ee:7021", "slot": "0000:88:00.0", "do_reset": true, diff --git a/include/villas/fpga/card.hpp b/include/villas/fpga/card.hpp index f58a4c68..dedac077 100644 --- a/include/villas/fpga/card.hpp +++ b/include/villas/fpga/card.hpp @@ -22,10 +22,10 @@ class Card { public: bool polling; - - std::string name; // The name of the FPGA card + bool doReset; // Reset VILLASfpga during startup? + int affinity; // Affinity for MSI interrupts + std::string name; // The name of the FPGA card std::shared_ptr vfioContainer; - std::shared_ptr vfioDevice; // Slave address space ID to access the PCIe address space from the // FPGA @@ -52,6 +52,7 @@ class Card std::map> memoryBlocksMapped; Logger logger; + }; } // namespace fpga diff --git a/include/villas/fpga/card_parser.hpp b/include/villas/fpga/card_parser.hpp new file mode 100644 index 00000000..4ba86dd3 --- /dev/null +++ b/include/villas/fpga/card_parser.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include "villas/exceptions.hpp" +#include "villas/log.hpp" +#include +#include + +class CardParser { +public: + std::shared_ptr logger; + + json_t *json_ips = nullptr; + json_t *json_paths = nullptr; + const char *pci_slot = nullptr; + const char *pci_id = nullptr; + int do_reset = 0; + int affinity = 0; + int polling = 0; + json_t *devices = nullptr; + std::vector device_names; + + CardParser(json_t *json_card) : logger(villas::logging.get("CardParser")) { + json_error_t err; + int ret = json_unpack_ex( + json_card, &err, 0, + "{ s: o, s?: i, s?: b, s?: s, s?: s, s?: b, s?: o, s?: o }", "ips", + &json_ips, "affinity", &affinity, "do_reset", &do_reset, "slot", + &pci_slot, "id", &pci_id, "polling", &polling, "paths", &json_paths, + "devices", &devices); + + if (ret != 0) + throw villas::ConfigError(json_card, err, "", "Failed to parse card"); + + // devices array parsing + size_t index; + json_t *value; + json_array_foreach(devices, index, value) { + auto str = json_string_value(value); + device_names.push_back(str); + } + } +}; diff --git a/include/villas/fpga/ip_loader.hpp b/include/villas/fpga/ip_loader.hpp new file mode 100644 index 00000000..2d88b64b --- /dev/null +++ b/include/villas/fpga/ip_loader.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "villas/exceptions.hpp" +#include "villas/log.hpp" +#include +#include +#include + +using villas::ConfigError; + +class IpLoader { +public: + std::shared_ptr logger; + + + IpLoader(json_t* json_ips, const std::filesystem::path& searchPath): logger(villas::logging.get("IpParser")) { + // Load IPs from a separate json file + if (!json_is_string(json_ips)) { + logger->debug("FPGA IP cores config item is not a string."); + throw ConfigError(json_ips, "node-config-fpga-ips", + "FPGA IP cores config item is not a string."); + } + if (!searchPath.empty()) { + std::filesystem::path json_ips_path = + searchPath / json_string_value(json_ips); + logger->debug("searching for FPGA IP cors config at {}", json_ips_path); + json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr); + } + if (json_ips == nullptr) { + json_ips = + json_load_file(json_string_value(json_ips), 0, nullptr); + logger->debug("searching for FPGA IP cors config at {}", + json_string_value(json_ips)); + if (json_ips == nullptr) { + throw ConfigError(json_ips, "node-config-fpga-ips", + "Failed to find FPGA IP cores config"); + } + } + + if (not json_is_object(json_ips)) + throw ConfigError(json_ips, "node-config-fpga-ips", + "FPGA IP core list must be an object!"); + } +}; diff --git a/include/villas/fpga/ips/intc.hpp b/include/villas/fpga/ips/intc.hpp index 7bbe31db..2ec4cf10 100644 --- a/include/villas/fpga/ips/intc.hpp +++ b/include/villas/fpga/ips/intc.hpp @@ -27,25 +27,25 @@ class InterruptController : public Core { virtual bool init() override; - bool enableInterrupt(IrqMaskType mask, bool polling); - bool enableInterrupt(IrqPort irq, bool polling) + virtual bool enableInterrupt(IrqMaskType mask, bool polling); + virtual bool enableInterrupt(IrqPort irq, bool polling) { return enableInterrupt(1 << irq.num, polling); } - bool disableInterrupt(IrqMaskType mask); - bool disableInterrupt(IrqPort irq) + virtual bool disableInterrupt(IrqMaskType mask); + virtual bool disableInterrupt(IrqPort irq) { return disableInterrupt(1 << irq.num); } - int waitForInterrupt(int irq); - int waitForInterrupt(IrqPort irq) + virtual int waitForInterrupt(int irq); + virtual int waitForInterrupt(IrqPort irq) { return waitForInterrupt(irq.num); } -private: +protected: static constexpr char registerMemory[] = "reg0"; diff --git a/include/villas/fpga/ips/platform_intc.hpp b/include/villas/fpga/ips/platform_intc.hpp new file mode 100644 index 00000000..36a25fb4 --- /dev/null +++ b/include/villas/fpga/ips/platform_intc.hpp @@ -0,0 +1,13 @@ +#pragma once + +#include + +class PlatformInterruptController + : public villas::fpga::ip::InterruptController { +public: + bool init() override; + + bool enableInterrupt(InterruptController::IrqMaskType mask, + bool polling) override; + bool enableInterrupt(IrqPort irq, bool polling) override; +}; diff --git a/include/villas/fpga/ips/zynq.hpp b/include/villas/fpga/ips/zynq.hpp new file mode 100644 index 00000000..4a953a4d --- /dev/null +++ b/include/villas/fpga/ips/zynq.hpp @@ -0,0 +1,78 @@ +/** Zynq VFIO connector node + * + * Author: Pascal Bauer + * SPDX-FileCopyrightText: 2017 Steffen Vogel + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#pragma once + +#include + +#include + +namespace villas { +namespace fpga { +namespace ip { + +class Zynq : public Core { +public: + friend class ZynqFactory; + + virtual + bool init() override; + +private: + static constexpr char axiInterface[] = "M_AXI"; + static constexpr char pcieMemory[] = "BAR0"; + + struct AxiBar { + uintptr_t base; + size_t size; + uintptr_t translation; + }; + + struct PciBar { + uintptr_t translation; + }; + + std::map axiToPcieTranslations; + std::map pcieToAxiTranslations; +}; + +class ZynqFactory : CoreFactory { + +public: + virtual + std::string getName() const + { + return "pcie"; + } + + virtual + std::string getDescription() const + { + return "Custom platform vfio connector"; + } + +private: + virtual + Vlnv getCompatibleVlnv() const + { + return Vlnv("xilinx.com:ip:zynq_ultra_ps_e:"); + } + + // Create a concrete IP instance + Core* make() const + { + return new Zynq; + }; + +protected: + virtual + void parse(Core &, json_t *) override; +}; + +} /* namespace ip */ +} /* namespace fpga */ +} /* namespace villas */ diff --git a/include/villas/fpga/pcie_card.hpp b/include/villas/fpga/pcie_card.hpp index 1f3afa15..96b50e12 100644 --- a/include/villas/fpga/pcie_card.hpp +++ b/include/villas/fpga/pcie_card.hpp @@ -61,13 +61,15 @@ class PCIeCard : public Card { { } public: // TODO: make this private - bool doReset; // Reset VILLASfpga during startup? - int affinity; // Affinity for MSI interrupts + std::shared_ptr pdev; // PCI device handle + std::shared_ptr vfioDevice; //? Only used by intc - std::shared_ptr pdev; // PCI device handle - - protected: - Logger getLogger() const { return villas::logging.get(name); } +protected: + Logger + getLogger() const + { + return villas::logging.get(name); + } }; class PCIeCardFactory : public plugin::Plugin { diff --git a/include/villas/fpga/platform_card.hpp b/include/villas/fpga/platform_card.hpp new file mode 100644 index 00000000..28052052 --- /dev/null +++ b/include/villas/fpga/platform_card.hpp @@ -0,0 +1,44 @@ +/** + * Author: Pascal Henry Bauer + * Based on the work of: Steffen Vogel and Daniel Krebs + * + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#pragma once + +#include +#include +#include + +namespace villas { +namespace fpga { + +class PlatformCard : public Card +{ +public: + PlatformCard(std::shared_ptr vfioContainer, + std::vector device_names); + + ~PlatformCard(){}; + + std::vector> devices; + + void connectVFIOtoIPS(); + bool mapMemoryBlock(const std::shared_ptr block) override; + +private: + +}; + +class PlatformCardFactory +{ +public: + static std::list> make(json_t *json, + std::shared_ptr vc, + const std::filesystem::path& searchPath); +}; + +} /* namespace fpga */ +} /* namespace villas */ \ No newline at end of file diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 34d65d4c..3fa1ffec 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -8,6 +8,7 @@ set(SOURCES vlnv.cpp card.cpp pcie_card.cpp + platform_card.cpp core.cpp node.cpp utils.cpp @@ -23,10 +24,12 @@ set(SOURCES ips/gpio.cpp ips/intc.cpp ips/pcie.cpp + ips/platform_intc.cpp ips/rtds.cpp ips/switch.cpp ips/timer.cpp ips/i2c.cpp + ips/zynq.cpp ips/rtds2gpu/rtds2gpu.cpp ips/rtds2gpu/xrtds2gpu.c diff --git a/lib/core.cpp b/lib/core.cpp index 2af5a175..43872b7a 100644 --- a/lib/core.cpp +++ b/lib/core.cpp @@ -1,26 +1,29 @@ /* FPGA IP component. * * Author: Steffen Vogel - * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, RWTH Aachen University - * SPDX-License-Identifier: Apache-2.0 + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power + * Systems, RWTH Aachen University SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include #include #include #include #include #include +#include #include #include #include #include #include +#include #include using namespace villas::fpga; @@ -30,6 +33,7 @@ using namespace villas::fpga::ip; // same order as they appear in this list, i.e. first here will be initialized // first. static std::list vlnvInitializationOrder = { + Vlnv("xilinx.com:ip:zynq_ultra_ps_e:"), Vlnv("xilinx.com:ip:axi_pcie:"), Vlnv("xilinx.com:module_ref:axi_pcie_intc:"), Vlnv("xilinx.com:ip:axis_switch:"), @@ -168,6 +172,27 @@ std::list> CoreFactory::make(Card *card, logger->debug("IRQ: {} -> {}:{}", irqName, irqControllerName, num); ip->irqs[irqName] = {num, intc, ""}; } + } else if (ip->getInstanceName().find("axi_dma_") != std::string::npos) { + logger->warn("Dma json does not contain an interrupt Controller. A " + "Platform Interrupt controller will be added"); + + //! TODO: Order of interrupts is hardcoded and not tested (may be reversed). Aviable in vfio device irq.id . + auto intc = new PlatformInterruptController(); + intc->id = id; + intc->logger = villas::logging.get(id.getName()); + intc->card = card; + + std::vector intc_names = {"mm2s_introut", "s2mm_introut"}; + int num = 0; + for (auto name : intc_names) + { + std::string irqControllerName = "PlatformInterruptController"; + logger->debug("IRQ: {} -> {}:{}", std::string(name), irqControllerName, num); + ip->irqs[std::string(name)] = {num, intc, ""}; + num++; + } + + intc->init(); } json_t *json_memory_view = json_object_get(json_ip, "memory-view"); diff --git a/lib/ips/intc.cpp b/lib/ips/intc.cpp index f1cefc94..07469483 100644 --- a/lib/ips/intc.cpp +++ b/lib/ips/intc.cpp @@ -21,6 +21,7 @@ using namespace villas::fpga::ip; InterruptController::~InterruptController() { + PCIeCard* card = dynamic_cast(card); card->vfioDevice->pciMsiDeinit(this->efds); } @@ -29,6 +30,7 @@ InterruptController::init() { const uintptr_t base = getBaseAddr(registerMemory); + PCIeCard* card = dynamic_cast(card); num_irqs = card->vfioDevice->pciMsiInit(efds); if (num_irqs < 0) return false; @@ -122,9 +124,8 @@ InterruptController::waitForInterrupt(int irq) { assert(irq < maxIrqs); - const uintptr_t base = getBaseAddr(registerMemory); - if (this->polling[irq]) { + const uintptr_t base = getBaseAddr(registerMemory); uint32_t isr, mask = 1 << irq; do { diff --git a/lib/ips/pcie.cpp b/lib/ips/pcie.cpp index c7861db5..67d5252b 100644 --- a/lib/ips/pcie.cpp +++ b/lib/ips/pcie.cpp @@ -22,6 +22,8 @@ AxiPciExpressBridge::init() { auto &mm = MemoryManager::get(); + PCIeCard* card = dynamic_cast(card); + // Throw an exception if the is no bus master interface and thus no // address space we can use for translation -> error card->addrSpaceIdHostToDevice = busMasterInterfaces.at(axiInterface); diff --git a/lib/ips/platform_intc.cpp b/lib/ips/platform_intc.cpp new file mode 100644 index 00000000..f72044db --- /dev/null +++ b/lib/ips/platform_intc.cpp @@ -0,0 +1,31 @@ +#include +#include +#include +#include +#include + +bool PlatformInterruptController::init(){ + auto platform_card = dynamic_cast(card); + + std::shared_ptr dma_vfio_device; + for (auto device : platform_card->devices) + { + if (device->getName().find("dma") != std::string::npos){ + dma_vfio_device = device; + break; + } + } + + if(dma_vfio_device == nullptr) throw std::logic_error("No vfio device with name containing dma found."); + + dma_vfio_device->platformInit(efds); +} + +bool PlatformInterruptController::enableInterrupt( + InterruptController::IrqMaskType mask, bool polling) { + logger->debug("Enabling Platform Interrupt"); +}; + +bool PlatformInterruptController::enableInterrupt(IrqPort irq, bool polling) { + logger->debug("Enabling Platform Interrupt"); +}; diff --git a/lib/ips/zynq.cpp b/lib/ips/zynq.cpp new file mode 100644 index 00000000..9fa17c27 --- /dev/null +++ b/lib/ips/zynq.cpp @@ -0,0 +1,95 @@ +/** AXI PCIe bridge + * + * Author: Daniel Krebs + * SPDX-FileCopyrightText: 2018 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#include +#include + +#include +#include + +#include +#include +#include + +#include + + +using namespace villas::fpga::ip; + + +bool +Zynq::init() +{ + auto &mm = MemoryManager::get(); + + // IPs that can access this address space will know it via their memory view + // const auto addrSpaceNameDeviceToHost = + // mm.getMasterAddrSpaceName("zynq_ultra_ps_e_0", "HPC1_DDR_LOW"); + //mm.getSlaveAddrSpaceName(getInstanceName(), pcieMemory); + + // Save ID in card so we can create mappings later when needed (e.g. when + // allocating DMA memory in host RAM) + card->addrSpaceIdDeviceToHost = + mm.getOrCreateAddressSpace("zynq_ultra_ps_e_0/HPC0_DDR_LOW"); + + dynamic_cast(card)->connectVFIOtoIPS(); + + return true; +} + +void +ZynqFactory::parse(Core &ip, json_t *cfg) +{ + CoreFactory::parse(ip, cfg); + + auto logger = getLogger(); + //auto &zynq = dynamic_cast(ip); + + // for (auto barType : std::list{ + // "axi_bars", + // "pcie_bars" + // }) { + // json_t *json_bars = json_object_get(cfg, barType.c_str()); + // if (not json_is_object(json_bars)) + // throw ConfigError(cfg, "", "Missing BAR config: {}", barType); + + // json_t* json_bar; + // const char* bar_name; + // json_object_foreach(json_bars, bar_name, json_bar) { + // unsigned int translation; + + // json_error_t err; + // int ret = json_unpack_ex(json_bar, &err, 0, "{ s: i }", + // "translation", &translation + // ); + // if (ret != 0) + // throw ConfigError(json_bar, err, "", "Cannot parse {}/{}", barType, bar_name); + + // if (barType == "axi_bars") { + // json_int_t base, high, size; + // int ret = json_unpack_ex(json_bar, &err, 0, "{ s: I, s: I, s: I }", + // "baseaddr", &base, + // "highaddr", &high, + // "size", &size + // ); + // if (ret != 0) + // throw ConfigError(json_bar, err, "", "Cannot parse {}/{}", barType, bar_name); + + // pcie.axiToPcieTranslations[bar_name] = { + // .base = static_cast(base), + // .size = static_cast(size), + // .translation = translation + // }; + // } else + // pcie.pcieToAxiTranslations[bar_name] = { + // .translation = translation + // }; + // } + // } +} + +static ZynqFactory p; diff --git a/lib/platform_card.cpp b/lib/platform_card.cpp new file mode 100644 index 00000000..75e75e95 --- /dev/null +++ b/lib/platform_card.cpp @@ -0,0 +1,203 @@ +/** + * Author: Pascal Henry Bauer + * Based on the work of: Steffen Vogel and Daniel Krebs + * + * + * SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power + *Systems, EONERC SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ +#include "villas/fpga/ips/dma.hpp" +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace villas; +using namespace villas::fpga; + + +PlatformCard::PlatformCard( + std::shared_ptr vfioContainer, + std::vector device_names + ) +{ + this->vfioContainer = vfioContainer; + this->logger = villas::logging.get("PlatformCard"); + + // Create VFIO Group + const int IOMMU_GROUP = 2; //TODO: find Group + auto group = std::make_shared(IOMMU_GROUP, true); + vfioContainer->attachGroup(group); + + // Open VFIO Devices + for(std::string device_name : device_names){ + auto vfioDevice = std::make_shared( + device_name, + group->getFileDescriptor()); + group->attachDevice(vfioDevice); + this->devices.push_back(vfioDevice); + } + + // Map all vfio devices in card to process + std::map, const void*> mapped_memory; + for (auto device : devices) + { + const void* mapping = device->regionMap(0); + if (mapping == MAP_FAILED) { + logger->error("Failed to mmap() device"); + } + logger->debug("memory mapped: {}", device->getName()); + mapped_memory.insert({device, mapping}); + } + + // Create mappings from process space to vfio devices + auto &mm = MemoryManager::get(); + size_t srcVertexId = mm.getProcessAddressSpace(); + for (auto pair : mapped_memory) + { + const size_t mem_size = pair.first->regionGetSize(0); + size_t targetVertexId = mm.getOrCreateAddressSpace(pair.first->getName()); + mm.createMapping(reinterpret_cast(pair.second), + 0, mem_size, "process to vfio", srcVertexId, targetVertexId); + logger->debug("create edge from process to {}", pair.first->getName()); + } +} + +void PlatformCard::connectVFIOtoIPS() +{ + auto &mm = MemoryManager::get(); + const size_t ip_mem_size = 65536; + size_t srcVertexId = mm.getOrCreateAddressSpace("a0000000.dma"); + size_t targetVertexId = mm.getOrCreateAddressSpace("axi_dma_0/Reg"); + mm.createMapping(0, 0, ip_mem_size, "vfio to ip", srcVertexId, + targetVertexId); + + // Switch + srcVertexId = mm.getOrCreateAddressSpace("a0010000.axis_switch"); + targetVertexId = mm.getOrCreateAddressSpace("axis_interconnect_0_xbar/Reg"); + mm.createMapping(0, 0, ip_mem_size, "vfio to ip", srcVertexId, + targetVertexId); + + for(auto device : devices) + { + std::string addr; + std::string name; + + std::istringstream iss(device->getName()); + std::getline(iss, addr, '.'); + std::getline(iss, name, '.'); + } +} + +bool PlatformCard::mapMemoryBlock(const std::shared_ptr block) { + if (not vfioContainer->isIommuEnabled()) { + logger->warn("VFIO mapping not supported without IOMMU"); + return false; + } + + auto &mm = MemoryManager::get(); + const auto &addrSpaceId = block->getAddrSpaceId(); + + if (memoryBlocksMapped.find(addrSpaceId) != memoryBlocksMapped.end()) + // Block already mapped + return true; + else + logger->debug("Create VFIO-Platform mapping for {}", addrSpaceId); + + auto translationFromProcess = mm.getTranslationFromProcess(addrSpaceId); + uintptr_t processBaseAddr = translationFromProcess.getLocalAddr(0); + uintptr_t iovaAddr = + vfioContainer->memoryMap(processBaseAddr, UINTPTR_MAX, block->getSize()); + + if (iovaAddr == UINTPTR_MAX) { + logger->error("Cannot map memory at {:#x} of size {:#x}", processBaseAddr, + block->getSize()); + return false; + } + + mm.createMapping(iovaAddr, 0, block->getSize(), "VFIO-D2H", + this->addrSpaceIdDeviceToHost, addrSpaceId); + + auto space = mm.findAddressSpace("zynq_ultra_ps_e_0/HPC1_DDR_LOW"); + mm.createMapping(iovaAddr, 0, block->getSize(), "VFIO-D2H", + space, addrSpaceId); + + + // Remember that this block has already been mapped for later + memoryBlocksMapped.insert({addrSpaceId, block}); + + return true; +} + +std::list> +PlatformCardFactory::make(json_t *json, + std::shared_ptr vc, + const std::filesystem::path& searchPath) +{ + std::list> cards; + auto logger = villas::logging.get("PlatformCard"); + + const char *card_name; + json_t *json_card; + json_object_foreach(json, card_name, json_card) { + logger->info("Found config for FPGA card {}", card_name); + + // Parse and create card + CardParser parser(json_card); + + auto card = std::make_unique(vc, parser.device_names); + card->name = std::string(card_name); + card->affinity = parser.affinity; + card->doReset = parser.do_reset != 0; + card->polling = (parser.polling != 0); + + // if (not card->init()) { + // logger->warn("Cannot start FPGA card {}", card_name); + // continue; + // } + + //IpLoader ipLoader(parser.json_ips, searchPath); + + //Load IPs from a separate json file + if (!json_is_string(parser.json_ips)) { + logger->debug("FPGA IP cores config item is not a string."); + throw ConfigError(parser.json_ips, "node-config-fpga-ips", + "FPGA IP cores config item is not a string."); + } + if (!searchPath.empty()) { + std::filesystem::path json_ips_path = + searchPath / json_string_value(parser.json_ips); + logger->debug("searching for FPGA IP cors config at {}", json_ips_path); + parser.json_ips = json_load_file(json_ips_path.c_str(), 0, nullptr); + } + if (parser.json_ips == nullptr) { + parser.json_ips = + json_load_file(json_string_value(parser.json_ips), 0, nullptr); + logger->debug("searching for FPGA IP cors config at {}", + json_string_value(parser.json_ips)); + if (parser.json_ips == nullptr) { + throw ConfigError(parser.json_ips, "node-config-fpga-ips", + "Failed to find FPGA IP cores config"); + } + } + + if (not json_is_object(parser.json_ips)) + throw ConfigError(parser.json_ips, "node-config-fpga-ips", + "FPGA IP core list must be an object!"); + + card->ips = ip::CoreFactory::make(card.get(), parser.json_ips); + if (card->ips.empty()) + throw ConfigError(parser.json_ips, "node-config-fpga-ips", "Cannot initialize IPs of FPGA card {}", card_name); + + cards.push_back(std::move(card)); + } + return cards; +} diff --git a/scripts/bind_platform_vfio.sh b/scripts/bind_platform_vfio.sh new file mode 100644 index 00000000..593e1386 --- /dev/null +++ b/scripts/bind_platform_vfio.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Bind Platform fpga to vfio +# +# Author: Pascal Bauer +# SPDX-FileCopyrightText: 2017 Institute for Automation of Complex Power Systems, EONERC +# SPDX-License-Identifier: Apache-2.0 +################################################################################## + +modprobe vfio_platform reset_required=0 + +# Unbind Device from driver +echo a0000000.dma > /sys/bus/platform/drivers/xilinx-vdma/unbind +# Bind device +echo vfio-platform > /sys/bus/platform/devices/a0000000.dma/driver_override +echo a0000000.dma > /sys/bus/platform/drivers/vfio-platform/bind + +# Other vfio devices without driver override +echo vfio-platform > /sys/bus/platform/devices/a0010000.axis_switch/driver_override +echo a0010000.axis_switch > /sys/bus/platform/drivers_probe diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2d158b2f..be2bab5b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,4 +17,9 @@ target_link_libraries(villas-fpga-pipe PUBLIC ) +add_executable(platform platform.cpp) +target_link_libraries(platform PUBLIC + villas-fpga +) + add_executable(pcimem pcimem.c) diff --git a/src/platform.cpp b/src/platform.cpp new file mode 100644 index 00000000..5e59b149 --- /dev/null +++ b/src/platform.cpp @@ -0,0 +1,187 @@ +/** + * SPDX-FileCopyrightText: 2018 Institute for Automation of Complex Power Systems, EONERC + * SPDX-License-Identifier: Apache-2.0 + *********************************************************************************/ + +#include "villas/fpga/ips/switch.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace villas; +using namespace fpga; + +static auto logger = villas::logging.get("PLATFORM CTRL"); + +void writeToDmaFromStdIn(std::shared_ptr dma) { + auto &alloc = villas::HostRam::getAllocator(); + const std::shared_ptr block = + alloc.allocateBlock(0x200 * sizeof(float)); + villas::MemoryAccessor mem = *block; + dma->makeAccesibleFromVA(block); + + logger->info("Please enter values to write to the device, separated by ';'"); + + while (true) { + // Read values from stdin + std::string line; + std::getline(std::cin, line); + auto values = villas::utils::tokenize(line, ";"); + + size_t i = 0; + for (auto &value : values) { + if (value.empty()) + continue; + + const float number = std::stof(value); + mem[i++] = number; + } + + // Initiate write transfer + bool state = dma->write(*block, i * sizeof(float)); + if (!state) + logger->error("Failed to write to device"); + + auto writeComp = dma->writeComplete(); + logger->debug("Wrote {} bytes", writeComp.bytes); + } +} + +void readFromDmaToStdOut( + std::shared_ptr dma, + std::unique_ptr formatter) { + auto &alloc = villas::HostRam::getAllocator(); + + const std::shared_ptr block[] = { + alloc.allocateBlock(0x200 * sizeof(uint32_t)), + alloc.allocateBlock(0x200 * sizeof(uint32_t))}; + villas::MemoryAccessor mem[] = {*block[0], *block[1]}; + + for (auto b : block) { + dma->makeAccesibleFromVA(b); + } + + size_t cur = 0, next = 1; + std::ios::sync_with_stdio(false); + + // Setup read transfer + dma->read(*block[0], block[0]->getSize()); + + while (true) { + logger->trace("Read from stream and write to address {}:{:p}", + block[next]->getAddrSpaceId(), block[next]->getOffset()); + // We could use the number of interrupts to determine if we missed a chunk of data + dma->read(*block[next], block[next]->getSize()); + auto c = dma->readComplete(); + + if (c.interrupts > 1) { + logger->warn("Missed {} interrupts", c.interrupts - 1); + } + + logger->debug("bytes: {}, intrs: {}, bds: {}", c.bytes, c.interrupts, + c.bds); + try { + for (size_t i = 0; i * 4 < c.bytes; i++) { + int32_t ival = mem[cur][i]; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstrict-aliasing" + float fval = + *((float *)(&ival)); // cppcheck-suppress invalidPointerCast +#pragma GCC diagnostic pop + formatter->format(fval); + printf("%#x\n", ival); + } + formatter->output(std::cout); + } catch (const std::exception &e) { + logger->warn("Failed to output data: {}", e.what()); + } + + cur = next; + next = (next + 1) % (sizeof(mem) / sizeof(mem[0])); + } +} + + + +std::shared_ptr +setupCard(const std::string &configFilePath, const std::string &fpgaName) +{ + auto configDir = std::filesystem::path(configFilePath).parent_path(); + std::vector modules {"vfio"}; + auto vfioContainer = std::make_shared(modules); + + // Parse FPGA configuration + FILE* f = fopen(configFilePath.c_str(), "r"); + if (!f) + throw RuntimeError("Cannot open config file: {}", configFilePath); + + json_t* json = json_loadf(f, 0, nullptr); + if (!json) { + logger->error("Cannot parse JSON config"); + fclose(f); + throw RuntimeError("Cannot parse JSON config"); + } + + fclose(f); + + json_t* fpgas = json_object_get(json, "fpgas"); + if (fpgas == nullptr) { + logger->error("No section 'fpgas' found in config"); + exit(1); + } + + // // Get the FPGA card plugin + // auto fpgaCardFactory = plugin::registry->lookup("pcie"); + // if (fpgaCardFactory == nullptr) { + // logger->error("No FPGA plugin found"); + // exit(1); + // } + + // Create all FPGA card instances using the corresponding plugin + auto cards = PlatformCardFactory::make(fpgas, vfioContainer, configDir); + + std::shared_ptr card; + for (auto &fpgaCard : cards) { + if (fpgaCard->name == fpgaName) { + return fpgaCard; + } + } + + // Deallocate JSON config + json_decref(json); + + if (!card) + throw RuntimeError("FPGA card {} not found in config or not working", fpgaName); + + return card; +} + + +int main() +{ + logging.setLevel(spdlog::level::trace); + logger->set_level(spdlog::level::trace); + + std::shared_ptr card = setupCard("/home/root/fpga/build/src/fpgas.json","zcu106"); + + auto dma = std::dynamic_pointer_cast( + card->lookupIp(fpga::Vlnv("xilinx.com:ip:axi_dma:"))); + + auto axi_switch = std::dynamic_pointer_cast( + card->lookupIp(fpga::Vlnv("xilinx.com:ip:axis_switch:"))); + + axi_switch->connectInternal("S00_AXIS", "M00_AXIS"); + + // std::string outputFormat = "short"; + // auto formatter = fpga::getBufferedSampleFormatter(outputFormat, 16); + // readFromDmaToStdOut(dma, std::move(formatter)); + writeToDmaFromStdIn(dma); + + return 0; +} \ No newline at end of file diff --git a/src/villas-fpga-ctrl.cpp b/src/villas-fpga-ctrl.cpp index ba332ce4..25c60751 100644 --- a/src/villas-fpga-ctrl.cpp +++ b/src/villas-fpga-ctrl.cpp @@ -279,4 +279,4 @@ int main(int argc, char *argv[]) { } return 0; -} +} \ No newline at end of file diff --git a/thirdparty/libxil b/thirdparty/libxil index 0294cee4..1e51dba5 160000 --- a/thirdparty/libxil +++ b/thirdparty/libxil @@ -1 +1 @@ -Subproject commit 0294cee460cb6e4c09881c90b9b255ba9d7a99b3 +Subproject commit 1e51dba58390563987622d100aacfe9bf02413ae