From 320da081e9e7a560cb4ab012e8c2e2084bd9dab1 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Thu, 5 Dec 2024 17:41:47 +0100 Subject: [PATCH 01/12] created example base, argument parsing --- 68_JpegLoading/CMakeLists.txt | 27 ++++++++++++++++++ 68_JpegLoading/main.cpp | 46 +++++++++++++++++++++++++++++++ 68_JpegLoading/pipeline.groovy | 50 ++++++++++++++++++++++++++++++++++ CMakeLists.txt | 1 + 4 files changed, 124 insertions(+) create mode 100644 68_JpegLoading/CMakeLists.txt create mode 100644 68_JpegLoading/main.cpp create mode 100644 68_JpegLoading/pipeline.groovy diff --git a/68_JpegLoading/CMakeLists.txt b/68_JpegLoading/CMakeLists.txt new file mode 100644 index 000000000..54aa0c43b --- /dev/null +++ b/68_JpegLoading/CMakeLists.txt @@ -0,0 +1,27 @@ +include(common RESULT_VARIABLE RES) +if(NOT RES) + message(FATAL_ERROR "common.cmake not found. Should be in {repo_root}/cmake directory") +endif() + +nbl_create_executable_project("" "" "" "" "${NBL_EXECUTABLE_PROJECT_CREATION_PCH_TARGET}") + +if(NBL_EMBED_BUILTIN_RESOURCES) + set(_BR_TARGET_ ${EXECUTABLE_NAME}_builtinResourceData) + set(RESOURCE_DIR "app_resources") + + get_filename_component(_SEARCH_DIRECTORIES_ "${CMAKE_CURRENT_SOURCE_DIR}" ABSOLUTE) + get_filename_component(_OUTPUT_DIRECTORY_SOURCE_ "${CMAKE_CURRENT_BINARY_DIR}/src" ABSOLUTE) + get_filename_component(_OUTPUT_DIRECTORY_HEADER_ "${CMAKE_CURRENT_BINARY_DIR}/include" ABSOLUTE) + + file(GLOB_RECURSE BUILTIN_RESOURCE_FILES RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}/${RESOURCE_DIR}" CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${RESOURCE_DIR}/*") + foreach(RES_FILE ${BUILTIN_RESOURCE_FILES}) + LIST_BUILTIN_RESOURCE(RESOURCES_TO_EMBED "${RES_FILE}") + endforeach() + + ADD_CUSTOM_BUILTIN_RESOURCES(${_BR_TARGET_} RESOURCES_TO_EMBED "${_SEARCH_DIRECTORIES_}" "${RESOURCE_DIR}" "nbl::this_example::builtin" "${_OUTPUT_DIRECTORY_HEADER_}" "${_OUTPUT_DIRECTORY_SOURCE_}") + + LINK_BUILTIN_RESOURCES_TO_TARGET(${EXECUTABLE_NAME} ${_BR_TARGET_}) +endif() + +add_dependencies(${EXECUTABLE_NAME} argparse) +target_include_directories(${EXECUTABLE_NAME} PUBLIC $) \ No newline at end of file diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp new file mode 100644 index 000000000..c613679d1 --- /dev/null +++ b/68_JpegLoading/main.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2018-2024 - DevSH Graphics Programming Sp. z O.O. +// This file is part of the "Nabla Engine". +// For conditions of distribution and use, see copyright notice in nabla.h +#include "nbl/application_templates/MonoAssetManagerAndBuiltinResourceApplication.hpp" + +#include "nlohmann/json.hpp" +#include "argparse/argparse.hpp" + +using json = nlohmann::json; + +using namespace nbl; +using namespace core; +using namespace system; +using namespace asset; +using namespace video; + +class JpegLoadingSampleApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication +{ +public: + inline bool onAppInitialized(smart_refctd_ptr&& system) override + { + argparse::ArgumentParser program("JPEG Loader"); + + program.add_argument("--directory") + .default_value(false) + .implicit_value(true) + .help("Path to the directory with JPEGs (not recursive)"); + + program.add_argument("--output") + .default_value(false) + .implicit_value(true) + .help("Path to the json file with benchmark results"); + + try + { + program.parse_args({ argv.data(), argv.data() + argv.size() }); + } + catch (const std::exception& err) + { + std::cerr << err.what() << std::endl << program; + return 1; + } + } +}; + +NBL_MAIN_FUNC(JpegLoadingSampleApp); \ No newline at end of file diff --git a/68_JpegLoading/pipeline.groovy b/68_JpegLoading/pipeline.groovy new file mode 100644 index 000000000..ece6ffebc --- /dev/null +++ b/68_JpegLoading/pipeline.groovy @@ -0,0 +1,50 @@ +import org.DevshGraphicsProgramming.Agent +import org.DevshGraphicsProgramming.BuilderInfo +import org.DevshGraphicsProgramming.IBuilder + +class CJpegLoading extends IBuilder +{ + public CJpegLoading(Agent _agent, _info) + { + super(_agent, _info) + } + + @Override + public boolean prepare(Map axisMapping) + { + return true + } + + @Override + public boolean build(Map axisMapping) + { + IBuilder.CONFIGURATION config = axisMapping.get("CONFIGURATION") + IBuilder.BUILD_TYPE buildType = axisMapping.get("BUILD_TYPE") + + def nameOfBuildDirectory = getNameOfBuildDirectory(buildType) + def nameOfConfig = getNameOfConfig(config) + + agent.execute("cmake --build ${info.rootProjectPath}/${nameOfBuildDirectory}/${info.targetProjectPathRelativeToRoot} --target ${info.targetBaseName} --config ${nameOfConfig} -j12 -v") + + return true + } + + @Override + public boolean test(Map axisMapping) + { + return true + } + + @Override + public boolean install(Map axisMapping) + { + return true + } +} + +def create(Agent _agent, _info) +{ + return new CJpegLoading(_agent, _info) +} + +return this \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bf3eb696..e1195dd39 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,6 +83,7 @@ if(NBL_BUILD_EXAMPLES) add_subdirectory(0_ImportanceSamplingEnvMaps EXCLUDE_FROM_ALL) #TODO: integrate back into 42 add_subdirectory(67_RayQueryGeometry EXCLUDE_FROM_ALL) + add_subdirectory(68_JpegLoading EXCLUDE_FROM_ALL) NBL_HOOK_COMMON_API("${NBL_COMMON_API_TARGETS}") endif() \ No newline at end of file From d44fff78b5a08cbca5e49adf8b070f9aa7a7e01c Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sat, 7 Dec 2024 13:31:41 +0100 Subject: [PATCH 02/12] Added missing ctor, corrected argument parsing mechanism --- 68_JpegLoading/main.cpp | 70 ++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index c613679d1..1cc268132 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -1,8 +1,14 @@ -// Copyright (C) 2018-2024 - DevSH Graphics Programming Sp. z O.O. +// Copyright (C) 2018-2024 - DevSH GrapMonoAssetManagerAndBuiltinResourceApplicationhics Programming Sp. z O.O. // This file is part of the "Nabla Engine". // For conditions of distribution and use, see copyright notice in nabla.h #include "nbl/application_templates/MonoAssetManagerAndBuiltinResourceApplication.hpp" +#include +#include +#include +#include +#include + #include "nlohmann/json.hpp" #include "argparse/argparse.hpp" @@ -10,37 +16,71 @@ using json = nlohmann::json; using namespace nbl; using namespace core; +using namespace hlsl; using namespace system; using namespace asset; +using namespace ui; using namespace video; -class JpegLoadingSampleApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication +constexpr static uint32_t s_maxWorkers = 8; + +class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication { + using base_t = application_templates::MonoAssetManagerAndBuiltinResourceApplication; + using clock_t = std::chrono::steady_clock; + constexpr static inline uint32_t m_maxWorkers = 8; public: + using base_t::base_t; + inline bool onAppInitialized(smart_refctd_ptr&& system) override { - argparse::ArgumentParser program("JPEG Loader"); + argparse::ArgumentParser program("Color Space"); - program.add_argument("--directory") - .default_value(false) - .implicit_value(true) - .help("Path to the directory with JPEGs (not recursive)"); + program.add_argument("--directory") + .required() + .help("Path to a directory where all JPEG files are stored (not recursive)"); - program.add_argument("--output") - .default_value(false) - .implicit_value(true) - .help("Path to the json file with benchmark results"); + program.add_argument("--output") + .default_value("output.json") + .help("Path to the file where the benchmark result will be stored"); - try + try { program.parse_args({ argv.data(), argv.data() + argv.size() }); } catch (const std::exception& err) { - std::cerr << err.what() << std::endl << program; + std::cerr << err.what() << std::endl << program; // NOTE: std::cerr because logger isn't initialized yet return 1; } - } + + if (!base_t::onAppInitialized(std::move(system))) + return false; + + options.directory = program.get("--directory"); + options.outputFile = program.get("--output"); + + // TODO: Load JPEGs + // TODO: Measure the time + + return true; + } + + inline bool keepRunning() override + { + return false; + } + + inline void workLoopBody() override + { + + } +private: + struct NBL_APP_OPTIONS + { + std::string directory; + std::string outputFile; + } options; }; -NBL_MAIN_FUNC(JpegLoadingSampleApp); \ No newline at end of file +NBL_MAIN_FUNC(JpegLoaderApp) \ No newline at end of file From ec93998ffe6439c19d6648d6a5d30510e7b2f0c8 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sat, 7 Dec 2024 14:15:10 +0100 Subject: [PATCH 03/12] added basic thread pool class --- 68_JpegLoading/main.cpp | 58 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 1cc268132..364cae2d9 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include "nlohmann/json.hpp" #include "argparse/argparse.hpp" @@ -22,7 +23,60 @@ using namespace asset; using namespace ui; using namespace video; -constexpr static uint32_t s_maxWorkers = 8; +class ThreadPool { +public: + using task_t = std::function; + + ThreadPool() { + for (size_t i = 0; i < m_maxWorkers; i++) { + m_threads.emplace_back([this]() { + while (true) { + task_t task; + + { + std::unique_lock lock(m_queueMutex); + m_taskAvailable.wait(lock, [&]() { return !m_tasks.empty() || m_stop; }); + + if (m_stop && m_tasks.empty()) + return; + + task = std::move(m_tasks.front()); + m_tasks.pop(); + } + + task(); + } + }); + } + } + + ~ThreadPool() { + { + std::lock_guard lock(m_queueMutex); + m_stop = true; + } + + m_taskAvailable.notify_all(); + + for (auto& thread : m_threads) { + thread.join(); + } + } + + void addTask(task_t task) { + std::lock_guard queueLock(m_queueMutex); + m_tasks.emplace(std::move(task)); + m_taskAvailable.notify_one(); + } +private: + const size_t m_maxWorkers = std::thread::hardware_concurrency(); + std::vector m_threads; + std::condition_variable m_taskAvailable; + bool m_stop = false; // stops the threads on object destruction + + std::queue m_tasks; + std::mutex m_queueMutex; +}; class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication { @@ -60,6 +114,8 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui options.directory = program.get("--directory"); options.outputFile = program.get("--output"); + + ThreadPool threadPool; // TODO: Load JPEGs // TODO: Measure the time From 09e36c7a86d8ba80a4dd9ac61c1620f3749a562c Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 9 Dec 2024 17:14:22 +0100 Subject: [PATCH 04/12] return false instead of 1 in argument parsing try/catch --- 68_JpegLoading/main.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 364cae2d9..a9bb7eff0 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -105,7 +105,7 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui catch (const std::exception& err) { std::cerr << err.what() << std::endl << program; // NOTE: std::cerr because logger isn't initialized yet - return 1; + return false; } if (!base_t::onAppInitialized(std::move(system))) @@ -116,6 +116,7 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui ThreadPool threadPool; + // TODO: Load JPEGs // TODO: Measure the time From c77c9c85da12898febabc91ce798661d60e823b8 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Mon, 9 Dec 2024 19:27:59 +0100 Subject: [PATCH 05/12] single-threaded image loading with time measuring --- 68_JpegLoading/main.cpp | 91 +++++++++++------------------------------ 1 file changed, 25 insertions(+), 66 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index a9bb7eff0..5ca08f2b4 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -3,12 +3,7 @@ // For conditions of distribution and use, see copyright notice in nabla.h #include "nbl/application_templates/MonoAssetManagerAndBuiltinResourceApplication.hpp" -#include -#include -#include -#include -#include -#include +#include #include "nlohmann/json.hpp" #include "argparse/argparse.hpp" @@ -23,66 +18,11 @@ using namespace asset; using namespace ui; using namespace video; -class ThreadPool { -public: - using task_t = std::function; - - ThreadPool() { - for (size_t i = 0; i < m_maxWorkers; i++) { - m_threads.emplace_back([this]() { - while (true) { - task_t task; - - { - std::unique_lock lock(m_queueMutex); - m_taskAvailable.wait(lock, [&]() { return !m_tasks.empty() || m_stop; }); - - if (m_stop && m_tasks.empty()) - return; - - task = std::move(m_tasks.front()); - m_tasks.pop(); - } - - task(); - } - }); - } - } - - ~ThreadPool() { - { - std::lock_guard lock(m_queueMutex); - m_stop = true; - } - - m_taskAvailable.notify_all(); - - for (auto& thread : m_threads) { - thread.join(); - } - } - - void addTask(task_t task) { - std::lock_guard queueLock(m_queueMutex); - m_tasks.emplace(std::move(task)); - m_taskAvailable.notify_one(); - } -private: - const size_t m_maxWorkers = std::thread::hardware_concurrency(); - std::vector m_threads; - std::condition_variable m_taskAvailable; - bool m_stop = false; // stops the threads on object destruction - - std::queue m_tasks; - std::mutex m_queueMutex; -}; - class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication { - using base_t = application_templates::MonoAssetManagerAndBuiltinResourceApplication; using clock_t = std::chrono::steady_clock; - constexpr static inline uint32_t m_maxWorkers = 8; + using clock_resolution_t = std::chrono::milliseconds; + using base_t = application_templates::MonoAssetManagerAndBuiltinResourceApplication; public: using base_t::base_t; @@ -114,11 +54,30 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui options.directory = program.get("--directory"); options.outputFile = program.get("--output"); + auto start = clock_t::now(); + + { // TODO: Make this multi-threaded + constexpr auto cachingFlags = static_cast(IAssetLoader::ECF_DONT_CACHE_REFERENCES & IAssetLoader::ECF_DONT_CACHE_TOP_LEVEL); + const IAssetLoader::SAssetLoadParams loadParams(0ull, nullptr, cachingFlags, IAssetLoader::ELPF_NONE, m_logger.get()); + + for (auto& item : std::filesystem::directory_iterator(options.directory)) + { + + auto& path = item.path(); + auto extension = path.extension(); + + if (path.has_extension() && extension == ".jpg") + { + m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); + m_assetMgr->getAsset(path.generic_string(), loadParams); // discard the loaded image + } + } + } - ThreadPool threadPool; + auto stop = clock_t::now(); + auto passed = std::chrono::duration_cast(stop - start).count(); - // TODO: Load JPEGs - // TODO: Measure the time + m_logger->log("Process took %llu ms", ILogger::ELL_INFO, passed); return true; } From 0dd4022887e898ad8ec7a35a273f2bddcb32170f Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Wed, 11 Dec 2024 19:48:53 +0100 Subject: [PATCH 06/12] made jpeg loading multithreaded using thread pool --- 68_JpegLoading/main.cpp | 89 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 5ca08f2b4..9947e50b8 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -18,6 +18,68 @@ using namespace asset; using namespace ui; using namespace video; +class ThreadPool +{ + using task_t = std::function; +public: + ThreadPool(size_t workers = std::thread::hardware_concurrency()) + { + for (size_t i = 0; i < workers; i++) + { + m_workers.emplace_back([this] { + task_t task; + + while (1) + { + { + std::unique_lock lock(m_queueLock); + m_taskAvailable.wait(lock, [this] { return !m_tasks.empty() || m_shouldStop; }); + + if (m_shouldStop && m_tasks.empty()) { + return; + } + + task = std::move(m_tasks.front()); + m_tasks.pop(); + } + + task(); + } + }); + } + } + + ~ThreadPool() + { + { + std::lock_guard lock(m_queueLock); + m_shouldStop = true; + } + + m_taskAvailable.notify_all(); + + for (auto& worker : m_workers) + { + worker.join(); + } + } + + void enqueue(task_t task) + { + { + std::lock_guard lock(m_queueLock); + m_tasks.emplace(std::move(task)); + } + m_taskAvailable.notify_one(); + } +private: + std::mutex m_queueLock; + std::condition_variable m_taskAvailable; + std::vector m_workers; + std::queue m_tasks; + bool m_shouldStop = false; +}; + class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication { using clock_t = std::chrono::steady_clock; @@ -54,31 +116,38 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui options.directory = program.get("--directory"); options.outputFile = program.get("--output"); + // check if directory exists + if (!std::filesystem::exists(options.directory)) + { + logFail("Provided directory doesn't exist"); + return false; + } + auto start = clock_t::now(); - { // TODO: Make this multi-threaded + { + ThreadPool tp; + constexpr auto cachingFlags = static_cast(IAssetLoader::ECF_DONT_CACHE_REFERENCES & IAssetLoader::ECF_DONT_CACHE_TOP_LEVEL); const IAssetLoader::SAssetLoadParams loadParams(0ull, nullptr, cachingFlags, IAssetLoader::ELPF_NONE, m_logger.get()); for (auto& item : std::filesystem::directory_iterator(options.directory)) { - auto& path = item.path(); - auto extension = path.extension(); - - if (path.has_extension() && extension == ".jpg") + if (path.has_extension() && path.extension() == ".jpg") { - m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); - m_assetMgr->getAsset(path.generic_string(), loadParams); // discard the loaded image + tp.enqueue([=] { + m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); + m_assetMgr->getAsset(path.generic_string(), loadParams); + }); } } } auto stop = clock_t::now(); - auto passed = std::chrono::duration_cast(stop - start).count(); - - m_logger->log("Process took %llu ms", ILogger::ELL_INFO, passed); + auto time = std::chrono::duration_cast(stop - start).count(); + m_logger->log("Process took %llu ms", ILogger::ELL_INFO, time); return true; } From db2bd65862b511616b20bc69d8cb39e1e05c5462 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Wed, 11 Dec 2024 20:04:12 +0100 Subject: [PATCH 07/12] added results parsing --- 68_JpegLoading/main.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 9947e50b8..fa1338efb 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -124,6 +124,7 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui } auto start = clock_t::now(); + std::vector files; { ThreadPool tp; @@ -136,6 +137,7 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui auto& path = item.path(); if (path.has_extension() && path.extension() == ".jpg") { + files.emplace_back(std::move(path.generic_string())); tp.enqueue([=] { m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); m_assetMgr->getAsset(path.generic_string(), loadParams); @@ -148,6 +150,21 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui auto time = std::chrono::duration_cast(stop - start).count(); m_logger->log("Process took %llu ms", ILogger::ELL_INFO, time); + + // Dump data to JSON + json j; + j["loaded_files"] = files; + j["duration_ms"] = time; + + std::ofstream output(options.outputFile); + if (!output.good()) + { + logFail("Failed to open %S", options.outputFile); + return false; + } + + output << j; + return true; } From 8e8dd96c39e3809df127a6526cdfbbf5c3fc0f94 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Wed, 11 Dec 2024 20:13:09 +0100 Subject: [PATCH 08/12] made m_shouldStop atomic --- 68_JpegLoading/main.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index fa1338efb..a24c0db68 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -51,11 +51,7 @@ class ThreadPool ~ThreadPool() { - { - std::lock_guard lock(m_queueLock); - m_shouldStop = true; - } - + m_shouldStop = true; m_taskAvailable.notify_all(); for (auto& worker : m_workers) @@ -77,7 +73,7 @@ class ThreadPool std::condition_variable m_taskAvailable; std::vector m_workers; std::queue m_tasks; - bool m_shouldStop = false; + std::atomic m_shouldStop; }; class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication From 8ad3d18fc785b5b3e92a57e983c0135f9beaf3ae Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Wed, 11 Dec 2024 20:16:52 +0100 Subject: [PATCH 09/12] set m_shouldStop to false by default --- 68_JpegLoading/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index a24c0db68..1866939e8 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -73,7 +73,7 @@ class ThreadPool std::condition_variable m_taskAvailable; std::vector m_workers; std::queue m_tasks; - std::atomic m_shouldStop; + std::atomic m_shouldStop = false; }; class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication From 85e0dad290007d001cb1d0d1a8c080423d623eb9 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Sun, 15 Dec 2024 12:22:31 +0100 Subject: [PATCH 10/12] set tab space to 3 --- 68_JpegLoading/main.cpp | 307 ++++++++++++++++++++-------------------- 1 file changed, 154 insertions(+), 153 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 1866939e8..ca3e7f226 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -20,165 +20,166 @@ using namespace video; class ThreadPool { - using task_t = std::function; +using task_t = std::function; public: - ThreadPool(size_t workers = std::thread::hardware_concurrency()) - { - for (size_t i = 0; i < workers; i++) - { - m_workers.emplace_back([this] { - task_t task; - - while (1) - { - { - std::unique_lock lock(m_queueLock); - m_taskAvailable.wait(lock, [this] { return !m_tasks.empty() || m_shouldStop; }); - - if (m_shouldStop && m_tasks.empty()) { - return; - } - - task = std::move(m_tasks.front()); - m_tasks.pop(); - } - - task(); - } - }); - } - } - - ~ThreadPool() - { - m_shouldStop = true; - m_taskAvailable.notify_all(); - - for (auto& worker : m_workers) - { - worker.join(); - } - } - - void enqueue(task_t task) - { - { - std::lock_guard lock(m_queueLock); - m_tasks.emplace(std::move(task)); - } - m_taskAvailable.notify_one(); - } -private: - std::mutex m_queueLock; - std::condition_variable m_taskAvailable; - std::vector m_workers; - std::queue m_tasks; - std::atomic m_shouldStop = false; + ThreadPool(size_t workers = std::thread::hardware_concurrency()) + { + for (size_t i = 0; i < workers; i++) + { + m_workers.emplace_back([this] { + task_t task; + + while (1) + { + { + std::unique_lock lock(m_queueLock); + m_taskAvailable.wait(lock, [this] { return !m_tasks.empty() || m_shouldStop; }); + + if (m_shouldStop && m_tasks.empty()) { + return; + } + + task = std::move(m_tasks.front()); + m_tasks.pop(); + } + + task(); + } + }); + } + } + + ~ThreadPool() + { + m_shouldStop = true; + m_taskAvailable.notify_all(); + + for (auto& worker : m_workers) + { + worker.join(); + } + } + + void enqueue(task_t task) + { + { + std::lock_guard lock(m_queueLock); + m_tasks.emplace(std::move(task)); + } + m_taskAvailable.notify_one(); + } + private: + std::mutex m_queueLock; + std::condition_variable m_taskAvailable; + std::vector m_workers; + std::queue m_tasks; + std::atomic m_shouldStop = false; }; class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBuiltinResourceApplication { - using clock_t = std::chrono::steady_clock; - using clock_resolution_t = std::chrono::milliseconds; - using base_t = application_templates::MonoAssetManagerAndBuiltinResourceApplication; -public: - using base_t::base_t; - - inline bool onAppInitialized(smart_refctd_ptr&& system) override - { - argparse::ArgumentParser program("Color Space"); - - program.add_argument("--directory") - .required() - .help("Path to a directory where all JPEG files are stored (not recursive)"); - - program.add_argument("--output") - .default_value("output.json") - .help("Path to the file where the benchmark result will be stored"); - - try - { - program.parse_args({ argv.data(), argv.data() + argv.size() }); - } - catch (const std::exception& err) - { - std::cerr << err.what() << std::endl << program; // NOTE: std::cerr because logger isn't initialized yet - return false; - } - - if (!base_t::onAppInitialized(std::move(system))) - return false; - - options.directory = program.get("--directory"); - options.outputFile = program.get("--output"); - - // check if directory exists - if (!std::filesystem::exists(options.directory)) - { - logFail("Provided directory doesn't exist"); - return false; - } - - auto start = clock_t::now(); - std::vector files; - - { - ThreadPool tp; - - constexpr auto cachingFlags = static_cast(IAssetLoader::ECF_DONT_CACHE_REFERENCES & IAssetLoader::ECF_DONT_CACHE_TOP_LEVEL); - const IAssetLoader::SAssetLoadParams loadParams(0ull, nullptr, cachingFlags, IAssetLoader::ELPF_NONE, m_logger.get()); - - for (auto& item : std::filesystem::directory_iterator(options.directory)) - { - auto& path = item.path(); - if (path.has_extension() && path.extension() == ".jpg") - { - files.emplace_back(std::move(path.generic_string())); - tp.enqueue([=] { - m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); - m_assetMgr->getAsset(path.generic_string(), loadParams); - }); - } - } - } - - auto stop = clock_t::now(); - auto time = std::chrono::duration_cast(stop - start).count(); - - m_logger->log("Process took %llu ms", ILogger::ELL_INFO, time); - - // Dump data to JSON - json j; - j["loaded_files"] = files; - j["duration_ms"] = time; - - std::ofstream output(options.outputFile); - if (!output.good()) - { - logFail("Failed to open %S", options.outputFile); - return false; - } - - output << j; - - return true; - } - - inline bool keepRunning() override - { - return false; - } - - inline void workLoopBody() override - { - - } + using clock_t = std::chrono::steady_clock; + using clock_resolution_t = std::chrono::milliseconds; + using base_t = application_templates::MonoAssetManagerAndBuiltinResourceApplication; + public: + using base_t::base_t; + + inline bool onAppInitialized(smart_refctd_ptr&& system) override + { + argparse::ArgumentParser program("Color Space"); + + program.add_argument("--directory") + .required() + .help("Path to a directory where all JPEG files are stored (not recursive)"); + + program.add_argument("--output") + .default_value("output.json") + .help("Path to the file where the benchmark result will be stored"); + + try + { + program.parse_args({ argv.data(), argv.data() + argv.size() }); + } + catch (const std::exception& err) + { + std::cerr << err.what() << std::endl << program; // NOTE: std::cerr because logger isn't initialized yet + return false; + } + + if (!base_t::onAppInitialized(std::move(system))) + return false; + + options.directory = program.get("--directory"); + options.outputFile = program.get("--output"); + + // check if directory exists + if (!std::filesystem::exists(options.directory)) + { + logFail("Provided directory doesn't exist"); + return false; + } + + auto start = clock_t::now(); + std::vector files; + + { + ThreadPool tp; + + constexpr auto cachingFlags = static_cast(IAssetLoader::ECF_DONT_CACHE_REFERENCES & IAssetLoader::ECF_DONT_CACHE_TOP_LEVEL); + const IAssetLoader::SAssetLoadParams loadParams(0ull, nullptr, cachingFlags, IAssetLoader::ELPF_NONE, m_logger.get()); + + for (auto& item : std::filesystem::directory_iterator(options.directory)) + { + auto& path = item.path(); + if (path.has_extension() && path.extension() == ".jpg") + { + files.emplace_back(std::move(path.generic_string())); + tp.enqueue([=] { + m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); + m_assetMgr->getAsset(path.generic_string(), loadParams); + }); + } + } + } + + auto stop = clock_t::now(); + auto time = std::chrono::duration_cast(stop - start).count(); + + m_logger->log("Process took %llu ms", ILogger::ELL_INFO, time); + + // Dump data to JSON + json j; + j["loaded_files"] = files; + j["duration_ms"] = time; + + std::ofstream output(options.outputFile); + if (!output.good()) + { + logFail("Failed to open %S", options.outputFile); + return false; + } + + output << j; + + return true; + } + + inline bool keepRunning() override + { + return false; + } + + inline void workLoopBody() override + { + + } + private: - struct NBL_APP_OPTIONS - { - std::string directory; - std::string outputFile; - } options; + struct NBL_APP_OPTIONS + { + std::string directory; + std::string outputFile; + } options; }; NBL_MAIN_FUNC(JpegLoaderApp) \ No newline at end of file From e589241cd64d02150c6eed66cf7e255ace95a713 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Wed, 18 Dec 2024 19:07:30 +0100 Subject: [PATCH 11/12] use IFile pointer instead of file path --- 68_JpegLoading/main.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index ca3e7f226..5b3127967 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -134,10 +134,18 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui if (path.has_extension() && path.extension() == ".jpg") { files.emplace_back(std::move(path.generic_string())); - tp.enqueue([=] { - m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); - m_assetMgr->getAsset(path.generic_string(), loadParams); - }); + + ISystem::future_t> future; + m_system->createFile(future, path, IFile::ECF_READ); + + if (auto pFile = future.acquire(); pFile && pFile->get()) + { + auto& file = *pFile; + tp.enqueue([=] { + m_logger->log("Loading %S", ILogger::ELL_INFO, path.c_str()); + m_assetMgr->getAsset(file.get(), path.generic_string(), loadParams); + }); + } } } } From ed112d1eed585925079bbd1d0387a0b475865e81 Mon Sep 17 00:00:00 2001 From: YasInvolved Date: Fri, 20 Dec 2024 17:37:18 +0100 Subject: [PATCH 12/12] file loading with ECF_MAPPABLE --- 68_JpegLoading/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/68_JpegLoading/main.cpp b/68_JpegLoading/main.cpp index 5b3127967..5ef9b637d 100644 --- a/68_JpegLoading/main.cpp +++ b/68_JpegLoading/main.cpp @@ -136,7 +136,7 @@ class JpegLoaderApp final : public application_templates::MonoAssetManagerAndBui files.emplace_back(std::move(path.generic_string())); ISystem::future_t> future; - m_system->createFile(future, path, IFile::ECF_READ); + m_system->createFile(future, path, IFile::ECF_READ | IFile::ECF_MAPPABLE); if (auto pFile = future.acquire(); pFile && pFile->get()) {