Skip to content

Commit

Permalink
Separate implementation from interface
Browse files Browse the repository at this point in the history
  • Loading branch information
@kingcrimsontianyu
kingcrimsontianyu committed Jan 7, 2025
1 parent 58a0784 commit 718b078
Show file tree
Hide file tree
Showing 13 changed files with 786 additions and 530 deletions.
6 changes: 4 additions & 2 deletions cpp/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# =============================================================================
# Copyright (c) 2021-2024, NVIDIA CORPORATION.
# Copyright (c) 2021-2025, NVIDIA CORPORATION.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
# in compliance with the License. You may obtain a copy of the License at
Expand Down Expand Up @@ -131,7 +131,9 @@ include(cmake/thirdparty/get_thread_pool.cmake)
# ##################################################################################################
# * library targets --------------------------------------------------------------------------------

set(SOURCES "src/file_handle.cpp")
set(SOURCES "src/file_handle.cpp" "src/cufile/config.cpp" "src/cufile/driver.cpp"
"src/shim/cuda.cpp" "src/shim/cufile.cpp" "src/shim/libcurl.cpp" "src/shim/utils.cpp"
)

if(KvikIO_REMOTE_SUPPORT)
list(APPEND SOURCES "src/remote_handle.cpp")
Expand Down
23 changes: 2 additions & 21 deletions cpp/include/kvikio/cufile/config.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2024, NVIDIA CORPORATION.
* Copyright (c) 2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -15,24 +15,9 @@
*/
#pragma once

#include <cstdlib>
#include <filesystem>
#include <string>

#include <kvikio/utils.hpp>

namespace kvikio {
namespace detail {

[[nodiscard]] inline const char* lookup_config_path()
{
const char* env = std::getenv("CUFILE_ENV_PATH_JSON");
if (env != nullptr && std::filesystem::exists(env)) { return env; }
if (std::filesystem::exists("/etc/cufile.json")) { return "/etc/cufile.json"; }
return "";
}

} // namespace detail

/**
* @brief Get the filepath to cuFile's config file (`cufile.json`) or the empty string
Expand All @@ -41,10 +26,6 @@ namespace detail {
*
* @return The filepath to the cufile.json file or the empty string if it isn't found.
*/
[[nodiscard]] KVIKIO_EXPORT inline const std::string& config_path()
{
static const std::string ret = detail::lookup_config_path();
return ret;
}
[[nodiscard]] KVIKIO_EXPORT const std::string& config_path();

} // namespace kvikio
232 changes: 22 additions & 210 deletions cpp/include/kvikio/cufile/driver.hpp
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2021-2024, NVIDIA CORPORATION.
* Copyright (c) 2021-2025, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -15,53 +15,24 @@
*/
#pragma once

#include <iostream>
#include <vector>

#include <kvikio/error.hpp>
#include <kvikio/shim/cufile.hpp>
#include <kvikio/shim/cufile_h_wrapper.hpp>

namespace kvikio {
namespace detail {

[[nodiscard]] inline bool get_driver_flag(unsigned int prop, unsigned int flag) noexcept
{
return (prop & (1U << flag)) != 0;
}

inline void set_driver_flag(unsigned int& prop, unsigned int flag, bool val) noexcept
{
if (val) {
prop |= (1U << flag);
} else {
prop &= ~(1U << flag);
}
}
} // namespace detail

#ifdef KVIKIO_CUFILE_FOUND

class DriverInitializer {
// Optional, if not used cuFiles opens the driver automatically
public:
DriverInitializer() { cuFileAPI::instance().driver_open(); }
DriverInitializer();

DriverInitializer(DriverInitializer const&) = delete;
DriverInitializer& operator=(DriverInitializer const&) = delete;
DriverInitializer(DriverInitializer&&) noexcept = delete;
DriverInitializer& operator=(DriverInitializer&&) noexcept = delete;

~DriverInitializer()
{
try {
cuFileAPI::instance().driver_close();
} catch (const CUfileException& e) {
std::cerr << "Unable to close GDS file driver: ";
std::cerr << e.what();
std::cerr << std::endl;
}
}
~DriverInitializer();
};

class DriverProperties {
Expand All @@ -71,204 +42,45 @@ class DriverProperties {

// Because Cython does not handle exceptions in the default
// constructor, we initialize `_props` lazily.
void lazy_init()
{
if (_initialized) { return; }
_initialized = true;
CUFILE_TRY(cuFileAPI::instance().DriverGetProperties(&_props));
}
void lazy_init();

public:
#ifdef KVIKIO_CUFILE_FOUND
DriverProperties() = default;

bool is_gds_available()
{
// If both the major and minor version is zero, the GDS driver isn't loaded.
return !(get_nvfs_major_version() == 0 && get_nvfs_minor_version() == 0);
}

[[nodiscard]] unsigned int get_nvfs_major_version()
{
lazy_init();
return _props.nvfs.major_version;
}

[[nodiscard]] unsigned int get_nvfs_minor_version()
{
lazy_init();
return _props.nvfs.minor_version;
}

[[nodiscard]] bool get_nvfs_allow_compat_mode()
{
lazy_init();
return detail::get_driver_flag(_props.nvfs.dcontrolflags, CU_FILE_ALLOW_COMPAT_MODE);
}

[[nodiscard]] bool get_nvfs_poll_mode()
{
lazy_init();
return detail::get_driver_flag(_props.nvfs.dcontrolflags, CU_FILE_USE_POLL_MODE);
}

[[nodiscard]] std::size_t get_nvfs_poll_thresh_size()
{
lazy_init();
return _props.nvfs.poll_thresh_size;
}

void set_nvfs_poll_mode(bool enable)
{
lazy_init();
CUFILE_TRY(cuFileAPI::instance().DriverSetPollMode(enable, get_nvfs_poll_thresh_size()));
detail::set_driver_flag(_props.nvfs.dcontrolflags, CU_FILE_USE_POLL_MODE, enable);
}

void set_nvfs_poll_thresh_size(std::size_t size_in_kb)
{
lazy_init();
CUFILE_TRY(cuFileAPI::instance().DriverSetPollMode(get_nvfs_poll_mode(), size_in_kb));
_props.nvfs.poll_thresh_size = size_in_kb;
}

[[nodiscard]] std::vector<CUfileDriverControlFlags> get_nvfs_statusflags()
{
lazy_init();
std::vector<CUfileDriverControlFlags> ret;
if (detail::get_driver_flag(_props.nvfs.dcontrolflags, CU_FILE_USE_POLL_MODE)) {
ret.push_back(CU_FILE_USE_POLL_MODE);
}
if (detail::get_driver_flag(_props.nvfs.dcontrolflags, CU_FILE_ALLOW_COMPAT_MODE)) {
ret.push_back(CU_FILE_ALLOW_COMPAT_MODE);
}
return ret;
}

[[nodiscard]] std::size_t get_max_device_cache_size()
{
lazy_init();
return _props.max_device_cache_size;
}

void set_max_device_cache_size(std::size_t size_in_kb)
{
lazy_init();
CUFILE_TRY(cuFileAPI::instance().DriverSetMaxCacheSize(size_in_kb));
_props.max_device_cache_size = size_in_kb;
}

[[nodiscard]] std::size_t get_per_buffer_cache_size()
{
lazy_init();
return _props.per_buffer_cache_size;
}

[[nodiscard]] std::size_t get_max_pinned_memory_size()
{
lazy_init();
return _props.max_device_pinned_mem_size;
}

void set_max_pinned_memory_size(std::size_t size_in_kb)
{
lazy_init();
CUFILE_TRY(cuFileAPI::instance().DriverSetMaxPinnedMemSize(size_in_kb));
_props.max_device_pinned_mem_size = size_in_kb;
}

[[nodiscard]] std::size_t get_max_batch_io_size()
{
#ifdef KVIKIO_CUFILE_BATCH_API_FOUND
lazy_init();
return _props.max_batch_io_size;
#else
return 0;
#endif
}
};

#else
struct DriverInitializer {
// Implement a non-default constructor to avoid `unused variable` warnings downstream
DriverInitializer() {}
};

struct DriverProperties {
// Implement a non-default constructor to avoid `unused variable` warnings downstream
DriverProperties() {}
DriverProperties();
#endif

static bool is_gds_available() { return false; }
bool is_gds_available();

[[nodiscard]] static unsigned int get_nvfs_major_version()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] unsigned int get_nvfs_major_version();

[[nodiscard]] static unsigned int get_nvfs_minor_version()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] unsigned int get_nvfs_minor_version();

[[nodiscard]] static bool get_nvfs_allow_compat_mode()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] bool get_nvfs_allow_compat_mode();

[[nodiscard]] static bool get_nvfs_poll_mode()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] bool get_nvfs_poll_mode();

[[nodiscard]] static std::size_t get_nvfs_poll_thresh_size()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::size_t get_nvfs_poll_thresh_size();

static void set_nvfs_poll_mode(bool enable)
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
void set_nvfs_poll_mode(bool enable);

static void set_nvfs_poll_thresh_size(std::size_t size_in_kb)
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
void set_nvfs_poll_thresh_size(std::size_t size_in_kb);

[[nodiscard]] static std::vector<CUfileDriverControlFlags> get_nvfs_statusflags()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::vector<CUfileDriverControlFlags> get_nvfs_statusflags();

[[nodiscard]] static std::size_t get_max_device_cache_size()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::size_t get_max_device_cache_size();

static void set_max_device_cache_size(std::size_t size_in_kb)
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
void set_max_device_cache_size(std::size_t size_in_kb);

[[nodiscard]] static std::size_t get_per_buffer_cache_size()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::size_t get_per_buffer_cache_size();

[[nodiscard]] static std::size_t get_max_pinned_memory_size()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::size_t get_max_pinned_memory_size();

static void set_max_pinned_memory_size(std::size_t size_in_kb)
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
void set_max_pinned_memory_size(std::size_t size_in_kb);

[[nodiscard]] std::size_t get_max_batch_io_size()
{
throw CUfileException("KvikIO not compiled with cuFile.h");
}
[[nodiscard]] std::size_t get_max_batch_io_size();
};
#endif

} // namespace kvikio
Loading

0 comments on commit 718b078

Please sign in to comment.