diff --git a/libmamba/CMakeLists.txt b/libmamba/CMakeLists.txt index 5fe133ceb6..f3629c4049 100644 --- a/libmamba/CMakeLists.txt +++ b/libmamba/CMakeLists.txt @@ -162,8 +162,8 @@ set( ${LIBMAMBA_SOURCE_DIR}/util/path_manip.cpp ${LIBMAMBA_SOURCE_DIR}/util/random.cpp ${LIBMAMBA_SOURCE_DIR}/util/string.cpp - ${LIBMAMBA_SOURCE_DIR}/util/url.cpp ${LIBMAMBA_SOURCE_DIR}/util/url_manip.cpp + ${LIBMAMBA_SOURCE_DIR}/util/url.cpp # Implementation of version and matching specs ${LIBMAMBA_SOURCE_DIR}/specs/archive.cpp ${LIBMAMBA_SOURCE_DIR}/specs/authentication_info.cpp @@ -178,8 +178,8 @@ set( ${LIBMAMBA_SOURCE_DIR}/specs/regex_spec.cpp ${LIBMAMBA_SOURCE_DIR}/specs/repo_data.cpp ${LIBMAMBA_SOURCE_DIR}/specs/unresolved_channel.cpp - ${LIBMAMBA_SOURCE_DIR}/specs/version.cpp ${LIBMAMBA_SOURCE_DIR}/specs/version_spec.cpp + ${LIBMAMBA_SOURCE_DIR}/specs/version.cpp # Solver generic interface ${LIBMAMBA_SOURCE_DIR}/solver/helpers.cpp ${LIBMAMBA_SOURCE_DIR}/solver/problems_graph.cpp @@ -192,61 +192,61 @@ set( ${LIBMAMBA_SOURCE_DIR}/solver/libsolv/solver.cpp ${LIBMAMBA_SOURCE_DIR}/solver/libsolv/unsolvable.cpp # Artifacts validation - ${LIBMAMBA_SOURCE_DIR}/validation/tools.cpp ${LIBMAMBA_SOURCE_DIR}/validation/errors.cpp ${LIBMAMBA_SOURCE_DIR}/validation/keys.cpp - ${LIBMAMBA_SOURCE_DIR}/validation/update_framework.cpp + ${LIBMAMBA_SOURCE_DIR}/validation/repo_checker.cpp + ${LIBMAMBA_SOURCE_DIR}/validation/tools.cpp ${LIBMAMBA_SOURCE_DIR}/validation/update_framework_v0_6.cpp ${LIBMAMBA_SOURCE_DIR}/validation/update_framework_v1.cpp - ${LIBMAMBA_SOURCE_DIR}/validation/repo_checker.cpp + ${LIBMAMBA_SOURCE_DIR}/validation/update_framework.cpp # Downloaders and mirrors - ${LIBMAMBA_SOURCE_DIR}/download/compression.hpp ${LIBMAMBA_SOURCE_DIR}/download/compression.cpp - ${LIBMAMBA_SOURCE_DIR}/download/curl.hpp + ${LIBMAMBA_SOURCE_DIR}/download/compression.hpp ${LIBMAMBA_SOURCE_DIR}/download/curl.cpp + ${LIBMAMBA_SOURCE_DIR}/download/curl.hpp ${LIBMAMBA_SOURCE_DIR}/download/downloader_impl.hpp ${LIBMAMBA_SOURCE_DIR}/download/downloader.cpp - ${LIBMAMBA_SOURCE_DIR}/download/mirror.cpp - ${LIBMAMBA_SOURCE_DIR}/download/mirror_map.cpp - ${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.hpp ${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.cpp + ${LIBMAMBA_SOURCE_DIR}/download/mirror_impl.hpp + ${LIBMAMBA_SOURCE_DIR}/download/mirror_map.cpp + ${LIBMAMBA_SOURCE_DIR}/download/mirror.cpp ${LIBMAMBA_SOURCE_DIR}/download/request.cpp # Core API (low-level) - ${LIBMAMBA_SOURCE_DIR}/core/singletons.cpp ${LIBMAMBA_SOURCE_DIR}/core/activation.cpp - ${LIBMAMBA_SOURCE_DIR}/core/package_database_loader.cpp ${LIBMAMBA_SOURCE_DIR}/core/channel_context.cpp ${LIBMAMBA_SOURCE_DIR}/core/context.cpp ${LIBMAMBA_SOURCE_DIR}/core/download_progress_bar.cpp + ${LIBMAMBA_SOURCE_DIR}/core/env_lockfile.cpp ${LIBMAMBA_SOURCE_DIR}/core/environments_manager.cpp ${LIBMAMBA_SOURCE_DIR}/core/error_handling.cpp - ${LIBMAMBA_SOURCE_DIR}/core/transaction_context.cpp - ${LIBMAMBA_SOURCE_DIR}/core/link.cpp + ${LIBMAMBA_SOURCE_DIR}/core/execution.cpp + ${LIBMAMBA_SOURCE_DIR}/core/fsutil.cpp ${LIBMAMBA_SOURCE_DIR}/core/history.cpp + ${LIBMAMBA_SOURCE_DIR}/core/link.cpp ${LIBMAMBA_SOURCE_DIR}/core/menuinst.cpp ${LIBMAMBA_SOURCE_DIR}/core/output.cpp - ${LIBMAMBA_SOURCE_DIR}/core/package_handling.cpp ${LIBMAMBA_SOURCE_DIR}/core/package_cache.cpp - ${LIBMAMBA_SOURCE_DIR}/core/prefix_data.cpp - ${LIBMAMBA_SOURCE_DIR}/core/progress_bar.cpp - ${LIBMAMBA_SOURCE_DIR}/core/progress_bar_impl.cpp - ${LIBMAMBA_SOURCE_DIR}/core/pinning.cpp + ${LIBMAMBA_SOURCE_DIR}/core/package_database_loader.cpp ${LIBMAMBA_SOURCE_DIR}/core/package_fetcher.cpp + ${LIBMAMBA_SOURCE_DIR}/core/package_handling.cpp ${LIBMAMBA_SOURCE_DIR}/core/package_paths.cpp + ${LIBMAMBA_SOURCE_DIR}/core/pinning.cpp + ${LIBMAMBA_SOURCE_DIR}/core/prefix_data.cpp + ${LIBMAMBA_SOURCE_DIR}/core/progress_bar_impl.cpp + ${LIBMAMBA_SOURCE_DIR}/core/progress_bar.cpp ${LIBMAMBA_SOURCE_DIR}/core/query.cpp ${LIBMAMBA_SOURCE_DIR}/core/repo_checker_store.cpp ${LIBMAMBA_SOURCE_DIR}/core/run.cpp ${LIBMAMBA_SOURCE_DIR}/core/shell_init.cpp + ${LIBMAMBA_SOURCE_DIR}/core/singletons.cpp ${LIBMAMBA_SOURCE_DIR}/core/subdirdata.cpp ${LIBMAMBA_SOURCE_DIR}/core/thread_utils.cpp + ${LIBMAMBA_SOURCE_DIR}/core/timeref.cpp + ${LIBMAMBA_SOURCE_DIR}/core/transaction_context.cpp ${LIBMAMBA_SOURCE_DIR}/core/transaction.cpp - ${LIBMAMBA_SOURCE_DIR}/core/util.cpp - ${LIBMAMBA_SOURCE_DIR}/core/fsutil.cpp ${LIBMAMBA_SOURCE_DIR}/core/util_os.cpp + ${LIBMAMBA_SOURCE_DIR}/core/util.cpp ${LIBMAMBA_SOURCE_DIR}/core/virtual_packages.cpp - ${LIBMAMBA_SOURCE_DIR}/core/env_lockfile.cpp - ${LIBMAMBA_SOURCE_DIR}/core/execution.cpp - ${LIBMAMBA_SOURCE_DIR}/core/timeref.cpp # API (high-level) ${LIBMAMBA_SOURCE_DIR}/api/c_api.cpp ${LIBMAMBA_SOURCE_DIR}/api/channel_loader.cpp @@ -301,19 +301,19 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/util/iterator.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/json.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/loop_control.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_linux.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_osx.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_unix.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os_win.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/util/os.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/parsers.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/path_manip.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/random.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/string.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/tuple_hash.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/type_traits.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/util/url.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/url_manip.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/util/url.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/variant_cmp.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/util/weakening_map.hpp # Implementation of version and matching specs @@ -321,16 +321,18 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/authentication_info.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/build_number_spec.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/channel.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/chimera_string_spec.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/conda_url.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/error.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/glob_spec.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/match_spec.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/package_info.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/platform.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/regex_spec.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/repo_data.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/unresolved_channel.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version_spec.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/specs/version.hpp # Solver generic interface ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/problems_graph.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/request.hpp @@ -342,63 +344,64 @@ set( ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/solver.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/solver/libsolv/unsolvable.hpp # Artifacts validation - ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/tools.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/errors.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/keys.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/repo_checker.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/tools.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework_v0_6.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework_v1.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/repo_checker.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/validation/update_framework.hpp # Downloaders and mirrors ${LIBMAMBA_INCLUDE_DIR}/mamba/download/downloader.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror_map.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/download/mirror.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/download/request.hpp # Core API (low-level) ${LIBMAMBA_INCLUDE_DIR}/mamba/core/activation.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/channel_context.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/common_types.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/context.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/download_progress_bar.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/env_lockfile.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/environments_manager.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_database_loader.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/error_handling.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/execution.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/fsutil.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/history.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/invoke.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/link.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/menuinst.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/mirror.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/output.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_cache.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_database_loader.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_fetcher.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_handling.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/package_paths.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/palette.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/pinning.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/prefix_data.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/progress_bar.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/pinning.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/query.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/repo_checker_store.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/run.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/shell_init.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/subdirdata.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/tasksync.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/thread_utils.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/timeref.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction_context.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/fsutil.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/transaction.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_os.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util_scope.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/core/util.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/core/virtual_packages.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/env_lockfile.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/tasksync.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/invoke.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/core/timeref.hpp # API (high-level) ${LIBMAMBA_INCLUDE_DIR}/mamba/api/c_api.h ${LIBMAMBA_INCLUDE_DIR}/mamba/api/channel_loader.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/clean.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/config.hpp - ${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration_impl.hpp + ${LIBMAMBA_INCLUDE_DIR}/mamba/api/configuration.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/constants.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/create.hpp ${LIBMAMBA_INCLUDE_DIR}/mamba/api/info.hpp diff --git a/libmamba/ext/solv-cpp/tests/CMakeLists.txt b/libmamba/ext/solv-cpp/tests/CMakeLists.txt index 6ae1bfc2bb..48d863e52d 100644 --- a/libmamba/ext/solv-cpp/tests/CMakeLists.txt +++ b/libmamba/ext/solv-cpp/tests/CMakeLists.txt @@ -11,6 +11,7 @@ add_executable( src/main.cpp src/msvc_catch_string_view.cpp src/pool_data.cpp + src/pool_data.hpp src/test_pool.cpp src/test_queue.cpp src/test_repo.cpp diff --git a/libmamba/include/mamba/core/mirror.hpp b/libmamba/include/mamba/core/mirror.hpp deleted file mode 100644 index 567d6154bf..0000000000 --- a/libmamba/include/mamba/core/mirror.hpp +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) 2023, QuantStack and Mamba Contributors -// -// Distributed under the terms of the BSD 3-Clause License. -// -// The full license is in the file LICENSE, distributed with this software. - -#ifndef MAMBA_CORE_MIRROR_HPP -#define MAMBA_CORE_MIRROR_HPP - -#include -#include -#include -#include -#include -#include - -namespace mamba -{ - class CURLHandle; - - enum class Protocol - { - FILE, - FTP, - HTTP, - OCI, - }; - - // Statistics of mirror - struct MirrorStats - { - // Maximum number of allowed parallel connections to this mirror. -1 means no - // limit. Dynamically adjusted (decreased) if no fatal (temporary) error will - // occur. - // TODO To be set using `ctx.threads_params.download_threads` - // Do we need it as an arg in Mirror constructor? - // Anyway, it will need to be linked to (or replacing?) `m_max_parallel_downloads` in - // CURLMultiHandle - // Same for the other stats? - long allowed_parallel_connections = -1; - - // The maximum number of tried parallel connections to this mirror - // (including unsuccessful). - std::size_t max_tried_parallel_connections = 0; - - // How many transfers from this mirror are currently in progress. - std::size_t running_transfers = 0; - - // How many transfers was finished successfully from the mirror. - std::size_t successful_transfers = 0; - - // How many transfers failed. - std::size_t failed_transfers = 0; - - // Maximum ranges supported in a single request. This will be automatically - // adjusted when mirrors respond with 200 to a range request - // zchunk case - std::size_t max_ranges = 256; - }; - - class Mirror - { - public: - - Mirror(const std::string& url); - - virtual ~Mirror(); - - Mirror(const Mirror&) = delete; - Mirror& operator=(const Mirror&) = delete; - Mirror(Mirror&&) = delete; - Mirror& operator=(Mirror&&) = delete; - - Protocol protocol() const; - const std::string& url() const; - std::chrono::system_clock::time_point next_retry() const; - const MirrorStats& stats() const; - - void set_max_ranges(std::size_t max_ranges); - void set_allowed_parallel_connections(long allowed_parallel_connections); - - bool need_wait_for_retry() const; - bool has_running_transfers() const; - - void increase_running_transfers(); - bool is_parallel_connections_limited_and_reached() const; - void update_statistics(bool transfer_success); - - virtual std::string format_url(const std::string& path) const; - virtual std::string get_auth_header(const std::string& path) const; - virtual bool needs_preparation(const std::string& path) const; - virtual void prepare(const std::string& path, CURLHandle& handle); - - protected: - - Protocol m_protocol; - - private: - - const std::string m_url; - - // TODO put these in a struct? - // Retry & backoff params - std::chrono::system_clock::time_point m_next_retry; - std::chrono::system_clock::duration m_retry_wait_seconds; - std::size_t m_retry_backoff_factor; - // count number of retries (this is not the same as failed transfers, as multiple - // transfers can be started at the same time, but should all be retried only once) - std::size_t m_retry_counter; - - MirrorStats m_stats; - }; - - class HTTPMirror : public Mirror - { - public: - - HTTPMirror(const std::string& url); - ~HTTPMirror(); - - bool authenticate(CURLHandle& handle, const std::string& user, const std::string& password); - }; - - // Utility function - // TODO leave it here or put it in a namespace, or move it to utils? - std::pair split_path_tag(const std::string& path); - - class OCIMirror : public Mirror - { - public: - - using proxy_map_type = std::map; - - OCIMirror(const std::string& host, const std::string& repo_prefix, const proxy_map_type& proxy_map); - - OCIMirror( - const std::string& host, - const std::string& repo_prefix, - const std::string& scope, - const std::string& username, - const std::string& password, - const proxy_map_type& proxy_map - ); - ~OCIMirror(); - - - std::string get_repo(const std::string& repo) const; - std::string get_auth_url(const std::string& repo, const std::string& scope) const; - std::string get_manifest_url(const std::string& repo, const std::string& reference) const; - std::string get_preupload_url(const std::string& repo) const; - - bool need_auth() const; - - std::string format_url(const std::string& path) const override; - std::string get_auth_header(const std::string& path) const override; - bool needs_preparation(const std::string& path) const override; - void prepare(const std::string& path, CURLHandle& handle) override; - - private: - - struct AuthData - { - std::string sha256sum; // TODO what about other checksums types? i.e md5 - std::string token; - }; - - std::map> m_path_map; - std::string m_repo_prefix; - std::string m_scope; - std::string m_username; - std::string m_password; - - // TODO do we really need this here? (only needs in `prepare` so far, add it as an arg - // there?) - proxy_map_type m_proxy_map; - - AuthData* get_data(const std::string& path) const; - }; - -} // namespace mamba - -#endif // MAMBA_CORE_MIRROR_HPP diff --git a/libmamba/src/core/mirror.cpp b/libmamba/src/core/mirror.cpp deleted file mode 100644 index 3ae56357ce..0000000000 --- a/libmamba/src/core/mirror.cpp +++ /dev/null @@ -1,444 +0,0 @@ -// Copyright (c) 2023, QuantStack and Mamba Contributors -// -// Distributed under the terms of the BSD 3-Clause License. -// -// The full license is in the file LICENSE, distributed with this software. - -#include - -#include "mamba/core/mirror.hpp" -#include "mamba/core/output.hpp" -#include "mamba/util/string.hpp" -#include "mamba/util/url_manip.hpp" - -#include "../download/curl.hpp" - -namespace mamba -{ - - /************************* - * Mirror implementation * - *************************/ - - Mirror::Mirror(const std::string& url) - : m_protocol(Protocol::HTTP) - , m_url(std::string(util::rstrip(url, '/'))) - , m_next_retry(std::chrono::system_clock::now()) - , m_retry_wait_seconds(std::chrono::milliseconds(200)) - , m_retry_backoff_factor(2) - , m_retry_counter(0) - { - } - - Mirror::~Mirror() = default; - - Protocol Mirror::protocol() const - { - return m_protocol; - } - - const std::string& Mirror::url() const - { - return m_url; - } - - std::chrono::system_clock::time_point Mirror::next_retry() const - { - return m_next_retry; - } - - const MirrorStats& Mirror::stats() const - { - return m_stats; - } - - void Mirror::set_max_ranges(std::size_t max_ranges) - { - m_stats.max_ranges = max_ranges; - } - - void Mirror::set_allowed_parallel_connections(long allowed_parallel_connections) - { - m_stats.allowed_parallel_connections = allowed_parallel_connections; - } - - bool Mirror::need_wait_for_retry() const - { - return m_retry_counter != 0 && m_next_retry > std::chrono::system_clock::now(); - } - - bool Mirror::has_running_transfers() const - { - return m_stats.running_transfers > 0; - } - - void Mirror::increase_running_transfers() - { - m_stats.running_transfers++; - if (m_stats.max_tried_parallel_connections < m_stats.running_transfers) - { - m_stats.max_tried_parallel_connections = m_stats.running_transfers; - } - } - - bool Mirror::is_parallel_connections_limited_and_reached() const - { - return m_stats.allowed_parallel_connections != -1 - && m_stats.running_transfers >= std::size_t(m_stats.allowed_parallel_connections); - } - - void Mirror::update_statistics(bool transfer_success) - { - m_stats.running_transfers--; - if (transfer_success) - { - m_stats.successful_transfers++; - } - else - { - m_stats.failed_transfers++; - if (m_stats.failed_transfers == 1 || m_next_retry < std::chrono::system_clock::now()) - { - m_retry_counter++; - m_retry_wait_seconds = m_retry_wait_seconds * m_retry_backoff_factor; - m_next_retry = std::chrono::system_clock::now() + m_retry_wait_seconds; - } - } - } - - std::string Mirror::format_url(const std::string& path) const - { - return util::url_concat(m_url, path); - } - - std::string Mirror::get_auth_header(const std::string&) const - { - return {}; - } - - bool Mirror::needs_preparation(const std::string&) const - { - return false; - } - - void Mirror::prepare(const std::string&, CURLHandle&) - { - } - - /***************************** - * HTTPMirror implementation * - *****************************/ - - HTTPMirror::HTTPMirror(const std::string& url) - : Mirror(url) - { - } - - HTTPMirror::~HTTPMirror() = default; - - // TODO Maybe move this curl stuff somewhere else? - // Or remove it if authentication won't be used - bool - HTTPMirror::authenticate(CURLHandle& handle, const std::string& user, const std::string& password) - { - if (!password.empty() && !user.empty()) - { - handle.set_opt(CURLOPT_USERNAME, user); - handle.set_opt(CURLOPT_PASSWORD, password); - return true; - } - else - { - LOG_WARNING << "Cannot authenticate: user or password empty"; - return false; - } - } - - // Utility function - std::pair split_path_tag(const std::string& path) - { - // for OCI, if we have a filename like "xtensor-0.23.10-h2acdbc0_0.tar.bz2" - // we want to split it to `xtensor:0.23.10-h2acdbc0-0` - if (util::ends_with(path, ".json")) - { - return { path, "latest" }; - } - - std::pair result; - auto parts = util::rsplit(path, "-", 2); - - if (parts.size() < 2) - { - LOG_ERROR << "Could not split filename into enough parts"; - throw std::runtime_error("Could not split filename into enough parts"); - } - - result.first = parts[0]; - - std::string tag; - if (parts.size() > 2) - { - std::string last_part = parts[2].substr(0, parts[2].find_first_of(".")); - tag = fmt::format("{}-{}", parts[1], last_part); - } - else - { - tag = parts[1]; - } - - util::replace_all(tag, "_", "-"); - result.second = tag; - - LOG_INFO << "Splitting " << path << " to name: " << result.first << " tag: " << result.second; - return result; - } - - /***************************** - * OCIMirror implementation * - *****************************/ - - // OCI Mirror: - // When knowing the SHA256 we can directly get to the blob - // When we do not know the SHA256 sum, we need to find the `latest` or some - // other blob - - // OCI upload process - // 4 steps: - // - first get auth token with push rights - // - then - - // This is what an OCI manifest (index) looks like: - // { - // "schemaVersion": 2, - // "config": { - // "mediaType": "application/vnd.unknown.config.v1+json", - // "digest": - // "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - // "size": 0 - // }, - // "layers": [ - // { - // "mediaType": "application/vnd.unknown.layer.v1+txt", - // "digest": - // "sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4", - // "size": 13, - // "annotations": { - // "org.opencontainers.image.title": "artifact.txt" - // } - // } - // ] - // } - - // Used for pull/read (doesn't need auth) - OCIMirror::OCIMirror( - const std::string& host, - const std::string& repo_prefix, - const proxy_map_type& proxy_map - ) - : Mirror(host) - , m_repo_prefix(repo_prefix) - , m_scope("pull") - , m_proxy_map(proxy_map) - { - m_protocol = Protocol::OCI; - } - - OCIMirror::OCIMirror( - const std::string& host, - const std::string& repo_prefix, - const std::string& scope, - const std::string& username, - const std::string& password, - const proxy_map_type& proxy_map - ) - : Mirror(host) - , m_repo_prefix(repo_prefix) - , m_scope(scope) - , m_username(username) - , m_password(password) - , m_proxy_map(proxy_map) - { - m_protocol = Protocol::OCI; - } - - OCIMirror::~OCIMirror() = default; - - std::string OCIMirror::get_repo(const std::string& repo) const - { - if (!m_repo_prefix.empty()) - { - return fmt::format("{}/{}", m_repo_prefix, repo); - } - else - { - return repo; - } - } - - std::string OCIMirror::get_auth_url(const std::string& repo, const std::string& scope) const - { - return fmt::format("{}/token?scope=repository:{}:{}", this->url(), get_repo(repo), scope); - } - - std::string OCIMirror::get_manifest_url(const std::string& repo, const std::string& reference) const - { - return fmt::format("{}/v2/{}/manifests/{}", this->url(), get_repo(repo), reference); - } - - std::string OCIMirror::get_preupload_url(const std::string& repo) const - { - return fmt::format("{}/v2/{}/blobs/uploads/", this->url(), get_repo(repo)); - } - - bool OCIMirror::need_auth() const - { - return !m_username.empty() && !m_password.empty(); - } - - std::string OCIMirror::format_url(const std::string& path) const - { - // TODO Maybe we would need a std::string checksum to pass as arg (to be empty in Mirror) - // (in case the map data is not set yet) - // to be checked in use/call of this method - - auto* data = get_data(path); - if (data) - { - auto [split_path, split_tag] = split_path_tag(path); - // Should be this format: - // https://ghcr.io/v2/wolfv/artifact/blobs/sha256:c5be3ea75353851e1fcf3a298af3b6cfd2af3d7ff018ce52657b6dbd8f986aa4 - return fmt::format( - "{}/v2/{}/blobs/sha256:{}", - this->url(), - get_repo(split_path), - data->sha256sum - ); - } - else - { - LOG_ERROR << "Checksum corresponding to " << path << " is not available"; - return {}; - } - } - - std::string OCIMirror::get_auth_header(const std::string& path) const - { - if (m_username.empty() && m_password.empty()) - { - return {}; - } - - auto* data = get_data(path); - if (data && !data->token.empty()) - { - return fmt::format("Authorization: Bearer {}", data->token); - } - else - { - return {}; - } - } - - bool OCIMirror::needs_preparation(const std::string& path) const - { - auto* data = get_data(path); - if ((!data || (data && data->token.empty())) && need_auth()) - { - return true; - } - - if (!data || (data && data->sha256sum.empty())) - { - return true; - } - - return false; - } - - void OCIMirror::prepare(const std::string& path, CURLHandle& handle) - { - auto [split_path, split_tag] = split_path_tag(path); - - auto* data = get_data(path); - if (!data) - { - m_path_map[split_path].reset(new AuthData); - data = m_path_map[split_path].get(); - } - - if (data && data->token.empty() && need_auth()) - { - std::string auth_url = get_auth_url(split_path, m_scope); - - // TODO to be checked - // Not sure this is necessary - // Do we need to call curl config beforehand using the right args in - // `configure_curl_handle` - handle.set_url(auth_url, m_proxy_map); - // IMPORTANT: Make sure the callbacks are set before calling this - // i.e CURLOPT_HEADERFUNCTION, CURLOPT_HEADERDATA, CURLOPT_WRITEFUNCTION, - // CURLOPT_WRITEDATA - - if (!m_username.empty()) - { - handle.set_opt(CURLOPT_USERNAME, m_username); - } - - if (!m_password.empty()) - { - handle.set_opt(CURLOPT_PASSWORD, m_password); - } - - auto set_data_callback = - [&data](bool success, const std::string& token, const std::string&) // Args got as - // a response - // to transfer - { - if (success && !token.empty()) - { - data->token = token; - } - }; - - // TODO set here a callback to `set_data_callback` in CURLHandle or other relevant - // class which performs the transfer finalization and sets data token - // handle.set_end_callback(set_data_callback); - } - else - { - std::string manifest_url = get_manifest_url(split_path, split_tag); - - handle.set_url(manifest_url, m_proxy_map) - .add_header(get_auth_header(path)) - .add_header("Accept: application/vnd.oci.image.manifest.v1+json"); - - auto set_data_callback = - [&data](bool success, const std::string&, const std::string& digest) - { - // digest is got from a json response like - // j["layers"][0]["digest"] - if (success && !digest.empty()) - { - assert(util::starts_with(digest, "sha256:")); - data->sha256sum = digest.substr(sizeof("sha256:") - 1); - } - }; - - // TODO set here a callback to `set_data_callback` in CURLHandle or other - // relevant class which performs the transfer finalization and sets data sha256sum - // handle.set_end_callback(set_data_callback); - } - } - - OCIMirror::AuthData* OCIMirror::get_data(const std::string& path) const - { - auto [split_path, _] = split_path_tag(path); - auto it = m_path_map.find(split_path); - if (it != m_path_map.end()) - { - return it->second.get(); - } - return nullptr; - } - -} // namespace mamba diff --git a/libmamba/tests/CMakeLists.txt b/libmamba/tests/CMakeLists.txt index a1069d4ea3..c506048b2a 100644 --- a/libmamba/tests/CMakeLists.txt +++ b/libmamba/tests/CMakeLists.txt @@ -54,12 +54,12 @@ set( src/specs/test_regex_spec.cpp src/specs/test_repo_data.cpp src/specs/test_unresolved_channel.cpp - src/specs/test_version.cpp src/specs/test_version_spec.cpp + src/specs/test_version.cpp # Solver tests + src/solver/test_problems_graph.cpp src/solver/test_request.cpp src/solver/test_solution.cpp - src/solver/test_problems_graph.cpp # Solver libsolv implementation tests src/solver/libsolv/test_database.cpp src/solver/libsolv/test_solver.cpp @@ -77,21 +77,21 @@ set( src/core/test_configuration.cpp src/core/test_cpp.cpp src/core/test_env_file_reading.cpp + src/core/test_env_lockfile.cpp src/core/test_environments_manager.cpp + src/core/test_execution.cpp + src/core/test_filesystem.cpp src/core/test_history.cpp + src/core/test_invoke.cpp src/core/test_lockfile.cpp - src/core/test_pinning.cpp src/core/test_output.cpp + src/core/test_pinning.cpp src/core/test_progress_bar.cpp src/core/test_shell_init.cpp + src/core/test_tasksync.cpp src/core/test_thread_utils.cpp - src/core/test_virtual_packages.cpp src/core/test_util.cpp - src/core/test_env_lockfile.cpp - src/core/test_execution.cpp - src/core/test_invoke.cpp - src/core/test_tasksync.cpp - src/core/test_filesystem.cpp + src/core/test_virtual_packages.cpp ) message(STATUS "Building libmamba C++ tests") diff --git a/micromamba/CMakeLists.txt b/micromamba/CMakeLists.txt index 7414429c4f..6431494c19 100644 --- a/micromamba/CMakeLists.txt +++ b/micromamba/CMakeLists.txt @@ -17,8 +17,9 @@ set( longpath.manifest ${CMAKE_CURRENT_SOURCE_DIR}/src/activate.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/clean.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/config.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/completer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/config.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/constructor.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/create.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/env.cpp @@ -26,7 +27,6 @@ set( ${CMAKE_CURRENT_SOURCE_DIR}/src/install.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/list.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/login.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/main.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/package.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/remove.cpp @@ -41,6 +41,7 @@ set( set( MICROMAMBA_HEADERS ${CMAKE_CURRENT_SOURCE_DIR}/src/common_options.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/constructor.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/umamba.hpp ${CMAKE_CURRENT_SOURCE_DIR}/src/version.hpp )