From 0df4d53020fff64d23db93c0471666ca47fe53bb Mon Sep 17 00:00:00 2001 From: Tyler <49161327+tylerpotts@users.noreply.github.com> Date: Thu, 18 Nov 2021 11:14:58 -0600 Subject: [PATCH] Ruamel dependency (#31) * Fix bug that prevented file writing, bump version * Fix bug where pip wasn't properly being added. Simplify env_deps in manifest class * Add function for writing yaml, add test for same * Bump version * Change dependency from pyyaml to ruamel.yaml --- conda_vendor/__main__.py | 4 +++- conda_vendor/conda_channel.py | 6 ++++-- conda_vendor/custom_manifest.py | 11 ++++++++--- conda_vendor/env_yaml_from_manifest.py | 13 +++++++++---- conda_vendor/manifest.py | 19 ++++++++++++++----- conda_vendor/version.py | 2 +- conda_vendor_container_env.yml | 1 + setup.py | 2 +- tests/conftest.py | 7 +++++-- tests/test_conda_channel.py | 9 ++++----- tests/test_custom_manifest.py | 6 ++++-- tests/test_env_yaml_from_manifest.py | 4 ++-- tests/test_integration.py | 6 ++++-- tests/test_manifest.py | 17 ++++++++++------- 14 files changed, 70 insertions(+), 37 deletions(-) diff --git a/conda_vendor/__main__.py b/conda_vendor/__main__.py index 1b933c0..7a136cf 100644 --- a/conda_vendor/__main__.py +++ b/conda_vendor/__main__.py @@ -9,7 +9,9 @@ ) from conda_vendor.version import __version__ from conda_vendor.manifest import combine_metamanifests, write_combined_manifest -import yaml + +from ruamel.yaml import YAML + import logging logger = logging.getLogger(__name__) diff --git a/conda_vendor/conda_channel.py b/conda_vendor/conda_channel.py index d69ec26..0fe86d6 100644 --- a/conda_vendor/conda_channel.py +++ b/conda_vendor/conda_channel.py @@ -7,7 +7,7 @@ from pathlib import Path import requests -import yaml +from ruamel.yaml import YAML from conda_lock.conda_lock import solve_specs_for_arch from conda_lock.src_parser.environment_yaml import parse_environment_file from requests.adapters import HTTPAdapter @@ -37,7 +37,9 @@ def __init__(self, *, channel_root=Path(), meta_manifest_path=None): def load_manifest(self, manifest_path): with open(manifest_path) as f: - return yaml.load(f, Loader=yaml.SafeLoader) + return YAML(typ="safe").load( + f, + ) def fetch_and_filter_repodata(self, conda_subdir, manifest_subset_metadata): # returns a channel specific repodata.json diff --git a/conda_vendor/custom_manifest.py b/conda_vendor/custom_manifest.py index 0fe5093..996e520 100644 --- a/conda_vendor/custom_manifest.py +++ b/conda_vendor/custom_manifest.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from pathlib import Path -import yaml +from ruamel.yaml import YAML logger = logging.getLogger(__name__) @@ -18,7 +18,9 @@ def __init__(self, manifest_path=Path()): def read_meta_manifest(self, manifest_path): with open(self.manifest_path) as f: - return yaml.load(f, Loader=yaml.SafeLoader) + return YAML(typ="safe").load( + f, + ) @abstractmethod def write_custom_manifest(self, output_path=None): @@ -38,7 +40,10 @@ def write_custom_manifest(self, output_file_path=None): logger.info(f"Output Custom Manifest : {str(output_file_path.absolute())}") with open(output_file_path, "w") as f: - yaml.dump(self.custom_manifest, f, sort_keys=False) + YAML().dump( + self.custom_manifest, + f, + ) @staticmethod def strip_lead_underscore(in_str): diff --git a/conda_vendor/env_yaml_from_manifest.py b/conda_vendor/env_yaml_from_manifest.py index e8ce328..6e1fac6 100644 --- a/conda_vendor/env_yaml_from_manifest.py +++ b/conda_vendor/env_yaml_from_manifest.py @@ -1,6 +1,6 @@ from pathlib import Path - -import yaml +from ruamel.yaml import YAML +from ruamel.yaml import YAML class YamlFromManifest: @@ -10,7 +10,9 @@ def __init__(self, channel_root, meta_manifest_path=Path()): def load_manifest(self, meta_manifest_path): with open(meta_manifest_path) as f: - return yaml.load(f, Loader=yaml.SafeLoader) + return YAML(typ="safe").load( + f, + ) def get_packages_from_manifest(self): i_bank_pkg_list = [] @@ -40,5 +42,8 @@ def create_yaml(self, channel_root, env_name): } fn = channel_root / f"local_{env_name}.yaml" with open(fn, "w") as f: - yaml.dump(env_yaml, f, sort_keys=False) + YAML().dump( + env_yaml, + f, + ) return env_yaml diff --git a/conda_vendor/manifest.py b/conda_vendor/manifest.py index 427e56f..d0f2a3b 100644 --- a/conda_vendor/manifest.py +++ b/conda_vendor/manifest.py @@ -6,9 +6,8 @@ import struct import sys from pathlib import Path - +from ruamel.yaml import YAML import requests -import yaml from conda_lock.conda_lock import solve_specs_for_arch from conda_lock.src_parser.environment_yaml import parse_environment_file from requests.adapters import HTTPAdapter @@ -118,7 +117,10 @@ def create_manifest(self, *, manifest_filename=None): outpath_file_name = self.manifest_root / cleaned_name logger.info(f"Creating Manifest {outpath_file_name.absolute()}") with open(outpath_file_name, "w") as f: - yaml.dump(manifest, f, sort_keys=False) + YAML().dump( + manifest, + f, + ) return manifest def get_manifest(self): @@ -214,10 +216,17 @@ def read_manifests(manifest_paths): manifest_list = [] for manifest_path in manifest_paths: with open(manifest_path, "r") as f: - manifest_list.append(yaml.safe_load(f)) + + manifest_list.append(YAML(typ="safe").load(f)) + return manifest_list def write_combined_manifest(manifest_path, combined_manifest): with open(manifest_path, "w") as f: - yaml.dump(combined_manifest, f, sort_keys=False) + + YAML().dump( + combined_manifest, + f, + ) + diff --git a/conda_vendor/version.py b/conda_vendor/version.py index d496681..158428b 100644 --- a/conda_vendor/version.py +++ b/conda_vendor/version.py @@ -1,4 +1,4 @@ -__version__ = "0.1.14" +__version__ = "0.1.15" if __name__ == "__main__": diff --git a/conda_vendor_container_env.yml b/conda_vendor_container_env.yml index f2752d3..1cb2712 100644 --- a/conda_vendor_container_env.yml +++ b/conda_vendor_container_env.yml @@ -4,3 +4,4 @@ channels: dependencies: - python=3.9.5 - conda-lock + - ruamel.yaml diff --git a/setup.py b/setup.py index e56f681..78ee529 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ packages=find_packages(exclude=("tests",), where="."), url="https://github.com/MetroStar/conda-vendor", entry_points={"console_scripts": ["conda-vendor = conda_vendor.__main__:cli"]}, - install_requires=["pyyaml", "conda-lock"], + install_requires=["ruamel.yaml", "conda-lock"], setup_requires=["wheel"], python_requires=">=3.6", long_description=long_description, diff --git a/tests/conftest.py b/tests/conftest.py index f83ace0..24a539a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,7 +3,7 @@ from unittest.mock import Mock import pytest -import yaml +from ruamel.yaml import YAML from conda_vendor.conda_channel import CondaChannel from conda_vendor.manifest import MetaManifest @@ -60,7 +60,10 @@ def get_path_location_for_manifest_fixture(tmpdir_factory): fn = tmpdir_factory.mktemp("minimal_env").join("env.yml") with open(fn, "w") as f: - yaml.dump(fixture_manifest, f, sort_keys=False) + YAML().dump( + fixture_manifest, + f, + ) return fn diff --git a/tests/test_conda_channel.py b/tests/test_conda_channel.py index 9334964..1fbab14 100644 --- a/tests/test_conda_channel.py +++ b/tests/test_conda_channel.py @@ -4,25 +4,24 @@ from unittest.mock import Mock, call, mock_open, patch import pytest -import yaml +from ruamel.yaml import YAML from requests import Response from requests.adapters import urldefragauth from yaml import safe_load -from yaml.loader import SafeLoader - +from ruamel.yaml import YAML from conda_vendor.conda_channel import CondaChannel, improved_download from conda_vendor.manifest import LockWrapper, MetaManifest from .conftest import mock_response -@patch("yaml.load") +@patch("ruamel.yaml.YAML") def test_load_manifest(mock, conda_channel_fixture, tmp_path): test_manifest_path = tmp_path / "test_manifest.yml" with open(test_manifest_path, "w") as y: y.write("test") conda_channel_fixture.load_manifest(test_manifest_path) - mock.assert_called_once_with = [test_manifest_path, yaml.SafeLoader] + mock.assert_called_once_with = [test_manifest_path] @patch("requests.Session.get") diff --git a/tests/test_custom_manifest.py b/tests/test_custom_manifest.py index a7a8ba1..45087b5 100644 --- a/tests/test_custom_manifest.py +++ b/tests/test_custom_manifest.py @@ -1,7 +1,7 @@ from unittest import TestCase from unittest.mock import Mock, call, mock_open, patch -import yaml +from ruamel.yaml import YAML from conda_vendor.custom_manifest import CustomManifest, IBManifest @@ -40,7 +40,9 @@ def test_write_custom_manifest(mock_read_meta_manifest, tmp_path): custom_channel.write_custom_manifest(test_output_path) with open(expected_custom_manifest_destination, "r") as f: - actual_custom_manifest = yaml.load(f, Loader=yaml.SafeLoader) + actual_custom_manifest = YAML(typ="safe").load( + f, + ) assert actual_custom_manifest == expected_custom_manifest diff --git a/tests/test_env_yaml_from_manifest.py b/tests/test_env_yaml_from_manifest.py index 8566de7..8b54be2 100644 --- a/tests/test_env_yaml_from_manifest.py +++ b/tests/test_env_yaml_from_manifest.py @@ -3,7 +3,7 @@ from unittest.mock import patch import pytest -import yaml +from ruamel.yaml import YAML from yaml import SafeLoader from conda_vendor.env_yaml_from_manifest import YamlFromManifest @@ -20,7 +20,7 @@ def yml_man_fixture(tmp_path, get_path_location_for_manifest_fixture): def test_YamlFromManifest_init_(yml_man_fixture): test_manifest_path = yml_man_fixture.meta_manifest_path with open(test_manifest_path, "r") as f: - expected = yaml.load(f, Loader=SafeLoader) + expected = YAML(typ="safe").load(f) result = yml_man_fixture.meta_manifest TestCase().assertDictEqual(expected, result) diff --git a/tests/test_integration.py b/tests/test_integration.py index f1d0904..151b1cc 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -3,7 +3,7 @@ from unittest import TestCase from unittest.mock import Mock, patch -import yaml +from ruamel.yaml import YAML from conda_vendor.cli import ( ironbank_from_meta_manifest, @@ -39,7 +39,9 @@ def test_get_packages_from_manifest(meta_manifest, expected_packages): minimal_conda_forge_environment, tmp_path, test_manifest_filename ) with open(expected_manifest_path) as f: - actual_manifest = yaml.load(f, Loader=yaml.SafeLoader) + actual_manifest = YAML(typ="safe").load( + f, + ) assert "main" in actual_manifest.keys() assert "conda-forge" in actual_manifest.keys() diff --git a/tests/test_manifest.py b/tests/test_manifest.py index 14981a8..5d066c4 100644 --- a/tests/test_manifest.py +++ b/tests/test_manifest.py @@ -12,7 +12,7 @@ from requests import Response import hashlib import json -import yaml +from ruamel.yaml import YAML from requests import Response from unittest import TestCase from unittest.mock import Mock, patch, call, mock_open @@ -227,7 +227,7 @@ def test_create_manifest(meta_manifest_fixture, tmp_path): test_manifest_fixture.create_manifest() with open(expected_path, "r") as f: - actual_manifest = yaml.load(f, Loader=SafeLoader) + actual_manifest = YAML(typ="safe").load(f) TestCase().assertDictEqual(actual_manifest, expected_manifest) @@ -342,10 +342,11 @@ def test_combine_metamanifests(tmp_path): test_manifests_list = [manifest_path_1, manifest_path_2] with open(manifest_path_1, "w") as f: - yaml.dump(test_manifest1, f) + YAML().dump(test_manifest1, f) with open(manifest_path_2, "w") as f: - yaml.dump(test_manifest2, f) + YAML().dump(test_manifest2, f) + actual_return = combine_metamanifests(test_manifests_list) assert actual_return == expected_return @@ -359,10 +360,10 @@ def test_read_manifests(tmp_path): yaml_path2 = tmp_path / "yaml2.yaml" with open(yaml_path1, "w") as f: - yaml.dump(yaml1, f) + YAML().dump(yaml1, f) with open(yaml_path2, "w") as f: - yaml.dump(yaml2, f) + YAML().dump(yaml2, f) expected_manifest_list = [yaml1, yaml2] actual_manifest_list = read_manifests([yaml_path1, yaml_path2]) @@ -377,6 +378,8 @@ def test_write_combined_manifest(tmp_path): write_combined_manifest(yaml_path, test_yaml) with open(yaml_path, "r") as f: - actual_yaml = yaml.safe_load(f) + + actual_yaml = YAML(typ="safe").load(f) + TestCase().assertDictEqual(actual_yaml, test_yaml)