From bffc1b6c6e78279842bd9dc558ed69e3a6c801ec Mon Sep 17 00:00:00 2001 From: Jonathan Gangi Date: Fri, 22 Nov 2024 13:38:30 -0300 Subject: [PATCH] Add wrapper for pubtools-marketplacesvm This commit introduces a wrapper for `pubtools-marketplacesvm` which aims to be eventually used to support VM images pushes on various cloudmarketplaces (such as AWS and Azure). With this wrapper it will be possible to easily integrate the tooling into Tekton CI using the Cloud Staged structure as input. Refers to SPSTRAT-464 --- .../marketplacesvm_push_wrapper | 1 + .../marketplacesvm_push_wrapper.py | 149 ++++++++++++++++++ .../test_marketplacesvm_push_wrapper.py | 118 ++++++++++++++ requirements.txt | 1 + 4 files changed, 269 insertions(+) create mode 120000 pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper create mode 100755 pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py create mode 100644 pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper new file mode 120000 index 0000000..2903306 --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper @@ -0,0 +1 @@ +marketplacesvm_push_wrapper.py \ No newline at end of file diff --git a/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py new file mode 100755 index 0000000..d733d8e --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/marketplacesvm_push_wrapper.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Python script to push staged content to various cloud marketplaces. + +This is a simple wrapper pubtools-marketplacesvm-push command that is able to push and publish +content to various cloud marketplaces. This wrapper supports pushing cloud images only from +staged source using the cloud schema. + +For more information please refer to documentation: +* https://github.com/release-engineering/pubtools-marketplacesvm +* https://release-engineering.github.io/pushsource/sources/staged.html#root-destination-cloud-images +* https://release-engineering.github.io/pushsource/schema/cloud.html#cloud-schema +""" +import argparse +import logging +import os +import re +import subprocess +import sys + +LOG = logging.getLogger("pubtools-marketplacesvm-wrapper") +DEFAULT_LOG_FMT = "%(asctime)s [%(levelname)-8s] %(message)s" +DEFAULT_DATE_FMT = "%Y-%m-%d %H:%M:%S %z" +COMMAND = "pubtools-marketplacesvm-marketplace-push" +CLOUD_MKTS_ENV_VARS_STRICT = ("CLOUD_CREDENTIALS",) + + +def parse_args(): + parser = argparse.ArgumentParser( + prog="marketplacesvm_push_wrapper", + description="Push staged cloud images to various cloud marketplaces.", + ) + + parser.add_argument("--dry-run", action="store_true", help="Log command to be executed") + parser.add_argument( + "--debug", + "-d", + action="count", + default=0, + help=("Show debug logs; can be provided up to three times to enable more logs"), + ) + + parser.add_argument( + "--source", + action="append", + help="Path(s) to staging directory", + required=True, + ) + + parser.add_argument( + "--nochannel", + action="store_true", + help=( + "Do as much as possible without making content available to end-users, then stop." + "May be used to improve the performance of a subsequent full push." + ), + ) + + starmap = parser.add_argument_group("Content mapping settings") + starmap.add_argument( + "--starmap-file", + help="YAML file containing the content mappings on StArMap APIv2 format.", + required=True, + ) + + return parser.parse_args() + + +def get_source_url(stagedirs): + for item in stagedirs: + if not re.match(r"^/[^,]{1,4000}/starmap/\w+\/*$", item): + raise ValueError("Not a valid staging directory: %s" % item) + + return f"staged:{','.join(stagedirs)}" + + +def settings_to_args(parsed): + settings_to_arg_map = { + 'starmap_file': '--repo-file', + "source": "", + } + out = [] + if parsed.nochannel: + out.append("--pre-push") + + for setting, arg in settings_to_arg_map.items(): + if value := getattr(parsed, setting): + if arg: + out.extend([arg]) + out.extend([value]) + + for _ in range(parsed.debug): + out.append("--debug") + + return out + + +def validate_env_vars(args): + assert all([os.getenv(item) for item in CLOUD_MKTS_ENV_VARS_STRICT]), ( + f"Provide all required CLOUD_MKTS environment variables: " + "{', '.join(CLOUD_MKTS_ENV_VARS_STRICT)}" + ) + + args.source = get_source_url(args.source) + return args + + +def main(): + args = validate_env_vars(parse_args()) + + loglevel = logging.DEBUG if args.debug else logging.INFO + + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setLevel(loglevel) + + logging.basicConfig( + level=loglevel, + format=DEFAULT_LOG_FMT, + datefmt=DEFAULT_DATE_FMT, + handlers=[stream_handler], + ) + + if args.dry_run: + LOG.info("This is a dry-run!") + + cmd_args = settings_to_args(args) + command = [COMMAND] + cmd_args + cmd_str = " ".join(command) + + if args.dry_run: + LOG.info("Would have run: %s", cmd_str) + else: + try: + LOG.info("Running %s", cmd_str) + subprocess.run(command, check=True) + except subprocess.CalledProcessError: + LOG.exception("Command %s failed, check exception for details", cmd_str) + raise + except Exception as exc: + LOG.exception("Unknown error occurred") + raise RuntimeError from exc + + +def entrypoint(): + main() + + +if __name__ == "__main__": + entrypoint() diff --git a/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py new file mode 100644 index 0000000..009efce --- /dev/null +++ b/pubtools-marketplacesvm-wrapper/test_marketplacesvm_push_wrapper.py @@ -0,0 +1,118 @@ +import logging +import os +import sys +from unittest.mock import patch + +import pytest + +import marketplacesvm_push_wrapper + + +@pytest.fixture() +def mock_mkt_env_vars(): + with patch.dict( + os.environ, {k: "test" for k in marketplacesvm_push_wrapper.CLOUD_MKTS_ENV_VARS_STRICT} + ): + yield + + +def test_no_args(capsys): + with pytest.raises(SystemExit): + marketplacesvm_push_wrapper.main() + + _, err = capsys.readouterr() + assert ( + "marketplacesvm_push_wrapper: error: the following arguments are required: --source, --starmap-file" + in err + ) + + +def test_dry_run(caplog, mock_mkt_env_vars): + args = [ + "", + "--dry-run", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" in caplog.messages + assert ( + "Would have run: pubtools-marketplacesvm-marketplace-push " + "--repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + +@patch("subprocess.run") +def test_basic_command(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--repo-file", + "mapping.yaml", + "staged:/test/starmap/1,/test/starmap/2", + ], + check=True, + ) + + +@patch("subprocess.run") +def test_basic_command_nochannel(mock_run, caplog, mock_mkt_env_vars): + args = [ + "", + "--nochannel", + "--source", + "/test/starmap/1", + "--source", + "/test/starmap/2", + "--starmap-file", + "mapping.yaml", + ] + + with patch.object(sys, "argv", args): + with caplog.at_level(logging.INFO): + marketplacesvm_push_wrapper.main() + assert "This is a dry-run!" not in caplog.messages + assert ( + "Running pubtools-marketplacesvm-marketplace-push " + "--pre-push --repo-file mapping.yaml " + "staged:/test/starmap/1,/test/starmap/2" + ) in caplog.messages + + mock_run.assert_called_once_with( + [ + "pubtools-marketplacesvm-marketplace-push", + "--pre-push", + "--repo-file", + "mapping.yaml", + "staged:/test/starmap/1,/test/starmap/2", + ], + check=True, + ) diff --git a/requirements.txt b/requirements.txt index 4df9895..e6394b9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ packageurl-python pubtools-pulp pubtools-exodus pubtools-content-gateway +pubtools-marketplacesvm