From 16e69314f78853a1ef13d8898620ac365eb387e1 Mon Sep 17 00:00:00 2001 From: Martin Malina Date: Thu, 12 Dec 2024 12:55:25 +0100 Subject: [PATCH] feat: implement updating of image tags Before this change there was no way to rerun a release pipeline with the intention of changing some tags - the create_container_image would detect that the image already exists in Pyxis and has the required repository defined and it would stop right there. Now this scenario is supported - if the image already exists and it includes the required repository, we check if the tags are the same. If not, we update them to the desired list of tags. Signed-off-by: Martin Malina --- pyxis/create_container_image.py | 106 +++++++++-------- pyxis/test_create_container_image.py | 163 ++++++++++++--------------- 2 files changed, 135 insertions(+), 134 deletions(-) diff --git a/pyxis/create_container_image.py b/pyxis/create_container_image.py index cf1bb9c..60a1d12 100755 --- a/pyxis/create_container_image.py +++ b/pyxis/create_container_image.py @@ -40,7 +40,7 @@ from datetime import datetime import json import logging -from typing import Any, Dict +from typing import Any, Dict, List, Optional from urllib.parse import urljoin import pyxis @@ -177,16 +177,16 @@ def find_image(pyxis_url, digest: str) -> Any: return query_results[0] -def repo_in_image(repository_str: str, image: Dict[str, Any]) -> bool: +def find_repo_in_image(repository_str: str, image: Dict[str, Any]) -> Optional[int]: """Check if a repository already exists in the ContainerImage - :return: True if repository_str string is found in the ContainerImage repositories, - False otherwise + :return: index of repository if repository_str string is found in the ContainerImage repositories, + None otherwise """ - for repository in image["repositories"]: + for index, repository in enumerate(image["repositories"]): if repository["repository"] == repository_str: - return True - return False + return index + return None def prepare_parsed_data(args) -> Dict[str, Any]: @@ -236,11 +236,8 @@ def prepare_parsed_data(args) -> Dict[str, Any]: return parsed_data -def pyxis_tags(args, date_now): +def pyxis_tags(tags, date_now): """Return list of tags formatted for pyxis""" - tags = args.tags.split() - if args.is_latest == "true": - tags.append("latest") return [ { "added_date": date_now, @@ -258,12 +255,12 @@ def repository_digest_values(args): return result -def create_container_image(args, parsed_data: Dict[str, Any]): +def create_container_image(args, parsed_data: Dict[str, Any], tags: List[str]): """Function to create a new containerImage entry in a pyxis instance""" LOGGER.info("Creating new container image") - repository = construct_repository(args) + repository = construct_repository(args, tags) # sum_layer_size_bytes isn't accepted in the parsed_data payload to pyxis sum_layer_size_bytes = parsed_data.pop("sum_layer_size_bytes", 0) @@ -300,18 +297,14 @@ def create_container_image(args, parsed_data: Dict[str, Any]): raise Exception("Image metadata was not successfully added to Pyxis.") -def add_container_image_repository(args: Dict[str, Any], image: Dict[str, Any]): - if not args.rh_push == "true": - LOGGER.info("--rh-push is not set. Skipping public registry association.") - return - - identifier = image["_id"] - LOGGER.info(f"Adding repository to container image {identifier}") +def update_container_image_repositories( + pyxis_url, image_id: str, repositories: List[Dict[str, Any]] +): + LOGGER.info(f"Updating repositories for container image {image_id}") - patch_url = urljoin(args.pyxis_url, f"v1/images/id/{identifier}") + patch_url = urljoin(pyxis_url, f"v1/images/id/{image_id}") - payload = {"repositories": image["repositories"]} - payload["repositories"].append(construct_repository(args)) + payload = {"repositories": repositories} rsp = pyxis.patch(patch_url, payload).json() @@ -322,7 +315,7 @@ def add_container_image_repository(args: Dict[str, Any], image: Dict[str, Any]): raise Exception("Image metadata was not successfully added to Pyxis.") -def construct_repository(args): +def construct_repository(args, tags): image_name = args.name image_registry = image_name.split("/")[0] image_repo = image_name.split("/", 1)[1] @@ -341,7 +334,7 @@ def construct_repository(args): "registry": "registry.access.redhat.com", "repository": proxymap(image_name), "push_date": date_now, - "tags": pyxis_tags(args, date_now), + "tags": pyxis_tags(tags, date_now), } else: repo = { @@ -349,7 +342,7 @@ def construct_repository(args): "registry": image_registry, "repository": image_repo, "push_date": date_now, - "tags": pyxis_tags(args, date_now), + "tags": pyxis_tags(tags, date_now), } repo.update(repository_digest_values(args)) @@ -367,27 +360,52 @@ def main(): # pragma: no cover parsed_data = prepare_parsed_data(args) - # First check if it exists at all + if args.rh_push == "true": + image_repo = proxymap(args.name) + else: + image_repo = args.name.split("/", 1)[1] + + tags = args.tags.split() + if args.is_latest == "true": + tags.append("latest") + + # First check if it exists at all. If not, create it LOGGER.info(f"Checking to see if digest {args.architecture_digest} exists in pyxis") image = find_image(args.pyxis_url, args.architecture_digest) - if image is not None: - identifier = image["_id"] - # Then, check if it already references the given repository - if repo_in_image(proxymap(args.name), image): - LOGGER.info( - f"Image with given docker_image_digest already exists as {identifier} " - f"and is associated with repository {args.name}. " - "Skipping the image creation." - ) - else: - LOGGER.info( - f"Image with given docker_image_digest exists as {identifier}, but " - f"is not yet associated with repository {args.name}." - ) - add_container_image_repository(args, image) - else: + if image is None: LOGGER.info("Image with given docker_image_digest doesn't exist yet.") - create_container_image(args, parsed_data) + create_container_image(args, parsed_data, tags) + return + + identifier = image["_id"] + # Then, check if it already references the given repository. If not, add the repo + repo_index = find_repo_in_image(image_repo, image) + repositories = image["repositories"] + if repo_index is None: + LOGGER.info( + f"Image with given docker_image_digest exists as {identifier}, but " + f"is not yet associated with repository {args.name}." + ) + repositories.append(construct_repository(args, tags)) + update_container_image_repositories(args.pyxis_url, identifier, repositories) + return + + # Then, check if the tags are different. If they are, update them + existing_tags = [tag["name"] for tag in repositories[repo_index]["tags"]] + if existing_tags != tags: + LOGGER.info( + f"Image with given docker_image_digest exists as {identifier} and " + f"is associated with repository {args.name}, but the tags differ." + ) + repositories[repo_index] = construct_repository(args, tags) + update_container_image_repositories(args.pyxis_url, identifier, repositories) + return + + LOGGER.info( + f"Image with given docker_image_digest already exists as {identifier} " + f"and is associated with repository {args.name} and tags {args.tags}. " + "Skipping the image creation." + ) if __name__ == "__main__": # pragma: no cover diff --git a/pyxis/test_create_container_image.py b/pyxis/test_create_container_image.py index 4b10c36..4f940e6 100644 --- a/pyxis/test_create_container_image.py +++ b/pyxis/test_create_container_image.py @@ -6,12 +6,12 @@ from create_container_image import ( proxymap, find_image, - repo_in_image, + find_repo_in_image, prepare_parsed_data, pyxis_tags, repository_digest_values, create_container_image, - add_container_image_repository, + update_container_image_repositories, construct_repository, ) @@ -84,21 +84,21 @@ def test_find_image__no_id_in_image(mock_get): # scenario where repo is present in the image -def test_repo_in_image__true(): +def test_find_repo_in_image__found(): image = {"repositories": [{"repository": "my/repo"}, {"repository": "foo/bar"}]} - result = repo_in_image("foo/bar", image) + result = find_repo_in_image("foo/bar", image) - assert result + assert result == 1 # scenario where repo is not present in the image -def test_repo_in_image__false(): +def test_find_repo_in_image__not_found(): image = {"repositories": [{"repository": "my/repo"}, {"repository": "foo/bar"}]} - result = repo_in_image("something/missing", image) + result = find_repo_in_image("something/missing", image) - assert not result + assert result is None @patch("create_container_image.pyxis.post") @@ -112,18 +112,19 @@ def test_create_container_image(mock_datetime, mock_post): args = MagicMock() args.pyxis_url = PYXIS_URL - args.tags = "some_version" args.certified = "false" args.rh_push = "false" args.architecture_digest = "arch specific digest" args.digest = "some_digest" args.media_type = "single architecture" args.name = "quay.io/some_repo" + tags = ["some_version"] # Act create_container_image( args, {"architecture": "ok"}, + tags, ) # Assert @@ -155,56 +156,10 @@ def test_create_container_image(mock_datetime, mock_post): ) -@patch("create_container_image.pyxis.patch") -@patch("create_container_image.datetime") -def test_add_container_image_repository(mock_datetime, mock_patch): - # Mock an _id in the response for logger check - mock_patch.return_value.json.return_value = {"_id": 0} - - # mock date - mock_datetime.now = MagicMock(return_value=datetime(1970, 10, 10, 10, 10, 10)) - - args = MagicMock() - args.pyxis_url = PYXIS_URL - args.tags = "some_version" - args.rh_push = "true" - args.architecture_digest = "arch specific digest" - args.media_type = "single architecture" - args.name = "quay.io/redhat-pending/some_product----some_repo" - - # Act - add_container_image_repository( - args, - {"_id": "some_id", "repositories": []}, - ) - - # Assert - mock_patch.assert_called_with( - PYXIS_URL + "v1/images/id/some_id", - { - "repositories": [ - { - "published": True, - "registry": "registry.access.redhat.com", - "repository": "some_product/some_repo", - "push_date": "1970-10-10T10:10:10.000000+00:00", - "tags": [ - { - "added_date": "1970-10-10T10:10:10.000000+00:00", - "name": "some_version", - } - ], - # Note, no manifest_list_digest here. Single arch. - "manifest_schema2_digest": "arch specific digest", - }, - ], - }, - ) - - @patch("create_container_image.pyxis.post") @patch("create_container_image.datetime") -def test_create_container_image_latest(mock_datetime, mock_post): +def test_create_container_image__top_layer_id(mock_datetime, mock_post): + """Scenario where top_layer_id and uncompressed_top_layer_id are defined in parsed_data""" # Mock an _id in the response for logger check mock_post.return_value.json.return_value = {"_id": 0} @@ -213,22 +168,24 @@ def test_create_container_image_latest(mock_datetime, mock_post): args = MagicMock() args.pyxis_url = PYXIS_URL - args.tags = "some_version" args.certified = "false" - args.is_latest = "true" args.rh_push = "false" args.digest = "some_digest" args.architecture_digest = "arch specific digest" args.media_type = "application/vnd.oci.image.index.v1+json" args.digest = "some_digest" args.name = "redhat.com/some_repo/foobar" + tags = ["some_version", "latest"] # Act create_container_image( args, { "architecture": "ok", + "top_layer_id": "some_top_id", + "uncompressed_top_layer_id": "other_top_id", }, + tags, ) # Assert @@ -260,13 +217,15 @@ def test_create_container_image_latest(mock_datetime, mock_post): "architecture": "ok", "parsed_data": {"architecture": "ok"}, "sum_layer_size_bytes": 0, + "top_layer_id": "some_top_id", + "uncompressed_top_layer_id": "other_top_id", }, ) @patch("create_container_image.pyxis.post") @patch("create_container_image.datetime") -def test_create_container_image_rh_push_multiple_tags(mock_datetime, mock_post): +def test_create_container_image__rh_push_multiple_tags(mock_datetime, mock_post): # Mock an _id in the response for logger check mock_post.return_value.json.return_value = {"_id": 0} @@ -275,7 +234,6 @@ def test_create_container_image_rh_push_multiple_tags(mock_datetime, mock_post): args = MagicMock() args.pyxis_url = PYXIS_URL - args.tags = "tagprefix tagprefix-timestamp" args.certified = "false" args.rh_push = "true" args.digest = "some_digest" @@ -283,6 +241,7 @@ def test_create_container_image_rh_push_multiple_tags(mock_datetime, mock_post): args.media_type = "application/vnd.oci.image.index.v1+json" args.digest = "some_digest" args.name = "quay.io/redhat-pending/some-product----some-image" + tags = ["tagprefix", "tagprefix-timestamp"] # Act create_container_image( @@ -290,6 +249,7 @@ def test_create_container_image_rh_push_multiple_tags(mock_datetime, mock_post): { "architecture": "ok", }, + tags, ) # Assert @@ -325,7 +285,7 @@ def test_create_container_image_rh_push_multiple_tags(mock_datetime, mock_post): ) -def test_create_container_image_no_digest(): +def test_create_container_image__no_digest(): args = MagicMock() with pytest.raises(Exception): @@ -335,10 +295,11 @@ def test_create_container_image_no_digest(): "architecture": "ok", "name": "redhat.com/some_repo/foobar", }, + [], ) -def test_create_container_image_no_name(): +def test_create_container_image__no_name(): args = MagicMock() with pytest.raises(Exception): @@ -348,9 +309,50 @@ def test_create_container_image_no_name(): "architecture": "ok", "digest": "some_digest", }, + [], ) +@patch("create_container_image.pyxis.patch") +def test_update_container_image_repositories(mock_patch): + image_id = "0000" + # Mock an _id in the response for logger check + mock_patch.return_value.json.return_value = {"_id": image_id} + repositories = [ + { + "published": True, + "registry": "registry.access.redhat.com", + "repository": "some-product/some-image", + "push_date": "1970-10-10T10:10:10.000000+00:00", + "tags": [ + { + "added_date": "1970-10-10T10:10:10.000000+00:00", + "name": "tagprefix", + }, + { + "added_date": "1970-10-10T10:10:10.000000+00:00", + "name": "tagprefix-timestamp", + }, + ], + "manifest_list_digest": "some_digest", + "manifest_schema2_digest": "arch specific digest", + } + ] + + # Act + update_container_image_repositories( + PYXIS_URL, + image_id, + repositories, + ) + + # Assert + mock_patch.assert_called_with( + PYXIS_URL + "v1/images/id/0000", + {"repositories": repositories}, + ) + + @patch("builtins.open") def test_prepare_parsed_data__success(mock_open): args = MagicMock() @@ -454,30 +456,13 @@ def test_prepare_parsed_data__with_layer_sizes(mock_open): } -def test_pyxis_tags__with_latest(): - args = MagicMock() - args.tags = "tag1 tag2" - args.is_latest = "true" - now = "now" - - tags = pyxis_tags(args, now) - - assert tags == [ - {"added_date": "now", "name": "tag1"}, - {"added_date": "now", "name": "tag2"}, - {"added_date": "now", "name": "latest"}, - ] - - -def test_pyxis_tags__without_latest(): - args = MagicMock() - args.tags = "tag1 tag2" - args.is_latest = "false" +def test_pyxis_tags(): + tags = ["tag1", "tag2"] now = "now" - tags = pyxis_tags(args, now) + generated_tags = pyxis_tags(tags, now) - assert tags == [ + assert generated_tags == [ {"added_date": "now", "name": "tag1"}, {"added_date": "now", "name": "tag2"}, ] @@ -514,12 +499,11 @@ def test_construct_repository__rh_push_true(mock_datetime): args.media_type = "application/vnd.docker.distribution.manifest.list.v2+json" args.architecture_digest = "arch specific digest" args.digest = "some_digest" - args.tags = "tagprefix tagprefix-timestamp" - args.is_latest = "false" args.rh_push = "true" args.name = "quay.io/redhat-pending/some-product----some-image" + tags = ["tagprefix", "tagprefix-timestamp"] - repo = construct_repository(args) + repo = construct_repository(args, tags) assert repo == { "published": True, @@ -548,12 +532,11 @@ def test_construct_repository__rh_push_false(mock_datetime): args.media_type = "application/vnd.docker.distribution.manifest.list.v2+json" args.architecture_digest = "arch specific digest" args.digest = "some_digest" - args.tags = "tagprefix tagprefix-timestamp" - args.is_latest = "true" args.rh_push = "false" args.name = "quay.io/some-org/some-image" + tags = ["tagprefix", "tagprefix-timestamp", "latest"] - repo = construct_repository(args) + repo = construct_repository(args, tags) assert repo == { "published": False,