fix(KFLUXBUGS-1848): associate existing images with a new repository

pyxis/create_container_image.py

@@ -132,29 +132,28 @@ def setup_argparser() -> Any:  # pragma: no cover
     return parser
 
 
-def image_already_exists(args, digest: str, repository: str) -> bool:
+def proxymap(repository: str) -> str:
+    """Map a backend repo name to its proxy equivalent.
+
+    i.e., map quay.io/redhat-pending/foo----bar to foo/bar
+    """
+    return repository.split("/")[-1].replace("----", "/")
+
+
+def image_already_exists(args, digest: str, repository: str) -> Any:
     """Function to check if a containerImage with the given digest and repository
     already exists in the pyxis instance
 
-    :return: True if one exists, else false
+    If `repository` is None, then the return True if the image exists at all.
+
+    :return: the image id, if one exists, else None if not found
     """
 
-    # we need the repository name without the registry and organization
-    # given:
-    #   quay.io/redhat-pending/ubi9----buildah
-    # we need
-    #   ubi9/buildah
-    #
-    # is we are given:
-    #  quay.io/konflux-ci/release-service-utils
-    # then we need:
-    #  release-service-utils
-    repo = repository.split("/")[2].replace("----", "/")
     # quote is needed to urlparse the quotation marks
-    filter_str = quote(
-        f'repositories.manifest_schema2_digest=="{digest}";'
-        f'not(deleted==true);repositories.repository=="{repo}"'
-    )
+    raw_filter = f'repositories.manifest_schema2_digest=="{digest}";not(deleted==true)'
+    if repository:
+        raw_filter += f';repositories.repository=="{proxymap(repository)}"'
+    filter_str = quote(raw_filter)
 
     check_url = urljoin(args.pyxis_url, f"v1/images?page_size=1&filter={filter_str}")
 
@@ -165,18 +164,14 @@ def image_already_exists(args, digest: str, repository: str) -> bool:
     query_results = rsp.json()["data"]
 
     if len(query_results) == 0:
-        LOGGER.info("Image with given docker_image_digest doesn't exist yet")
-        return False
+        return None
 
-    LOGGER.info(
-        "Image with given docker_image_digest already exists." "Skipping the image creation."
-    )
     if "_id" in query_results[0]:
-        LOGGER.info(f"The image id is: {query_results[0]['_id']}")
+        LOGGER.info(f"Found image id is: {query_results[0]['_id']}")
     else:
         raise Exception("Image metadata was found in Pyxis, but the id key was missing.")
 
-    return True
+    return query_results[0]
 
 
 def prepare_parsed_data(args) -> Dict[str, Any]:
@@ -228,6 +223,28 @@ def prepare_parsed_data(args) -> Dict[str, Any]:
     return parsed_data
 
 
+def pyxis_tags(args, date_now):
+    """Return list of tags formatted for pyxis"""
+    tags = args.tags.split()
+    if args.is_latest == "true":
+        tags.append("latest")
+    return [
+        {
+            "added_date": date_now,
+            "name": tag,
+        }
+        for tag in tags
+    ]
+
+
+def repository_digest_values(args, docker_image_digest):
+    """Return digest values for the repository entry in the image entity"""
+    result = {"manifest_schema2_digest": args.architecture_digest}
+    if args.media_type in MANIFEST_LIST_TYPES:
+        result["manifest_list_digest"] = docker_image_digest
+    return result
+
+
 def create_container_image(args, parsed_data: Dict[str, Any]):
     """Function to create a new containerImage entry in a pyxis instance"""
 
@@ -260,25 +277,14 @@ def create_container_image(args, parsed_data: Dict[str, Any]):
 
     upload_url = urljoin(args.pyxis_url, "v1/images")
 
-    tags = args.tags.split()
-    if args.is_latest == "true":
-        tags.append("latest")
-    pyxis_tags = [
-        {
-            "added_date": date_now,
-            "name": tag,
-        }
-        for tag in tags
-    ]
-
     container_image_payload = {
         "repositories": [
             {
                 "published": False,
                 "registry": image_registry,
                 "repository": image_repo,
                 "push_date": date_now,
-                "tags": pyxis_tags,
+                "tags": pyxis_tags(args, date_now),
             }
         ],
         "certified": json.loads(args.certified.lower()),
@@ -290,24 +296,22 @@ def create_container_image(args, parsed_data: Dict[str, Any]):
         "uncompressed_top_layer_id": uncompressed_top_layer_id,
     }
 
-    container_image_payload["repositories"][0][
-        "manifest_schema2_digest"
-    ] = args.architecture_digest
-    if args.media_type in MANIFEST_LIST_TYPES:
-        container_image_payload["repositories"][0][
-            "manifest_list_digest"
-        ] = docker_image_digest
+    container_image_payload["repositories"][0].update(
+        repository_digest_values(args, docker_image_digest)
+    )
 
     # For images released to registry.redhat.io we need a second repository item
     # with published=true and registry and repository converted.
     # E.g. if the name in the oras manifest result is
     # "quay.io/redhat-prod/rhtas-tech-preview----cosign-rhel9",
     # repository will be "rhtas-tech-preview/cosign-rhel9"
-    if args.rh_push == "true":
+    if not args.rh_push == "true":
+        LOGGER.info("--rh-push is not set. Skipping public registry association.")
+    else:
         repo = container_image_payload["repositories"][0].copy()
         repo["published"] = True
         repo["registry"] = "registry.access.redhat.com"
-        repo["repository"] = image_name.split("/")[-1].replace("----", "/")
+        repo["repository"] = proxymap(image_name)
         container_image_payload["repositories"].append(repo)
 
     rsp = pyxis.post(upload_url, container_image_payload).json()
@@ -319,6 +323,41 @@ def create_container_image(args, parsed_data: Dict[str, Any]):
         raise Exception("Image metadata was not successfully added to Pyxis.")
 
 
+def add_container_image_repository(args, parsed_data: Dict[str, Any], image: Dict[str, Any]):
+    if not args.rh_push == "true":
+        LOGGER.info("--rh-push is not set. Skipping public registry association.")
+        return
+
+    identifier = image["_id"]
+    LOGGER.info(f"Adding repository to container image {identifier}")
+
+    date_now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%f+00:00")
+
+    image_name = parsed_data["name"]
+    docker_image_digest = parsed_data["digest"]
+
+    patch_url = urljoin(args.pyxis_url, f"v1/images/id/{identifier}")
+
+    image["repositories"].append(
+        {
+            "published": True,
+            "registry": "registry.access.redhat.com",
+            "repository": proxymap(image_name),
+            "push_date": date_now,
+            "tags": pyxis_tags(args, date_now),
+        }
+    )
+    image["repositories"][-1].update(repository_digest_values(args, docker_image_digest))
+
+    rsp = pyxis.patch(patch_url, image).json()
+
+    # Make sure container metadata was successfully added to Pyxis
+    if "_id" in rsp:
+        LOGGER.info(f"The image id is: {rsp['_id']}")
+    else:
+        raise Exception("Image metadata was not successfully added to Pyxis.")
+
+
 def main():  # pragma: no cover
     """Main func"""
 
@@ -329,7 +368,25 @@ def main():  # pragma: no cover
 
     parsed_data = prepare_parsed_data(args)
 
-    if not image_already_exists(args, args.architecture_digest, args.name):
+    # First check if it exists at all
+    image = image_already_exists(args, args.architecture_digest, repository=None)
+    if image:
+        # Then, check if it exists in association with the given repository
+        identifier = image["_id"]
+        if image_already_exists(args, args.architecture_digest, repository=args.name):
+            LOGGER.info(
+                f"Image with given docker_image_digest already exists as {identifier} "
+                f"and is associated with repository {args.name}. "
+                "Skipping the image creation."
+            )
+        else:
+            LOGGER.info(
+                f"Image with given docker_image_digest exists as {identifier}, but "
+                f"is not yet associated with repository {args.name}."
+            )
+            add_container_image_repository(args, parsed_data, image)
+    else:
+        LOGGER.info("Image with given docker_image_digest doesn't exist yet.")
         create_container_image(args, parsed_data)
 
 

pyxis/pyxis.py

@@ -82,6 +82,34 @@ def post(url: str, body: Dict[str, Any]) -> requests.Response:
     return resp
 
 
+def patch(url: str, body: Dict[str, Any]) -> requests.Response:
+    """PATCH pyxis API request to given URL with given payload
+
+    Args:
+        url (str): Pyxis API URL
+        body (Dict[str, Any]): Request payload
+
+    :return: Pyxis response
+    """
+    global session
+    if session is None:
+        session = _get_session()
+
+    LOGGER.debug(f"PATCH request URL: {url}")
+    LOGGER.debug(f"PATCH request body: {body}")
+    resp = session.patch(url, json=body)
+
+    try:
+        LOGGER.debug(f"PATCH request response: {resp.text}")
+        resp.raise_for_status()
+    except requests.HTTPError:
+        LOGGER.exception(
+            f"Pyxis PATCH query failed with {url} - {resp.status_code} - {resp.text}"
+        )
+        raise
+    return resp
+
+
 def graphql_query(graphql_api: str, body: Dict[str, Any]) -> Dict[str, Any]:
     """Make a request to Pyxis GraphQL API
 

pyxis/test_create_container_image.py

@@ -6,6 +6,7 @@
 from create_container_image import (
     image_already_exists,
     create_container_image,
+    add_container_image_repository,
     prepare_parsed_data,
 )
 
@@ -60,6 +61,32 @@ def test_image_already_exists__image_does_not_exist(mock_get):
     assert not exists
 
 
+@patch("create_container_image.pyxis.get")
+def test_image_already_exists__image_does_exist_but_no_repo(mock_get):
+    # Arrange
+    mock_rsp = MagicMock()
+    mock_get.return_value = mock_rsp
+    args = MagicMock()
+    args.pyxis_url = mock_pyxis_url
+    args.architecture_digest = "some_digest"
+    args.name = "server/org/some_name"
+
+    # Image already exists
+    mock_rsp.json.return_value = {"data": [{"_id": 0}]}
+
+    # Act
+    exists = image_already_exists(args, args.architecture_digest, None)
+
+    # Assert
+    assert exists
+    mock_get.assert_called_once_with(
+        mock_pyxis_url
+        + "v1/images?page_size=1&filter="
+        + "repositories.manifest_schema2_digest%3D%3D%22some_digest%22"
+        + "%3Bnot%28deleted%3D%3Dtrue%29"
+    )
+
+
 @patch("create_container_image.pyxis.post")
 @patch("create_container_image.datetime")
 def test_create_container_image(mock_datetime, mock_post):
@@ -114,6 +141,54 @@ def test_create_container_image(mock_datetime, mock_post):
     )
 
 
+@patch("create_container_image.pyxis.patch")
+@patch("create_container_image.datetime")
+def test_add_container_image_repository(mock_datetime, mock_patch):
+    # Mock an _id in the response for logger check
+    mock_patch.return_value.json.return_value = {"_id": 0}
+
+    # mock date
+    mock_datetime.now = MagicMock(return_value=datetime(1970, 10, 10, 10, 10, 10))
+
+    args = MagicMock()
+    args.pyxis_url = mock_pyxis_url
+    args.tags = "some_version"
+    args.rh_push = "true"
+    args.architecture_digest = "arch specific digest"
+    args.media_type = "single architecture"
+
+    # Act
+    add_container_image_repository(
+        args,
+        {"architecture": "ok", "digest": "some_digest", "name": "quay.io/namespace/some_repo"},
+        {"_id": "some_id", "repositories": []},
+    )
+
+    # Assert
+    mock_patch.assert_called_with(
+        mock_pyxis_url + "v1/images/id/some_id",
+        {
+            "_id": "some_id",
+            "repositories": [
+                {
+                    "published": True,
+                    "registry": "registry.access.redhat.com",
+                    "repository": "some_repo",
+                    "push_date": "1970-10-10T10:10:10.000000+00:00",
+                    "tags": [
+                        {
+                            "added_date": "1970-10-10T10:10:10.000000+00:00",
+                            "name": "some_version",
+                        }
+                    ],
+                    # Note, no manifest_list_digest here. Single arch.
+                    "manifest_schema2_digest": "arch specific digest",
+                }
+            ],
+        },
+    )
+
+
 @patch("create_container_image.pyxis.post")
 @patch("create_container_image.datetime")
 def test_create_container_image_latest(mock_datetime, mock_post):