Reintroduce demisto-sdk to the new docker registry proxy

.changelog/4655.yml

@@ -0,0 +1,4 @@
+changes:
+- description: Reintroduce support for GAR DockerHub proxy when running in a Gitlab CI environment.
+  type: internal
+pr_number: 4655

demisto_sdk/commands/common/docker/dockerhub_client.py

@@ -4,7 +4,9 @@
 from typing import Any, Dict, List, Optional
 
 import dateparser
+import google.auth
 import requests
+from google.auth.transport.requests import Request
 from packaging.version import InvalidVersion, Version
 from requests.exceptions import ConnectionError, RequestException, Timeout
 
@@ -16,6 +18,8 @@
 DOCKERHUB_USER = "DOCKERHUB_USER"
 DOCKERHUB_PASSWORD = "DOCKERHUB_PASSWORD"
 DEFAULT_REPOSITORY = "demisto"
+IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI")
+DOCKER_IO = os.getenv("DOCKER_IO", "")
 
 
 class DockerHubAuthScope(StrEnum):
@@ -49,7 +53,7 @@ def __init__(
         password: str = "",
         verify_ssl: bool = False,
     ):
-        self.registry_api_url = registry or self.DEFAULT_REGISTRY
+        self.registry_api_url = get_registry_api_url(registry, self.DEFAULT_REGISTRY)
         self.docker_hub_api_url = docker_hub_api_url or self.DOCKER_HUB_API_BASE_URL
         self.username = username or os.getenv(DOCKERHUB_USER, "")
         self.password = password or os.getenv(DOCKERHUB_PASSWORD, "")
@@ -77,6 +81,18 @@ def get_token(
             repo: the repository to retrieve the token for.
             scope: the scope needed for the repository
         """
+        if IS_CONTENT_GITLAB_CI:
+            # If running in a Gitlab CI environment, try using the Google Cloud access token
+            logger.debug(
+                "Attempting to use Google Cloud access token for Docker Hub proxy authentication"
+            )
+            try:
+                if gcloud_access_token := get_gcloud_access_token():
+                    logger.debug("returning gcloud_access_token")
+                    return gcloud_access_token
+            except Exception as e:
+                logger.error(f"Failed to get gcloud access token: {e}")
+
         if token_metadata := self._docker_hub_auth_tokens.get(f"{repo}:{scope}"):
             now = datetime.now()
             if expiration_time := dateparser.parse(token_metadata.get("issued_at")):
@@ -371,6 +387,10 @@ def get_image_tag_metadata(self, docker_image: str, tag: str) -> Dict[str, Any]:
             tag: The tag of the docker image
         """
         try:
+            if IS_CONTENT_GITLAB_CI:
+                image_digest = self.get_image_digest(docker_image, tag=tag)
+                response = self.get_image_blobs(docker_image, image_digest=image_digest)
+                return response
             return self.do_docker_hub_get_request(
                 f"/repositories/{docker_image}/tags/{tag}"
             )
@@ -416,9 +436,12 @@ def get_docker_image_tag_creation_date(
             tag: The tag of the docker image
         """
         response = self.get_image_tag_metadata(docker_image, tag=tag)
-        return datetime.strptime(
-            response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ"
-        )
+        if creation_date := response.get("created"):
+            return datetime.strptime(creation_date, "%Y-%m-%dT%H:%M:%S.%fZ")
+        else:
+            return datetime.strptime(
+                response.get("last_updated", ""), "%Y-%m-%dT%H:%M:%S.%fZ"
+            )
 
     def get_latest_docker_image_tag(self, docker_image: str) -> Version:
         """
@@ -490,3 +513,118 @@ def get_repository_images_names(self, repo: str = DEFAULT_REPOSITORY) -> List[st
             for image_metadata in self.get_repository_images(repo)
             if image_metadata.get("name")
         ]
+
+
+### Google Artifactory functions ###
+
+
+def get_dockerhub_artifact_registry_url(base_path: str) -> str:
+    """
+    Parses a DockerHub Google Artifact Registry internal base path into a base url for DockerHub proxy API calls.
+    See Confluence page: Shared-Services GCP Services - GAR.
+
+    Args:
+    base_path (str): The base path of the Google Artifact Registry in the format 'region-domain/project/repository'.
+
+    Returns:
+        str: The base URL for the DockerHub proxy API calls based on the provided Artifact Registry path.
+        Example: For base_path 'us-docker.pkg.dev/my-project/my-repo',
+        the returned URL would be 'https://region-domain-docker.pkg.dev/v2/my-project/my-repo'
+
+    Raises:
+        ValueError: If the input base_path is not in the expected format.
+    """
+    logger.debug(
+        f"Trying to parse a DockerHub Google Artifact Registry internal base path {base_path=}"
+    )
+    # Split base path into region-domain, project, and repository
+    try:
+        region_domain, project, repository = base_path.split("/")
+    except ValueError:
+        raise ValueError(
+            "Invalid Artifact Registry path format. Expected format: 'region-domain/project/repository'"
+        )
+
+    # Construct and return the base URL for DockerHub proxy API calls
+    parsed_dockerhub_proxy_api_url = (
+        f"https://{region_domain}/v2/{project}/{repository}"
+    )
+    logger.debug(
+        f"Returning parsed DockerHub Google Artifact Registry API: {parsed_dockerhub_proxy_api_url=}"
+    )
+    return parsed_dockerhub_proxy_api_url
+
+
+def get_registry_api_url(registry: str, default_registry: str) -> str:
+    """
+    Determine the appropriate registry API URL based on the environment and provided parameters.
+
+    This function checks if the code is running in a GitLab CI environment and if a custom Docker.io URL is set.
+    If both conditions are true, it uses the custom Docker.io URL. Otherwise, it falls back to the provided
+    registry or the default registry.
+
+    Args:
+        registry (str): The registry URL provided by the user.
+        default_registry (str): The default registry URL to use if no custom URL is provided.
+
+    Returns:
+        str: The determined registry API URL to use for Docker operations.
+    """
+    logger.debug(f"dockerhub_client | {IS_CONTENT_GITLAB_CI=}, {DOCKER_IO=}")
+    if IS_CONTENT_GITLAB_CI and DOCKER_IO:
+        try:
+            logger.info(
+                "Running in a GitLab CI environment with custom DOCKER_IO environment variable, Trying to prase a DockerHub Google Artifact Registry from DOCKER_IO environment variable."
+            )
+            if parsed_dockerhub_proxy_api_url := get_dockerhub_artifact_registry_url(
+                DOCKER_IO
+            ):
+                return parsed_dockerhub_proxy_api_url
+            else:
+                logger.info(
+                    "Could not parse a valid API URL from the DOCKER_IO environment variable."
+                )
+        except Exception as e:
+            logger.info(
+                f"Could not parse a valid API URL from the DOCKER_IO environment variable, Error: {str(e)} "
+            )
+
+    logger.info(f"using provided or default registry, {default_registry=}")
+    return registry or default_registry
+
+
+def get_gcloud_access_token() -> Optional[str]:
+    """
+    Retrieves a Google Cloud access token using environment credentials.
+
+    This method attempts to obtain credentials from the environment and use them
+    to retrieve a valid Google Cloud access token. If successful, it returns the
+    token as a string. If unsuccessful, it returns None.
+
+    Returns:
+        str | None: The Google Cloud access token if successful, None otherwise.
+
+    Raises:
+        Exception: Any exception that occurs during the token retrieval process
+                   is caught and logged, but not re-raised.
+    """
+    try:
+        logger.debug("Trying to retrieve a Google Cloud access token.")
+        # Automatically obtain credentials from the environment
+        credentials, project_id = google.auth.default()
+
+        # Refresh the token if needed (ensures the token is valid)
+        credentials.refresh(Request())
+        # Extract the access token
+        access_token = credentials.token
+        if access_token:
+            logger.debug(
+                f"Successfully obtained Google Cloud access token, {project_id=}."
+            )
+            return access_token
+        else:
+            logger.debug("Failed to obtain Google Cloud access token.")
+            return None
+    except Exception as e:
+        logger.debug(f"Failed to get access token: {str(e)}")
+        return None

demisto_sdk/commands/common/docker_helper.py

@@ -31,6 +31,8 @@
 from demisto_sdk.commands.common.logger import logger
 from demisto_sdk.commands.common.tools import retry
 
+IS_CONTENT_GITLAB_CI = os.getenv("CONTENT_GITLAB_CI")
+DOCKER_IO = os.getenv("DOCKER_IO")
 DOCKER_CLIENT = None
 FILES_SRC_TARGET = List[Tuple[os.PathLike, str]]
 # this will be used to determine if the system supports mounts
@@ -54,6 +56,34 @@ class DockerException(Exception):
 
 
 def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):
+    """
+    Initialize and return a global Docker client to access and use a local Docker Daemon.
+
+    This function initializes a global Docker client if it doesn't exist, or returns the existing one.
+    It handles different environments, including GitLab CI, and attempts to log in to the Docker registry
+    if credentials are available.
+
+    Args:
+        timeout (int, optional): The timeout for Docker client operations in seconds. Defaults to 60.
+        log_prompt (str, optional): A prefix for log messages. Defaults to an empty string.
+
+    Returns:
+        docker.client.DockerClient: An initialized Docker client.
+
+    Raises:
+        docker.errors.DockerException: If initialization fails, likely due to Docker daemon not running.
+
+    Behavior:
+    1. Checks if a global Docker client already exists.
+    2. If in GitLab CI environment, attempts to create a client using the job environment.
+    3. If not in GitLab CI or if connecting via Gitlab CI environment fails,
+       attempts to log in to the Docker registry if credentials are available.
+    5. Logs various steps and outcomes of the initialization process.
+
+    Note:
+    - The function uses environment variables for Docker credentials.
+    - It handles both standard and SSH-based Docker connections.
+    """
     global DOCKER_CLIENT
     if DOCKER_CLIENT is None:
         if log_prompt:
@@ -63,6 +93,28 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):
         if ssh_client := os.getenv("DOCKER_SSH_CLIENT") is not None:
             logger.debug(f"{log_prompt} - Using ssh client setting: {ssh_client}")
         logger.debug(f"{log_prompt} - Using docker mounting: {CAN_MOUNT_FILES}")
+        try:
+            if IS_CONTENT_GITLAB_CI:
+                """In the case of running in Gitlab CI environment, try to init a docker client from the
+                job environment to utilize DockerHub API proxy requests (DOCKER_IO)"""
+                logger.info(
+                    "Gitlab CI use case detected, trying to create docker client from Gitlab CI job environment."
+                )
+                DOCKER_CLIENT = docker.from_env()
+                if DOCKER_CLIENT.ping():
+                    # see https://docker-py.readthedocs.io/en/stable/client.html#docker.client.DockerClient.ping for more information about ping().
+                    logger.info(
+                        "Successfully initialized docker client from Gitlab CI job environment."
+                    )
+                    return DOCKER_CLIENT
+                else:
+                    logger.warning(
+                        f"{log_prompt} - Failed to init docker client in Gitlab CI use case."
+                    )
+        except docker.errors.DockerException:
+            logger.warning(
+                f"{log_prompt} - Failed to init docker client in CONTENT_GITLAB_CI use case. "
+            )
         try:
             DOCKER_CLIENT = docker.from_env(timeout=timeout, use_ssh_client=ssh_client)  # type: ignore
         except docker.errors.DockerException:
@@ -90,8 +142,7 @@ def init_global_docker_client(timeout: int = 60, log_prompt: str = ""):
 
 def is_custom_registry():
     return (
-        not os.getenv("CONTENT_GITLAB_CI")
-        and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL
+        not IS_CONTENT_GITLAB_CI and DOCKER_REGISTRY_URL != DEFAULT_DOCKER_REGISTRY_URL
     )
 
 
@@ -153,6 +204,33 @@ def get_pip_requirements_from_file(requirements_file: Path) -> List[str]:
 
 
 class DockerBase:
+    """
+    Base class for Docker-related operations in the Demisto SDK.
+
+    This class utilizes any environment where a Docker Daemon is initialized,
+    and provides core functionality for working with Docker containers and images.
+
+    Attributes:
+        tmp_dir_name (tempfile.TemporaryDirectory): Temporary directory for Docker operations.
+        tmp_dir (Path): Path object for the temporary directory.
+        installation_scripts (dict): Mapping of container types to installation script paths.
+        changes (dict): Docker image changes for different container types.
+        requirements (Path): Path to the requirements.txt file.
+        _files_to_push_on_installation (List[Tuple[os.PathLike, str]]): Files to be pushed during installation.
+
+    Methods:
+        version(): Get the Docker version.
+        installation_files(container_type): Get installation files for a specific container type.
+        pull_image(image): Pull a Docker image.
+        is_image_available(image): Check if a Docker image is available.
+        copy_files_container(container, files): Copy files to a Docker container.
+        create_container(image, command, files_to_push, environment, **kwargs): Create a Docker container.
+        push_image(image, log_prompt): Push a Docker image to the repository.
+        create_image(base_image, image, container_type, install_packages, push, log_prompt): Create a new Docker image.
+        get_image_registry(image): Get the full image name with registry.
+        get_or_create_test_image(base_image, container_type, python_version, additional_requirements, push, should_pull, log_prompt): Get or create a test Docker image.
+    """
+
     def __init__(self):
         self.tmp_dir_name = tempfile.TemporaryDirectory(
             prefix=os.path.join(os.getcwd(), "tmp")
@@ -382,13 +460,13 @@ def create_image(
         container.commit(
             repository=repository, tag=tag, changes=self.changes[container_type]
         )
-        if os.getenv("CONTENT_GITLAB_CI"):
+        if IS_CONTENT_GITLAB_CI:
             container.commit(
                 repository=repository.replace(f"{DOCKER_REGISTRY_URL}/", ""),
                 tag=tag,
                 changes=self.changes[container_type],
             )
-        if push and os.getenv("CONTENT_GITLAB_CI"):
+        if push and IS_CONTENT_GITLAB_CI:
             self.push_image(image, log_prompt=log_prompt)
         return image
 
@@ -659,6 +737,17 @@ def get_python_version(image: Optional[str]) -> Optional[Version]:
         return python_version
     logger.debug(f"Could not get python version for {image=} from regex")
 
+    if IS_CONTENT_GITLAB_CI:
+        try:
+            logger.debug(
+                f"get python version for {image=} from available docker client"
+            )
+            return _get_python_version_from_image_client(image)
+        except Exception:
+            logger.debug(
+                f"Could not get python version for {image=} from available docker client"
+            )
+
     try:
         logger.debug(f"get python version for {image=} from dockerhub api")
         return _get_python_version_from_dockerhub_api(image)
@@ -710,7 +799,7 @@ def _get_python_version_from_dockerhub_api(image: str) -> Version:
         raise ValueError(f"Invalid docker image: {image}")
     else:
         repo, tag = image.split(":")
-    if os.getenv("CONTENT_GITLAB_CI"):
+    if IS_CONTENT_GITLAB_CI:
         # we need to remove the gitlab prefix, as we query the API
         repo = repo.replace(f"{DOCKER_REGISTRY_URL}/", "")
     try: