From 401264ed20744db0136e35ebcec12da1dbc91953 Mon Sep 17 00:00:00 2001 From: yengliong Date: Fri, 11 Oct 2024 18:10:01 +0800 Subject: [PATCH 1/2] [NEXMANAGE-853] Remove ORAS and update method The image can be downloaded from the release server now. It still requires the JWT token. This PR removes ORAS related module and update the method for downloading the image using requests module. Signed-off-by: yengliong --- inbm/Changelog.md | 3 +- .../dispatcher/sota/constants.py | 4 +- .../dispatcher/sota/downloader.py | 16 +- .../dispatcher/sota/oras_util.py | 238 ------------------ .../dispatcher/sota/os_factory.py | 15 +- .../dispatcher/sota/os_updater.py | 13 +- inbm/dispatcher-agent/dispatcher/sota/sota.py | 4 +- .../dispatcher/sota/tiber_util.py | 161 ++++++++++++ .../dispatcher/sota/update_tool_util.py | 58 ++--- .../tests/unit/sota/test_downloader.py | 4 +- .../{test_oras_util.py => test_tiber_util.py} | 46 ++-- .../tests/unit/sota/test_update_tool_util.py | 6 +- 12 files changed, 254 insertions(+), 314 deletions(-) delete mode 100644 inbm/dispatcher-agent/dispatcher/sota/oras_util.py create mode 100644 inbm/dispatcher-agent/dispatcher/sota/tiber_util.py rename inbm/dispatcher-agent/tests/unit/sota/{test_oras_util.py => test_tiber_util.py} (76%) diff --git a/inbm/Changelog.md b/inbm/Changelog.md index 07715157d..2b2602ee5 100644 --- a/inbm/Changelog.md +++ b/inbm/Changelog.md @@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ## NEXT - MMMM-DD-YY - +### Changed + - (NEXMANAGE-853) Remove ORAS and update method ## 4.2.6 - 2024-10-04 ### Added diff --git a/inbm/dispatcher-agent/dispatcher/sota/constants.py b/inbm/dispatcher-agent/dispatcher/sota/constants.py index e504264c2..43181d863 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/constants.py +++ b/inbm/dispatcher-agent/dispatcher/sota/constants.py @@ -17,8 +17,8 @@ # Tiber Update Tool file path TIBER_UPDATE_TOOL_PATH = get_canonical_representation_of_path('/usr/bin/os-update-tool.sh') -# ORAS release server access token path -ORAS_TOKEN_PATH = get_canonical_representation_of_path('/etc/intel_edge_node/tokens/platform-update-agent/' +# Release server access token path +RELEASE_SERVER_TOKEN_PATH = get_canonical_representation_of_path('/etc/intel_edge_node/tokens/platform-update-agent/' 'rs_access_token') SOTA_STATE = 'normal' diff --git a/inbm/dispatcher-agent/dispatcher/sota/downloader.py b/inbm/dispatcher-agent/dispatcher/sota/downloader.py index f2e033191..1145fba01 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/downloader.py +++ b/inbm/dispatcher-agent/dispatcher/sota/downloader.py @@ -12,7 +12,7 @@ from typing import Optional from .mender_util import read_current_mender_version from .sota_error import SotaError -from .oras_util import oras_download, read_oras_token +from .tiber_util import read_release_server_token, tiber_download from ..constants import UMASK_OTA from ..downloader import download from ..packagemanager.irepo import IRepo @@ -195,14 +195,14 @@ def download(self, if uri is None: raise SotaError("URI is None while performing TiberOS download") - password = read_oras_token() + password = read_release_server_token() - oras_download(dispatcher_broker=dispatcher_broker, - uri=uri, - repo=repo, - umask=UMASK_OTA, - username=username, - password=password) + tiber_download(dispatcher_broker=dispatcher_broker, + uri=uri, + repo=repo, + umask=UMASK_OTA, + username=username, + password=password) def check_release_date(self, release_date: Optional[str]) -> bool: raise NotImplementedError() \ No newline at end of file diff --git a/inbm/dispatcher-agent/dispatcher/sota/oras_util.py b/inbm/dispatcher-agent/dispatcher/sota/oras_util.py deleted file mode 100644 index cfd8a096a..000000000 --- a/inbm/dispatcher-agent/dispatcher/sota/oras_util.py +++ /dev/null @@ -1,238 +0,0 @@ -""" - ORAS tool will be called by dispatcher to perform the image downloading in TiberOS. - - Copyright (C) 2017-2024 Intel Corporation - SPDX-License-Identifier: Apache-2.0 -""" -import logging -import os -import requests -import json -import shlex -from urllib.parse import urlsplit, urlparse -from typing import Optional, Tuple -from inbm_common_lib.shell_runner import PseudoShellRunner -from inbm_common_lib.utility import CanonicalUri -from dispatcher.packagemanager.package_manager import verify_source -from dataclasses import dataclass -from ..packagemanager.irepo import IRepo -from ..dispatcher_broker import DispatcherBroker -from .constants import ORAS_TOKEN_PATH -from .sota_error import SotaError - -logger = logging.getLogger(__name__) - - -@dataclass -class ParsedURI: - source: str - registry_server: str - image: str - image_tag: str - image_full_path: str - repository_name: str - registry_manifest: str - - -def oras_download(dispatcher_broker: DispatcherBroker, uri: CanonicalUri, - repo: IRepo, username: Optional[str], password: str, umask: int) -> None: - """Downloads files and places capsule file in path mentioned by manifest file. - - @param dispatcher_broker: DispatcherBroker object used to communicate with other INBM services - @param uri: URI of the source location - @param repo: repository for holding the download - @param username: username to use for download - @param password: password to use for download - @param umask: file permission mask - @raises SotaError: any exception - """ - dispatcher_broker.telemetry(f'Package to be fetched from {uri.value}') - dispatcher_broker.telemetry( - 'Checking authenticity of package by checking signature and source') - - if not isinstance(uri, CanonicalUri): - raise SotaError("Internal error: uri improperly passed to download function") - - try: - parsed_uri = parse_uri(uri) - - source = parsed_uri.source - registry_server = parsed_uri.registry_server - image = parsed_uri.image - image_tag = parsed_uri.image_tag - repository_name = parsed_uri.repository_name - image_full_path = parsed_uri.image_full_path - registry_manifest = parsed_uri.registry_manifest - - except IndexError as err: - logger.error(f"IndexError occurs with uri {uri.value}: {err}") - raise SotaError(err) - - logger.debug(f"source: {source}, " - f"registry_server: {registry_server}, " - f"image: {image}, " - f"image_tag: {image_tag}, " - f"repository_name: {repository_name}, " - f"image_full_path: {image_full_path}, " - f"registry_manifest: {registry_manifest}") - - verify_source(source=source, dispatcher_broker=dispatcher_broker) - dispatcher_broker.telemetry('Source Verification check passed') - - enough_space = is_enough_space_to_download( - registry_manifest, repo, password) - - if not enough_space: - err_msg = " Insufficient free space available on " + shlex.quote(repo.get_repo_path()) + \ - " for " + str(uri.value) - raise SotaError(err_msg) - - if password: - logger.debug("RS password provided.") - else: - err_msg = " No JWT token. Abort the update. " - raise SotaError(err_msg) - - msg = f'Fetching software package from {image_full_path}' - dispatcher_broker.telemetry(msg) - - # Call oras to pull the image. The password is the JWT token. - (out, err_run, code) = PseudoShellRunner().run(f"oras pull {image_full_path} -o {repo.get_repo_path()} " - f"--password-stdin", stdin=password) - if code != 0: - if err_run is not None: - raise SotaError("Error to download OTA files with ORAS: " + err_run + ". Code: " + str(code)) - else: - raise SotaError("Error to download OTA files with ORAS. Code: " + str(code)) - else: - dispatcher_broker.telemetry('OTA Download Successful.') - - -def is_enough_space_to_download(manifest_uri: str, - destination_repo: IRepo, - jwt_token: str) -> bool: - """Checks if enough free space exists on platform to hold download. - - Calculates the file size from the OCI server and checks if required free space is available on - the platform. - @param manifest_uri: registry manifest uri - @param destination_repo: desired download destination - @param jwt_token: jwt_token provided for access the release server - """ - try: - logger.debug(f"Checking OCI artifact size with manifest uri: {manifest_uri}") - headers = { - "Authorization": f"Bearer {jwt_token}", - "Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.oci.artifact.manifest.v1+json" - } - response = requests.get(manifest_uri, headers=headers) - if response.status_code != 200: - raise SotaError(f"Failed to get the response from {manifest_uri}.") - data = json.loads(response.text) - logger.debug(f"resp={data}") - # Calculate the total size - file_size = 0 - for layer in data['layers']: - file_size += layer['size'] - logger.debug(f"Total file size: {file_size}") - - except (TypeError, KeyError, json.JSONDecodeError, SotaError) as err: - err_msg = f"Error getting artifact size from {manifest_uri} with token Error: {err}" - logger.error(err_msg) - raise SotaError(err_msg) - - if destination_repo.exists(): - get_free_space = destination_repo.get_free_space() - free_space: int = int(get_free_space) - else: - raise SotaError("Repository does not exist : " + - shlex.quote(destination_repo.get_repo_path())) - - logger.debug("get_free_space: " + repr(get_free_space)) - logger.debug("Free space available on destination_repo is " + repr(free_space)) - logger.debug("Free space needed on destination repo is " + repr(file_size)) - return True if free_space > file_size else False - - -def parse_uri(uri: CanonicalUri) -> ParsedURI: - """ Parse a CanonicalUri and extract its components for a container image URI. - - In case of uri.value = https://registry-rs.internal.ledgepark.intel.com/one-intel-edge/tiberos:latest - source = https://registry-rs.internal.ledgepark.intel.com/one-intel-edge - registry_server = registry-rs.internal.ledgepark.intel.com - image = tiberos - image_tag = latest - image_full_path = registry-rs.internal.ledgepark.intel.com/one-intel-edge/tiberos:latest - repository_name = one-intel-edge - registry_manifest = https://registry-rs.internal.ledgepark.intel.com/v2/one-intel-edge/tiberos/manifest/latest - - @param uri: A URI object with a value containing a full image URI. - @return: A data structure containing the following fields: - - - source: The base URL of the registry, including the repository (e.g., 'https://registry-rs.internal.ledgepark.intel.com/one-intel-edge') - - registry_server: The server hosting the registry (e.g., 'registry-rs.internal.ledgepark.intel.com') - - image: The image name (e.g., 'tiberos') - - image_tag: The image tag (e.g., 'latest') - - image_full_path: The full path to the image with tag (e.g., 'registry-rs.internal.ledgepark.intel.com/one-intel-edge/tiberos:latest') - - repository_name: The repository name (e.g., 'one-intel-edge') - - registry_manifest: The URL to the image's manifest (e.g., 'https://registry-rs.internal.ledgepark.intel.com/v2/one-intel-edge/tiberos/manifest/latest') - - Raises: - ValueError: If the URI path does not contain enough components to extract the necessary information. - - """ - source = uri.value[:-(len(uri.value.split('/')[-1]) + 1)] - parsed_uri = urlparse(uri.value) - registry_server = parsed_uri.netloc - parsed_uri.geturl() - path_parts = parsed_uri.path.strip('/').split('/') - if len(path_parts) < 2: - raise SotaError(f"URI path does not contain enough components: {uri.value}") - # Extract repository name and image details - repository_name = '/'.join(path_parts[:-1]) - # Extract image and tag (default tag is 'latest' if not provided) - image = path_parts[-1].split(':')[0] - image_tag = path_parts[-1].split(':')[1] if ':' in path_parts[-1] else 'latest' - - # Construct the required fields - image_full_path = f"{registry_server}/{repository_name}/{image}:{image_tag}" - image_full_path = shlex.quote(image_full_path) - registry_manifest = ( - f"{parsed_uri.scheme}://{registry_server}/v2/" - f"{repository_name}/{image}/manifests/{image_tag}" - ) - - # Return the populated ParsedURI data structure - return ParsedURI( - source=source, - registry_server=registry_server, - image=str(image), - image_tag=image_tag, - image_full_path=image_full_path, - repository_name=str(repository_name), - registry_manifest=registry_manifest, - ) - - -def read_oras_token() -> str: - """Read oras JWT token from a path configured by Tiber OS node-agent. The node agent will renew the token when - the token is expired. - - @return: JWT token to access release server - """ - token = None - try: - if os.path.exists(ORAS_TOKEN_PATH): - with open(ORAS_TOKEN_PATH, 'r') as f: - token = f.read().strip() - return token - else: - msg = f"{ORAS_TOKEN_PATH} not exist." - except OSError as err: - raise SotaError(f"Error while performing TiberOS download: {err}") - - if token is None: - msg = f"No JWT token found." - - logger.error(msg) - raise SotaError(f"Error while performing TiberOS download: {msg}") diff --git a/inbm/dispatcher-agent/dispatcher/sota/os_factory.py b/inbm/dispatcher-agent/dispatcher/sota/os_factory.py index 4f85d054a..4eb6f6cd2 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/os_factory.py +++ b/inbm/dispatcher-agent/dispatcher/sota/os_factory.py @@ -30,18 +30,20 @@ class SotaOsFactory: def __init__(self, dispatcher_broker: DispatcherBroker, sota_repos: Optional[str] = None, package_list: list[str] = [], - signature: Optional[str] = None) -> None: + signature: Optional[str] = None, uri: Optional[str] = None) -> None: """Initializes OsFactory. @param dispatcher_broker: DispatcherBroker object used to communicate with other INBM services @param sota_repos: new Ubuntu/Debian mirror (or None) @param package_list: list of packages to install/update (or empty for all--general upgrade) @param signature: signature used to verify image + @param uri: uri provided in the manifest """ self._sota_repos = sota_repos self._package_list = package_list self._dispatcher_broker = dispatcher_broker self._signature = signature + self._uri = uri @staticmethod def verify_os_supported() -> str: @@ -78,10 +80,10 @@ def get_os(self, os_type) -> "ISotaOs": #TODO: Remove this when confirmed that TiberOS is in use elif os_type == LinuxDistType.Mariner.name: logger.debug("Mariner returned") - return TiberOSBasedSotaOs(self._dispatcher_broker, self._signature) + return TiberOSBasedSotaOs(self._dispatcher_broker, self._signature, self._uri) elif os_type == LinuxDistType.tiber.name: logger.debug("TiberOS returned") - return TiberOSBasedSotaOs(self._dispatcher_broker, self._signature) + return TiberOSBasedSotaOs(self._dispatcher_broker, self._signature, self._uri) raise ValueError('Unsupported OS type: ' + os_type) @@ -261,14 +263,17 @@ def create_downloader(self) -> Downloader: class TiberOSBasedSotaOs(ISotaOs): """TiberOSBasedSotaOs class, child of ISotaOs""" - def __init__(self, dispatcher_broker: DispatcherBroker, signature: Optional[str] = None) -> None: + def __init__(self, dispatcher_broker: DispatcherBroker, signature: Optional[str] = None, + uri: Optional[str] = None) -> None: """Constructor. @param dispatcher_broker: DispatcherBroker object used to communicate with other INBM services @param signature: signature used to verify image + @param uri: uri provided in the manifest """ self._dispatcher_broker = dispatcher_broker self._signature = signature + self._uri = uri def create_setup_helper(self) -> SetupHelper: logger.debug("") @@ -280,7 +285,7 @@ def create_rebooter(self) -> Rebooter: def create_os_updater(self) -> OsUpdater: logger.debug("") - return TiberOSUpdater(signature=self._signature) + return TiberOSUpdater(signature=self._signature, uri=self._uri) def create_snapshotter(self, sota_cmd: str, snap_num: Optional[str], proceed_without_rollback: bool, reboot_device: bool) -> Snapshot: diff --git a/inbm/dispatcher-agent/dispatcher/sota/os_updater.py b/inbm/dispatcher-agent/dispatcher/sota/os_updater.py index dcc6e2660..d2cf44e3a 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/os_updater.py +++ b/inbm/dispatcher-agent/dispatcher/sota/os_updater.py @@ -20,7 +20,7 @@ from inbm_common_lib.utility import get_canonical_representation_of_path from .command_list import CommandList -from .constants import MENDER_FILE_PATH +from .constants import MENDER_FILE_PATH, SOTA_CACHE from .converter import size_to_bytes from .sota_error import SotaError from ..common import uri_utilities @@ -398,10 +398,12 @@ class TiberOSUpdater(OsUpdater): """TiberOSUpdater class, child of OsUpdater @param signature: signature used to verify image + @param uri: uri provided in the manifest """ - def __init__(self, signature: Optional[str] = None) -> None: + def __init__(self, signature: Optional[str] = None, uri: Optional[str] = None) -> None: super().__init__() self._signature = signature + self._uri = uri def update_remote_source(self, uri: Optional[CanonicalUri], signature: Optional[str], repo: irepo.IRepo) -> List[str]: @@ -430,5 +432,10 @@ def no_download(self) -> list[str]: def download_only(self) -> list[str]: """Return the UT write command""" - cmds = [update_tool_write_command(self._signature)] + # Extract the file path from uri. + file_path = None + if self._uri: + file_path = os.path.join(SOTA_CACHE, self._uri.split('/')[-1]) + + cmds = [update_tool_write_command(self._signature, file_path)] return CommandList(cmds).cmd_list \ No newline at end of file diff --git a/inbm/dispatcher-agent/dispatcher/sota/sota.py b/inbm/dispatcher-agent/dispatcher/sota/sota.py index ce650c99c..501265aff 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/sota.py +++ b/inbm/dispatcher-agent/dispatcher/sota/sota.py @@ -239,7 +239,7 @@ def execute(self, proceed_without_rollback: bool, skip_sleeps: bool = False) -> raise SotaError(F'parsing and validating package list: {self._package_list} failed') os_factory = SotaOsFactory(self._dispatcher_broker, - self._sota_repos, validated_package_list) + self._sota_repos, validated_package_list, self._signature, self._uri) try: os_type = detect_os() except ValueError as e: @@ -447,7 +447,7 @@ def check(self) -> None: if validated_package_list is None: raise SotaError(F'parsing and validating package list: {self._package_list} failed') os_factory = SotaOsFactory(self._dispatcher_broker, - self._sota_repos, validated_package_list) + self._sota_repos, validated_package_list, self._signature, self._uri) try: os_type = detect_os() except ValueError as e: diff --git a/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py b/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py new file mode 100644 index 000000000..f2f77ded3 --- /dev/null +++ b/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py @@ -0,0 +1,161 @@ +""" + ORAS tool will be called by dispatcher to perform the image downloading in TiberOS. + + Copyright (C) 2017-2024 Intel Corporation + SPDX-License-Identifier: Apache-2.0 +""" +import logging +import os +import requests +from requests import HTTPError +from requests.exceptions import ProxyError, ChunkedEncodingError, ContentDecodingError, ConnectionError + +import shlex +from urllib.parse import urlsplit +from typing import Optional, Any +from inbm_common_lib.utility import CanonicalUri +from dispatcher.packagemanager.package_manager import verify_source +from ..packagemanager.irepo import IRepo +from ..dispatcher_broker import DispatcherBroker +from .constants import RELEASE_SERVER_TOKEN_PATH +from .sota_error import SotaError + +logger = logging.getLogger(__name__) + + +def tiber_download(dispatcher_broker: DispatcherBroker, uri: CanonicalUri, + repo: IRepo, username: Optional[str], password: str, umask: int) -> None: + """Downloads files and places capsule file in path mentioned by manifest file. + + @param dispatcher_broker: DispatcherBroker object used to communicate with other INBM services + @param uri: URI of the source location + @param repo: repository for holding the download + @param username: username to use for download + @param password: password to use for download + @param umask: file permission mask + @raises SotaError: any exception + """ + dispatcher_broker.telemetry(f'Package to be fetched from {uri.value}') + dispatcher_broker.telemetry( + 'Checking authenticity of package by checking signature and source') + + if not isinstance(uri, CanonicalUri): + raise SotaError("Internal error: uri improperly passed to download function") + + source = uri.value[:-(len(uri.value.split('/')[-1]) + 1)] + file_name = os.path.basename(urlsplit(uri.value).path) + logger.debug(f"source: {source}, filename: {file_name}") + + verify_source(source=source, dispatcher_broker=dispatcher_broker) + dispatcher_broker.telemetry('Source Verification check passed') + + if password: + logger.debug("RS password provided.") + else: + err_msg = " No JWT token. Abort the update. " + raise SotaError(err_msg) + + # Specify the token in header. + headers = { + "Authorization": f"Bearer {password}" + } + + enough_space = is_enough_space_to_download(uri.value, repo, headers) + + if not enough_space: + err_msg = " Insufficient free space available on " + shlex.quote(repo.get_repo_path()) + \ + " for " + str(uri.value) + raise SotaError(err_msg) + + msg = f'Fetching software package from {uri.value}' + dispatcher_broker.telemetry(msg) + + if uri.value.startswith("http://"): + info_msg = "The file requested from repo is being downloaded over an insecure(non-TLS) session..." + logger.info(info_msg) + dispatcher_broker.telemetry(info_msg) + + try: + with requests.get(url=uri.value, headers=headers) as response: + repo.add(filename=file_name, contents=response.content) + except (HTTPError, OSError) as err: + raise SotaError(f'OTA Fetch Failed: {err}') + + dispatcher_broker.telemetry('OTA Download Successful') + + +def is_enough_space_to_download(manifest_uri: str, + destination_repo: IRepo, + headers: Any) -> bool: + """Checks if enough free space exists on platform to hold download. + + Calculates the file size from the OCI server and checks if required free space is available on + the platform. + @param manifest_uri: registry manifest uri + @param destination_repo: desired download destination + @param headers: headers that contains jwt_token to access the release server + """ + try: + logger.debug(f"Checking file size with manifest uri: {manifest_uri}") + + with requests.get(url=manifest_uri, headers=headers) as response: + response.raise_for_status() + # Read Content-Length header + try: + content_length = int(response.headers.get("Content-Length", "0")) + except ValueError: + content_length = 0 + + if content_length == 0: + # Stream file to measure the file size + for chunk in response.iter_content(chunk_size=16384): + if chunk: + content_length += len(chunk) + + except HTTPError as e: + if e.response: + status_code = e.response.status_code + else: + status_code = 0 + raise SotaError('Failed to access URI:' 'Status code for ' + manifest_uri + + ' is ' + str(status_code) + ". Invalid URI or Token might be expired.") + except (ProxyError, ChunkedEncodingError, ContentDecodingError, ConnectionError) as e: + raise SotaError(str(e)) + + logger.debug("File size: " + repr(content_length)) + file_size: int = int(content_length) + if destination_repo.exists(): + get_free_space = destination_repo.get_free_space() + free_space: int = int(get_free_space) + else: + raise SotaError("Repository does not exist : " + + shlex.quote(destination_repo.get_repo_path())) + + logger.debug("get_free_space: " + repr(get_free_space)) + logger.debug("Free space available on destination_repo is " + repr(free_space)) + logger.debug("Free space needed on destination repo is " + repr(file_size)) + return True if free_space > file_size else False + + +def read_release_server_token(token_path: str = RELEASE_SERVER_TOKEN_PATH) -> str: + """Read release server JWT token from a path configured by Tiber OS node-agent. The node agent will renew + the token when the token is expired. + + @return: JWT token to access release server + """ + token = None + try: + if os.path.exists(token_path): + with open(token_path, 'r') as f: + token = f.read().strip() + return token + else: + msg = f"{token_path} not exist." + except OSError as err: + raise SotaError(f"Error while performing TiberOS download: {err}") + + if token is None: + msg = f"No JWT token found." + + logger.error(msg) + raise SotaError(f"Error while performing TiberOS download: {msg}") diff --git a/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py b/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py index 4f41ca320..745469e4f 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py +++ b/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py @@ -8,8 +8,7 @@ import hashlib import logging from typing import Optional -from ..packagemanager.irepo import IRepo -from .constants import TIBER_UPDATE_TOOL_PATH, SOTA_CACHE +from .constants import TIBER_UPDATE_TOOL_PATH from inbm_common_lib.shell_runner import PseudoShellRunner from .sota_error import SotaError logger = logging.getLogger(__name__) @@ -24,24 +23,27 @@ def update_tool_rollback_command() -> None: raise SotaError(f"Failed to run UT rollback command. Error:{err}") -def update_tool_write_command(signature: Optional[str] = None, repo: Optional[IRepo] = None) -> str: +def update_tool_write_command(signature: Optional[str] = None, file_path: Optional[str] = None) -> str: """Call UT command to write the image into secondary partition. If signature is provided, it performs signature check and passes the verified file to UT. @param signature: signature used to verify image - @param repo: directory that contains the downloaded artifacts + @param file_path: raw image file path @return: UT command to run """ - raw_img_path = None - if signature: - raw_img_path = verify_signature(repo.get_repo_path(), signature) if repo \ - else verify_signature(SOTA_CACHE, signature) - logger.debug("") + if signature is None: + raise SotaError("Signature is None.") + + if file_path is None: + raise SotaError("Raw image file path is None.") + + if signature and file_path: + if verify_signature(file_path, signature): + logger.debug("Signature check passed.") + else: + raise SotaError("Signature checks failed.") - if raw_img_path: - return str(TIBER_UPDATE_TOOL_PATH + " -w" + " -u " + raw_img_path) - else: - raise SotaError("Signature checks failed. No matching file found.") + return str(TIBER_UPDATE_TOOL_PATH + " -w" + " -u " + file_path) def update_tool_commit_command() -> int: @@ -72,23 +74,23 @@ def update_tool_apply_command() -> str: return TIBER_UPDATE_TOOL_PATH + " -a" -def verify_signature(repo: str, signature: str) -> str: - """Perform signature check. Multiple files may have been downloaded from OCI. - The method below will iterate over all files in the repo, calculate the SHA256sum for each file, - and compare it with the provided signature. +def verify_signature(file_path: str, signature: str) -> bool: + """Perform signature check. The method will calculate the SHA256sum of the file and + compare it with the provided signature. - @return: File that matches the signature + @param signature: signature used to verify image + @param file_path: raw image file path + @return: True if the signature matches; False if the signature verification failed. """ try: logger.debug("Perform signature check on the downloaded file.") - for filename in os.listdir(repo): - filepath = os.path.join(repo, filename) - if os.path.isfile(filepath): - with open(filepath, 'rb') as file: - file_checksum = hashlib.sha256(file.read()).hexdigest() - if file_checksum == signature: - return filepath - - raise SotaError("Signature checks failed. No matching file found.") + with open(file_path, 'rb') as file: + file_checksum = hashlib.sha256(file.read()).hexdigest() + if file_checksum == signature: + return True + + logger.error("Signature checks failed.") + return False except OSError as err: - raise SotaError(err) + logger.error(f"Error during signature checks: {err}") + return False diff --git a/inbm/dispatcher-agent/tests/unit/sota/test_downloader.py b/inbm/dispatcher-agent/tests/unit/sota/test_downloader.py index 7c709cf3f..a5c4075b8 100644 --- a/inbm/dispatcher-agent/tests/unit/sota/test_downloader.py +++ b/inbm/dispatcher-agent/tests/unit/sota/test_downloader.py @@ -117,8 +117,8 @@ def _build_mock_repo(num_files=0): mem_repo.add("test" + str(i + 1) + ".rpm", b"0123456789") return mem_repo - @patch('dispatcher.sota.downloader.read_oras_token', return_value="mock_password") - @patch('dispatcher.sota.downloader.oras_download') + @patch('dispatcher.sota.downloader.read_release_server_token', return_value="mock_password") + @patch('dispatcher.sota.downloader.tiber_download') def test_tiberos_download_successful(self, mock_download, mock_read_token) -> None: self.release_date = self.username = None password = "mock_password" diff --git a/inbm/dispatcher-agent/tests/unit/sota/test_oras_util.py b/inbm/dispatcher-agent/tests/unit/sota/test_tiber_util.py similarity index 76% rename from inbm/dispatcher-agent/tests/unit/sota/test_oras_util.py rename to inbm/dispatcher-agent/tests/unit/sota/test_tiber_util.py index 7f3ef9b2b..d157048aa 100644 --- a/inbm/dispatcher-agent/tests/unit/sota/test_oras_util.py +++ b/inbm/dispatcher-agent/tests/unit/sota/test_tiber_util.py @@ -1,15 +1,17 @@ import unittest -from typing import Optional +import tempfile import os +import shutil from ..common.mock_resources import * from inbm_common_lib.utility import canonicalize_uri from dispatcher.dispatcher_exception import DispatcherException from dispatcher.packagemanager.memory_repo import MemoryRepo +from dispatcher.packagemanager.local_repo import DirectoryRepo from dispatcher.sota.os_factory import SotaOsFactory from dispatcher.sota.sota import SOTA from dispatcher.sota.sota_error import SotaError -from dispatcher.sota.oras_util import parse_uri +from dispatcher.sota.tiber_util import read_release_server_token from dispatcher.constants import CACHE from inbm_lib.xmlhandler import XmlHandler from unittest.mock import patch, MagicMock @@ -75,12 +77,10 @@ def setUp(cls) -> None: cls.sota_instance.factory = SotaOsFactory( MockDispatcherBroker.build_mock_dispatcher_broker(), None, []).get_os('tiber') - @patch("inbm_common_lib.shell_runner.PseudoShellRunner.run", return_value=('200', "", 0)) - @patch('json.loads', return_value=mock_resp) @patch('requests.get') - @patch('dispatcher.sota.downloader.read_oras_token', return_value="mock_password") - @patch('dispatcher.sota.oras_util.verify_source') - def test_download_successful(self, mock_verify_source, mock_read_token, mock_get, mock_loads, mock_run) -> None: + @patch('dispatcher.sota.downloader.read_release_server_token', return_value="mock_password") + @patch('dispatcher.sota.tiber_util.verify_source') + def test_download_successful(self, mock_verify_source, mock_read_token, mock_get) -> None: self.release_date = self.username = self.password = None mock_url = canonicalize_uri("https://registry-rs.internal.ledgepark.intel.com/one-intel-edge/tiberos:latest") mock_response = MagicMock() @@ -103,20 +103,7 @@ def test_download_successful(self, mock_verify_source, mock_read_token, mock_get mock_verify_source.assert_called_once() mock_read_token.assert_called_once() - mock_get.assert_called_once() - mock_loads.assert_called_once() - mock_run.assert_called_once() - - def test_parse_uri(self) -> None: - uri = canonicalize_uri("https://registry-rs.internal.ledgepark.intel.com/one-intel-edge/test/tiberos:latest") - parsed_uri = parse_uri(uri) - self.assertEqual(parsed_uri.source, 'https://registry-rs.internal.ledgepark.intel.com/one-intel-edge/test') - self.assertEqual(parsed_uri.registry_server, 'registry-rs.internal.ledgepark.intel.com') - self.assertEqual(parsed_uri.image, 'tiberos') - self.assertEqual(parsed_uri.image_tag, 'latest') - self.assertEqual(parsed_uri.repository_name, 'one-intel-edge/test') - self.assertEqual(parsed_uri.image_full_path, 'registry-rs.internal.ledgepark.intel.com/one-intel-edge/test/tiberos:latest') - self.assertEqual(parsed_uri.registry_manifest, 'https://registry-rs.internal.ledgepark.intel.com/v2/one-intel-edge/test/tiberos/manifests/latest') + assert mock_get.call_count == 2 @staticmethod def _build_mock_repo(num_files=0): @@ -124,4 +111,19 @@ def _build_mock_repo(num_files=0): if num_files != 0: for i in range(0, num_files): mem_repo.add("test" + str(i + 1) + ".raw.xz", b"0123456789") - return mem_repo \ No newline at end of file + return mem_repo + + def test_read_release_server_token_successful(self) -> None: + directory = tempfile.mkdtemp() + try: + repo = DirectoryRepo(directory) + repo.add("rs_access_token", b"mock_token123") + token = read_release_server_token(token_path=os.path.join(directory, "rs_access_token")) + finally: + shutil.rmtree(directory) + + self.assertEqual(token, "mock_token123") + + def test_read_release_server_token_failed_with_fake_path(self) -> None: + with self.assertRaises(SotaError): + read_release_server_token(token_path="/fake/path/rs_access_token") diff --git a/inbm/dispatcher-agent/tests/unit/sota/test_update_tool_util.py b/inbm/dispatcher-agent/tests/unit/sota/test_update_tool_util.py index e2ab2dcc7..10c003a7a 100644 --- a/inbm/dispatcher-agent/tests/unit/sota/test_update_tool_util.py +++ b/inbm/dispatcher-agent/tests/unit/sota/test_update_tool_util.py @@ -20,14 +20,14 @@ def test_update_tool_write_command_return_file_path(self) -> None: # Calculate the SHA256 checksum sha256_hash = hashlib.sha256() - with open(os.path.join(directory, "test"), 'rb') as file: + file_path = os.path.join(directory, "test") + with open(file_path, 'rb') as file: for chunk in iter(lambda: file.read(4096), b''): sha256_hash.update(chunk) checksum = sha256_hash.hexdigest() - expected_cmd = f'{TIBER_UPDATE_TOOL_PATH} -w -u {os.path.join(repo.get_repo_path(), "test")}' - cmd = update_tool_write_command(signature=checksum, repo=repo) + cmd = update_tool_write_command(signature=checksum, file_path=file_path) self.assertEqual(cmd, expected_cmd) finally: shutil.rmtree(directory) From 9d8049cefad80ec5eb472cae2ec38440b938b003 Mon Sep 17 00:00:00 2001 From: yengliong Date: Tue, 15 Oct 2024 14:44:07 +0800 Subject: [PATCH 2/2] Address comments --- .../dispatcher/sota/downloader.py | 2 +- .../dispatcher/sota/os_updater.py | 11 +++++---- .../dispatcher/sota/tiber_util.py | 12 +++++----- .../dispatcher/sota/update_tool_util.py | 24 ++++++++++--------- 4 files changed, 26 insertions(+), 23 deletions(-) diff --git a/inbm/dispatcher-agent/dispatcher/sota/downloader.py b/inbm/dispatcher-agent/dispatcher/sota/downloader.py index 1145fba01..aa6278447 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/downloader.py +++ b/inbm/dispatcher-agent/dispatcher/sota/downloader.py @@ -202,7 +202,7 @@ def download(self, repo=repo, umask=UMASK_OTA, username=username, - password=password) + token=password) def check_release_date(self, release_date: Optional[str]) -> bool: raise NotImplementedError() \ No newline at end of file diff --git a/inbm/dispatcher-agent/dispatcher/sota/os_updater.py b/inbm/dispatcher-agent/dispatcher/sota/os_updater.py index d2cf44e3a..9566c3ffa 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/os_updater.py +++ b/inbm/dispatcher-agent/dispatcher/sota/os_updater.py @@ -6,18 +6,16 @@ SPDX-License-Identifier: Apache-2.0 """ -import abc import logging import re import os from pathlib import Path from typing import List, Optional, Union -from abc import ABC, ABCMeta, abstractmethod - +from abc import ABCMeta, abstractmethod +from urllib.parse import urlparse from inbm_common_lib.utility import CanonicalUri from inbm_common_lib.shell_runner import PseudoShellRunner from inbm_lib.constants import DOCKER_CHROOT_PREFIX, CHROOT_PREFIX -from inbm_common_lib.utility import get_canonical_representation_of_path from .command_list import CommandList from .constants import MENDER_FILE_PATH, SOTA_CACHE @@ -435,7 +433,10 @@ def download_only(self) -> list[str]: # Extract the file path from uri. file_path = None if self._uri: - file_path = os.path.join(SOTA_CACHE, self._uri.split('/')[-1]) + parsed_uri = urlparse(self._uri) + filename = os.path.basename(parsed_uri.path) + if filename: + file_path = os.path.join(SOTA_CACHE, filename) cmds = [update_tool_write_command(self._signature, file_path)] return CommandList(cmds).cmd_list \ No newline at end of file diff --git a/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py b/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py index f2f77ded3..83327de7c 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py +++ b/inbm/dispatcher-agent/dispatcher/sota/tiber_util.py @@ -1,5 +1,5 @@ """ - ORAS tool will be called by dispatcher to perform the image downloading in TiberOS. + Tiber Util module will be called by dispatcher to perform the image downloading in TiberOS. Copyright (C) 2017-2024 Intel Corporation SPDX-License-Identifier: Apache-2.0 @@ -24,14 +24,14 @@ def tiber_download(dispatcher_broker: DispatcherBroker, uri: CanonicalUri, - repo: IRepo, username: Optional[str], password: str, umask: int) -> None: + repo: IRepo, username: Optional[str], token: str, umask: int) -> None: """Downloads files and places capsule file in path mentioned by manifest file. @param dispatcher_broker: DispatcherBroker object used to communicate with other INBM services @param uri: URI of the source location @param repo: repository for holding the download @param username: username to use for download - @param password: password to use for download + @param token: token to use for download @param umask: file permission mask @raises SotaError: any exception """ @@ -49,15 +49,15 @@ def tiber_download(dispatcher_broker: DispatcherBroker, uri: CanonicalUri, verify_source(source=source, dispatcher_broker=dispatcher_broker) dispatcher_broker.telemetry('Source Verification check passed') - if password: - logger.debug("RS password provided.") + if token: + logger.debug("RS token provided.") else: err_msg = " No JWT token. Abort the update. " raise SotaError(err_msg) # Specify the token in header. headers = { - "Authorization": f"Bearer {password}" + "Authorization": f"Bearer {token}" } enough_space = is_enough_space_to_download(uri.value, repo, headers) diff --git a/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py b/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py index 745469e4f..1fb3be09d 100644 --- a/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py +++ b/inbm/dispatcher-agent/dispatcher/sota/update_tool_util.py @@ -26,6 +26,8 @@ def update_tool_rollback_command() -> None: def update_tool_write_command(signature: Optional[str] = None, file_path: Optional[str] = None) -> str: """Call UT command to write the image into secondary partition. If signature is provided, it performs signature check and passes the verified file to UT. + TODO: For now the signature matches to hash received from MM or PUA. Please update the method once the + integration of the public key into TiberOS is confirmed. @param signature: signature used to verify image @param file_path: raw image file path @@ -38,8 +40,8 @@ def update_tool_write_command(signature: Optional[str] = None, file_path: Option raise SotaError("Raw image file path is None.") if signature and file_path: - if verify_signature(file_path, signature): - logger.debug("Signature check passed.") + if verify_hash(file_path, signature): + logger.debug("Signature checks passed.") else: raise SotaError("Signature checks failed.") @@ -74,23 +76,23 @@ def update_tool_apply_command() -> str: return TIBER_UPDATE_TOOL_PATH + " -a" -def verify_signature(file_path: str, signature: str) -> bool: - """Perform signature check. The method will calculate the SHA256sum of the file and - compare it with the provided signature. +def verify_hash(file_path: str, hash: str) -> bool: + """Perform hash verification checks. The method will calculate the SHA256sum of the file and + compare it with the provided hash. - @param signature: signature used to verify image + @param hash: checksum used to verify image @param file_path: raw image file path - @return: True if the signature matches; False if the signature verification failed. + @return: True if the hash matches; False if the hash verification failed. """ try: - logger.debug("Perform signature check on the downloaded file.") + logger.debug("Perform hash verification checks on the downloaded file.") with open(file_path, 'rb') as file: file_checksum = hashlib.sha256(file.read()).hexdigest() - if file_checksum == signature: + if file_checksum == hash: return True - logger.error("Signature checks failed.") + logger.error("Hash verification checks failed.") return False except OSError as err: - logger.error(f"Error during signature checks: {err}") + logger.error(f"Error during hash verification checks: {err}") return False