From 53f020cafe123dcdb7021c1a3e9f9454c49b75d7 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Wed, 25 Jan 2023 19:00:07 +0100 Subject: [PATCH 1/9] Parametrizing GDK CLI to accept different recipies and configuration files --- gdk/_version.py | 2 +- gdk/commands/component/BuildCommand.py | 31 ++++++-- gdk/commands/component/PublishCommand.py | 91 ++++++++++++++++++++++-- gdk/commands/component/project_utils.py | 62 +++++++++++----- gdk/common/configuration.py | 10 +-- gdk/common/consts.py | 2 +- gdk/common/exceptions/error_messages.py | 2 +- gdk/common/utils.py | 2 - gdk/static/cli_model.json | 52 ++++++++++++++ 9 files changed, 218 insertions(+), 36 deletions(-) diff --git a/gdk/_version.py b/gdk/_version.py index 6849410a..a82b376d 100644 --- a/gdk/_version.py +++ b/gdk/_version.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.1.1" diff --git a/gdk/commands/component/BuildCommand.py b/gdk/commands/component/BuildCommand.py index fe36b3e3..302d9482 100644 --- a/gdk/commands/component/BuildCommand.py +++ b/gdk/commands/component/BuildCommand.py @@ -1,5 +1,6 @@ import json import logging +import os import platform import shutil import subprocess as sp @@ -16,7 +17,18 @@ class BuildCommand(Command): def __init__(self, command_args) -> None: super().__init__(command_args, "build") - self.project_config = project_utils.get_project_config_values() + + logging.debug("Command '{}' arguments: {}".format(self.name, command_args)) + + project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file + project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir) + project_recipe_filename = command_args["recipe"] if command_args["recipe"] != None else None + + logging.debug("Project config filename: {}".format(project_config_filename)) + logging.debug("Project build directory: {}".format(project_build_directory)) + logging.debug("Project recipe filename: {}".format(project_recipe_filename)) + + self.project_config = project_utils.get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, None, None) self.supported_build_sytems = project_utils.get_supported_component_builds() def run(self): @@ -74,7 +86,7 @@ def create_gg_build_directories(self): # Clean build directory if it exists already. utils.clean_dir(self.project_config["gg_build_directory"]) - logging.debug("Creating '{}' directory with artifacts and recipes.".format(consts.greengrass_build_dir)) + logging.debug("Creating '{}' directory with artifacts and recipes.".format(self.project_config["gg_build_directory"])) # Create build artifacts and recipe directories Path.mkdir(self.project_config["gg_build_recipes_dir"], parents=True, exist_ok=True) Path.mkdir(self.project_config["gg_build_component_artifacts_dir"], parents=True, exist_ok=True) @@ -143,11 +155,12 @@ def run_build_command(self): None """ try: + project_config_filename = self.project_config["project_config_filename"] build_system = self.project_config["component_build_config"]["build_system"] build_command = self.get_build_cmd_from_platform(build_system) logging.warning( f"This component is identified as using '{build_system}' build system. If this is incorrect, please exit and" - f" specify custom build command in the '{consts.cli_project_config_file}'." + f" specify custom build command in the '{project_config_filename}'." ) if build_system == "zip": logging.info("Zipping source code files of the component.") @@ -222,6 +235,9 @@ def _ignore_files_during_zip(self, path, names): consts.cli_project_config_file, consts.greengrass_build_dir, self.project_config["component_recipe_file"].name, + self.project_config["project_config_filename"], + self.project_recipe_filename, + self.project_config["gg_build_directory"].name, "test*", ".*", "node_modules", @@ -408,7 +424,14 @@ def create_build_recipe_file(self): ) ) parsed_component_recipe = self.project_config["parsed_component_recipe"] - component_recipe_file_name = self.project_config["component_recipe_file"].name + + # Standardize output recipe filename in case of custom input + logging.debug("File name: {}".format(self.project_config["component_recipe_file"].name)) + file_ext = os.path.splitext(self.project_config["component_recipe_file"].name) + logging.debug("File split: {}".format(file_ext)) + component_recipe_file_name = "recipe{}".format(file_ext[1]) + logging.debug("Final standardized recipe filename based on extention: {}".format(component_recipe_file_name)) + parsed_component_recipe["ComponentName"] = self.project_config["component_name"] parsed_component_recipe["ComponentVersion"] = self.project_config["component_version"] parsed_component_recipe["ComponentPublisher"] = self.project_config["component_author"] diff --git a/gdk/commands/component/PublishCommand.py b/gdk/commands/component/PublishCommand.py index 9712a4c1..677a0d49 100644 --- a/gdk/commands/component/PublishCommand.py +++ b/gdk/commands/component/PublishCommand.py @@ -4,21 +4,101 @@ import gdk.commands.component.component as component import gdk.commands.component.project_utils as project_utils +from gdk.common import consts import gdk.common.exceptions.error_messages as error_messages import gdk.common.utils as utils import yaml from botocore.exceptions import ClientError from gdk.commands.Command import Command +import shutil +import os class PublishCommand(Command): def __init__(self, command_args) -> None: super().__init__(command_args, "publish") - self.project_config = project_utils.get_project_config_values() - self.service_clients = project_utils.get_service_clients(self.project_config["region"]) + local = command_args["local"] if command_args["local"] != None else False + project_publish_directory = command_args["publish_dir"] + project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file + project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else consts.greengrass_build_dir + project_recipe_filename = "{}/recipes/recipe.json".format(project_build_directory) + + logging.debug("Local deployment : {}".format(local)) + logging.debug("Project publish directory : {}".format(project_config_filename)) + logging.debug("Project config filename : {}".format(project_config_filename)) + logging.debug("Project build directory : {}".format(project_build_directory)) + logging.debug("Project recipe filename : {}".format(project_recipe_filename)) + + self.project_config = project_utils.get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, project_publish_directory, local) + + logging.debug("Project configuration: {}".format(self.project_config)) def run(self): + if self.project_config["gg_local_deployment"]: + self.run_local() + else: + self.run_remote() + + def run_local(self): + self.project_config["account_number"] = "123456789012" + + component_name = self.project_config["component_name"] + component_version = self.get_component_version_from_config() + + logging.info("Publishing artifact '{}' with version '{}' locally".format(component_name, component_version)) + + build_directory = Path(self.project_config["gg_build_directory"]) + publish_directory = Path(self.project_config["gg_publish_directory"]) + build_recipes_directory = Path(self.project_config["gg_build_recipes_dir"]) + build_artifacts_directory = Path(self.project_config["gg_build_artifacts_dir"]) + publish_recipes_directory = Path(self.project_config["gg_publish_recipes_dir"]) + publish_artifacts_directory = Path(self.project_config["gg_publish_artifacts_dir"]) + + if build_directory.resolve() == publish_directory.resolve(): + logging.info("Local publish directory not provided or matching the build one, nothing to do.") + return + + if self.project_config["gg_publish_directory"] is None or not publish_directory.is_dir(): + raise RuntimeError("Invalid local component publish directory '{}'. Does it exist?".format(publish_directory.resolve())) + + publish_recipe_file_name = self.update_and_create_recipe_file(self.project_config["component_name"], self.project_config["component_version"]) + + logging.debug("Recipe file to be published: {}".format(publish_recipe_file_name)) + + publish_recipe_file = Path(build_recipes_directory).joinpath(publish_recipe_file_name) + + # Copy recipes + + logging.debug("Cleanup destination directory '{}' before copying new recipes.".format(publish_recipes_directory.resolve())) + shutil.rmtree(publish_recipes_directory.resolve()) + publish_recipes_directory.mkdir(parents=True, exist_ok=True) + + destination_receipe_file = Path(publish_recipes_directory.resolve()).joinpath(publish_recipe_file.name) + logging.debug("Copying recipes file '{}' into '{}'".format(publish_recipe_file, destination_receipe_file)) + + if destination_receipe_file.exists(): + os.remove(destination_receipe_file.resolve()) + + shutil.copy(publish_recipe_file.resolve(), destination_receipe_file.resolve()) + + # Copyt artifacts + + logging.debug("Cleanup destination directory '{}' before copying new artifacts.".format(publish_artifacts_directory.resolve())) + shutil.rmtree(publish_artifacts_directory.resolve()) + publish_artifacts_directory.mkdir(parents=True, exist_ok=True) + + logging.debug("Copy artifact dir '{}' into '{}'".format(build_artifacts_directory.resolve(), publish_artifacts_directory.resolve())) + + shutil.copytree(build_artifacts_directory.resolve(), publish_artifacts_directory.resolve(), dirs_exist_ok=True) + + logging.info("Component '{}' with version '{}' published locally.".format(component_name, component_version)) + + def run_remote(self): + logging.info("Publishing artifact remotely") + + self.service_clients = project_utils.get_service_clients(self.project_config["region"]) + try: self.project_config["account_number"] = self.get_account_number() if self.arguments["bucket"]: @@ -35,7 +115,7 @@ def run(self): logging.warning( f"The component '{component_name}' is not built.\nSo, building the component before publishing it." ) - component.build({}) + component.build({}) # FIXME: review according to the provided recipe file as input logging.info(f"Publishing the component '{component_name}' with the given project configuration.") logging.info("Uploading the component built artifacts to s3 bucket.") self.upload_artifacts_s3(component_name, component_version) @@ -346,7 +426,7 @@ def update_and_create_recipe_file(self, component_name, component_version): # Update the version of the component in the recipe parsed_component_recipe["ComponentVersion"] = component_version - self.create_publish_recipe_file(component_name, component_version, parsed_component_recipe) + return self.create_publish_recipe_file(component_name, component_version, parsed_component_recipe) def create_publish_recipe_file(self, component_name, component_version, parsed_component_recipe): """ @@ -375,9 +455,12 @@ def create_publish_recipe_file(self, component_name, component_version, parsed_c publish_recipe_file_name, self.project_config["gg_build_recipes_dir"] ) ) + if publish_recipe_file_name.endswith(".json"): prf.write(json.dumps(parsed_component_recipe, indent=4)) else: yaml.dump(parsed_component_recipe, prf) + + return publish_recipe_file_name except Exception as e: raise Exception("""Failed to create publish recipe file at '{}'.\n{}""".format(publish_recipe_file, e)) diff --git a/gdk/commands/component/project_utils.py b/gdk/commands/component/project_utils.py index e014ebaf..1d7b6bdb 100644 --- a/gdk/commands/component/project_utils.py +++ b/gdk/commands/component/project_utils.py @@ -30,7 +30,7 @@ def get_supported_component_builds(): return None -def get_recipe_file(): +def get_recipe_file(project_recipe_filename): """ Finds recipe file based on component name and its extension. @@ -48,21 +48,30 @@ def get_recipe_file(): ------- recipe_file(Path): Path of the identified recipe file. """ - # Search for json files in current directory that contain component name and ends in .json. - logging.debug("Looking for recipe file in the project directory.") - json_file = list(Path(utils.current_directory).glob("recipe.json")) - yaml_file = list(Path(utils.current_directory).glob("recipe.yaml")) + + if project_recipe_filename is None: + # In case of missing recipe filename as argument, let's look for default ones + logging.debug("Looking for default recipe file name (recipe.json or recipe.yaml)") - if not json_file and not yaml_file: - logging.error("Could not find 'recipe.json' or 'recipe.yaml' in the project directory.") - raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + json_file = list(Path(utils.current_directory).glob("recipe.json")) + yaml_file = list(Path(utils.current_directory).glob("recipe.yaml")) - if json_file and yaml_file: - logging.error("Found both 'recipe.json' and 'recipe.yaml' in the given project directory.") - raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + if not json_file and not yaml_file: + logging.error("Could not find 'recipe.json' or 'recipe.yaml' in the project directory.") + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + + if json_file and yaml_file: + logging.error("Found both 'recipe.json' and 'recipe.yaml' in the given project directory.") + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + + recipe_file = (json_file + yaml_file)[0].resolve() + else: + # Try to use the provided recipe file name + logging.debug("Using provided recipe file '{}'".format(project_recipe_filename)) + recipe_file = Path(project_recipe_filename).resolve() + + logging.info("Found component recipe file '{}'".format(recipe_file.absolute())) - recipe_file = (json_file + yaml_file)[0].resolve() - logging.info("Found component recipe file '{}' in the project directory.".format(recipe_file.name)) return recipe_file @@ -94,11 +103,11 @@ def parse_recipe_file(component_recipe_file): raise Exception("""Unable to parse the recipe file - {}.\n{}""".format(component_recipe_file.name, e)) -def get_project_config_values(): +def get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, project_publish_directory, local_deployment): # Get component configuration from the greengrass project config file. - logging.info("Getting project configuration from {}".format(consts.cli_project_config_file)) - project_config = config_actions.get_configuration()["component"] + logging.info("Getting project configuration from {}".format(project_config_filename)) + project_config = config_actions.get_configuration(project_config_filename)["component"] # Since there's only one key in the component configuration, use next() instead of looping in. component_name = next(iter(project_config)) @@ -110,19 +119,31 @@ def get_project_config_values(): region = component_config["publish"]["region"] # Build directories - gg_build_directory = Path(utils.current_directory).joinpath(consts.greengrass_build_dir).resolve() + logging.info("Build directory: '{}'".format(project_build_directory)) + gg_build_directory = Path(project_build_directory).resolve() gg_build_artifacts_dir = Path(gg_build_directory).joinpath("artifacts").resolve() gg_build_recipes_dir = Path(gg_build_directory).joinpath("recipes").resolve() gg_build_component_artifacts_dir = Path(gg_build_artifacts_dir).joinpath(component_name, component_version).resolve() + # Publish directories + gg_publish_directory = Path(project_publish_directory if project_publish_directory != None else gg_build_directory).resolve() + logging.info("Publish directory: '{}'".format(gg_publish_directory)) + gg_publish_artifacts_dir = Path(gg_publish_directory).joinpath("artifacts").resolve() + gg_publish_recipes_dir = Path(gg_publish_directory).joinpath("recipes").resolve() + gg_publish_component_artifacts_dir = Path(gg_publish_artifacts_dir).joinpath(component_name, component_version).resolve() + + # Local or remote deployment + gg_local_deployment = local_deployment + # Get recipe file - component_recipe_file = get_recipe_file() + component_recipe_file = get_recipe_file(project_recipe_filename) # Get parsed recipe file parsed_component_recipe = parse_recipe_file(component_recipe_file) # Create dictionary with all the above values vars = {} + vars["project_config_filename"] = project_config_filename vars["component_name"] = component_name vars["component_version"] = component_version vars["component_author"] = component_author @@ -133,6 +154,11 @@ def get_project_config_values(): vars["gg_build_artifacts_dir"] = gg_build_artifacts_dir vars["gg_build_recipes_dir"] = gg_build_recipes_dir vars["gg_build_component_artifacts_dir"] = gg_build_component_artifacts_dir + vars["gg_publish_directory"] = gg_publish_directory + vars["gg_publish_artifacts_dir"] = gg_publish_artifacts_dir + vars["gg_publish_recipes_dir"] = gg_publish_recipes_dir + vars["gg_publish_component_artifacts_dir"] = gg_publish_component_artifacts_dir + vars["gg_local_deployment"] = gg_local_deployment vars["component_recipe_file"] = component_recipe_file vars["parsed_component_recipe"] = parsed_component_recipe return vars diff --git a/gdk/common/configuration.py b/gdk/common/configuration.py index 394654b5..4b58c479 100644 --- a/gdk/common/configuration.py +++ b/gdk/common/configuration.py @@ -9,7 +9,7 @@ from packaging.version import Version -def get_configuration(): +def get_configuration(project_config_filename): """ Loads the configuration from the greengrass project config file as a json object. @@ -23,7 +23,7 @@ def get_configuration(): ------- config_data(dict): Greengrass project configuration as a dictionary object if the config is valid. """ - project_config_file = _get_project_config_file() + project_config_file = _get_project_config_file(project_config_filename) with open(project_config_file, "r") as config_file: config_data = json.loads(config_file.read()) try: @@ -72,7 +72,7 @@ def validate_cli_version(config_data): ) -def _get_project_config_file(): +def _get_project_config_file(gdk_config_file_name): """ Returns path of the config file present in the greengrass project directory. @@ -88,7 +88,7 @@ def _get_project_config_file(): ------- config_file(pathlib.Path): Path of the config file. """ - config_file = Path(utils.current_directory).joinpath(consts.cli_project_config_file).resolve() + config_file = Path(utils.current_directory).joinpath(gdk_config_file_name).resolve() if not utils.file_exists(config_file): - raise Exception(error_messages.CONFIG_FILE_NOT_EXISTS) + raise Exception(error_messages.CONFIG_FILE_NOT_EXISTS.format(gdk_config_file_name)) return config_file diff --git a/gdk/common/consts.py b/gdk/common/consts.py index 341fbc01..0e55bdc0 100644 --- a/gdk/common/consts.py +++ b/gdk/common/consts.py @@ -32,4 +32,4 @@ # DEFAULT LOGGING log_format = "[%(asctime)s] %(levelname)s - %(message)s" -date_format = "%Y-%m-%d %H:%M:%S" +date_format = "%Y-%m-%d %H:%M:%S" \ No newline at end of file diff --git a/gdk/common/exceptions/error_messages.py b/gdk/common/exceptions/error_messages.py index 832b2186..cca777f8 100644 --- a/gdk/common/exceptions/error_messages.py +++ b/gdk/common/exceptions/error_messages.py @@ -1,6 +1,6 @@ # FILE CONFIG_FILE_NOT_EXISTS = ( - "Config file doesn't exist. Please initialize the project using a template or a repository before using gdk commands." + "Config file '{}' doesn't exist. Please initialize the project using a template or a repository before using gdk commands." ) CONFIG_SCHEMA_FILE_NOT_EXISTS = "Configuration validation failed. Config schema file doesn't exist." PROJECT_RECIPE_FILE_NOT_FOUND = ( diff --git a/gdk/common/utils.py b/gdk/common/utils.py index 3d330d2d..2169576c 100644 --- a/gdk/common/utils.py +++ b/gdk/common/utils.py @@ -7,7 +7,6 @@ import requests from packaging.version import Version - def get_static_file_path(file_name): """ Returns the path of the file assuming that is in static directory. @@ -129,7 +128,6 @@ def cli_version_check(): f" `{update_command}`.\n" ) - error_line = "\n=============================== ERROR ===============================\n" help_line = "\n=============================== HELP ===============================\n" current_directory = Path(".").resolve() diff --git a/gdk/static/cli_model.json b/gdk/static/cli_model.json index 0972dd50..5856dd63 100644 --- a/gdk/static/cli_model.json +++ b/gdk/static/cli_model.json @@ -79,6 +79,29 @@ "help": "Initialize the project with a component template or repository from Greengrass Software Catalog." }, "build": { + "arguments": { + "gdk-config": { + "name": [ + "-c", + "--gdk-config" + ], + "help": "Specify alternative GDK configuration file." + }, + "recipe": { + "name": [ + "-r", + "--recipe" + ], + "help": "Specify alternative recipe file." + }, + "build-dir": { + "name": [ + "-u", + "--build-dir" + ], + "help": "Specify alternative component build directory." + } + }, "help": "Build GreengrassV2 component artifacts and recipes from its source code." }, "publish": { @@ -90,6 +113,35 @@ "--bucket" ], "help": "Name of the s3 bucket to use for uploading component artifacts." + }, + "gdk-config": { + "name": [ + "-c", + "--gdk-config" + ], + "help": "Specify alternative GDK configuration file." + }, + "build-dir": { + "name": [ + "-u", + "--build-dir" + ], + "help": "Specify alternative component build directory." + }, + "publish-dir": { + "name": [ + "-p", + "--publish-dir" + ], + "help": "Specify local component publish directory." + }, + "local": { + "name": [ + "-l", + "--local" + ], + "action": "store_true", + "help": "Publish component locally to build directory." } } }, From 29c6419445ce7b0f585ec569bc6f0fa50092bb8f Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Wed, 15 Feb 2023 16:49:56 +0100 Subject: [PATCH 2/9] Increasing minor version: 1.1.0 -> 1.2.0 --- gdk/_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gdk/_version.py b/gdk/_version.py index a82b376d..c68196d1 100644 --- a/gdk/_version.py +++ b/gdk/_version.py @@ -1 +1 @@ -__version__ = "1.1.1" +__version__ = "1.2.0" From 5d7fcec26a7a9bd9af0e93fe103cce049486e1a8 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Wed, 15 Feb 2023 16:54:11 +0100 Subject: [PATCH 3/9] Adding new parameters for build and publish, adding new command push --- gdk/commands/component/BuildCommand.py | 14 +- gdk/commands/component/PublishCommand.py | 198 +++-------------------- gdk/static/cli_model.json | 47 ++++-- 3 files changed, 75 insertions(+), 184 deletions(-) diff --git a/gdk/commands/component/BuildCommand.py b/gdk/commands/component/BuildCommand.py index 302d9482..84221f91 100644 --- a/gdk/commands/component/BuildCommand.py +++ b/gdk/commands/component/BuildCommand.py @@ -22,13 +22,23 @@ def __init__(self, command_args) -> None: project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir) - project_recipe_filename = command_args["recipe"] if command_args["recipe"] != None else None + project_recipe_filename = command_args["recipe"] if command_args["recipe"] != None else project_utils.find_recipe_file_in_path("recipe", utils.current_directory) logging.debug("Project config filename: {}".format(project_config_filename)) logging.debug("Project build directory: {}".format(project_build_directory)) logging.debug("Project recipe filename: {}".format(project_recipe_filename)) - self.project_config = project_utils.get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, None, None) + logging.debug("Loading project configuration values") + self.project_config = project_utils.get_project_config_values(project_config_filename, project_build_directory) + + logging.debug("Loading recipe values") + recipe_values = project_utils.get_project_recipe_values(project_recipe_filename) + + logging.debug("Merging recipe into project configuration value") + self.project_config.update(recipe_values) + + logging.debug("Project configuration: {}".format(self.project_config)) + self.supported_build_sytems = project_utils.get_supported_component_builds() def run(self): diff --git a/gdk/commands/component/PublishCommand.py b/gdk/commands/component/PublishCommand.py index 677a0d49..57e427b8 100644 --- a/gdk/commands/component/PublishCommand.py +++ b/gdk/commands/component/PublishCommand.py @@ -4,101 +4,52 @@ import gdk.commands.component.component as component import gdk.commands.component.project_utils as project_utils -from gdk.common import consts import gdk.common.exceptions.error_messages as error_messages import gdk.common.utils as utils import yaml from botocore.exceptions import ClientError from gdk.commands.Command import Command -import shutil -import os +from gdk.common import consts class PublishCommand(Command): def __init__(self, command_args) -> None: super().__init__(command_args, "publish") + logging.debug("Arguments: {}".format(command_args)) - local = command_args["local"] if command_args["local"] != None else False - project_publish_directory = command_args["publish_dir"] project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file - project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else consts.greengrass_build_dir - project_recipe_filename = "{}/recipes/recipe.json".format(project_build_directory) + project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir) - logging.debug("Local deployment : {}".format(local)) - logging.debug("Project publish directory : {}".format(project_config_filename)) - logging.debug("Project config filename : {}".format(project_config_filename)) - logging.debug("Project build directory : {}".format(project_build_directory)) - logging.debug("Project recipe filename : {}".format(project_recipe_filename)) - - self.project_config = project_utils.get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, project_publish_directory, local) + component_name = command_args["component_name"] + component_version = command_args["component_version"] - logging.debug("Project configuration: {}".format(self.project_config)) + logging.debug("Loading project configuration values") + self.project_config = project_utils.get_project_config_values(project_config_filename, project_build_directory, component_name, component_version) - def run(self): - if self.project_config["gg_local_deployment"]: - self.run_local() + if component_name is not None and component_version is not None: + self.project_config["component_name"] = component_name + self.project_config["component_version"] = component_version + project_recipe_filename = project_utils.find_recipe_file_in_path("{}-{}".format(component_name, component_version), "{}/recipes".format(project_build_directory)) + elif component_name is None and component_version is None: + project_recipe_filename = project_utils.find_recipe_file_in_path("recipe", "{}/recipes".format(project_build_directory)) else: - self.run_remote() - - def run_local(self): - self.project_config["account_number"] = "123456789012" - - component_name = self.project_config["component_name"] - component_version = self.get_component_version_from_config() - - logging.info("Publishing artifact '{}' with version '{}' locally".format(component_name, component_version)) - - build_directory = Path(self.project_config["gg_build_directory"]) - publish_directory = Path(self.project_config["gg_publish_directory"]) - build_recipes_directory = Path(self.project_config["gg_build_recipes_dir"]) - build_artifacts_directory = Path(self.project_config["gg_build_artifacts_dir"]) - publish_recipes_directory = Path(self.project_config["gg_publish_recipes_dir"]) - publish_artifacts_directory = Path(self.project_config["gg_publish_artifacts_dir"]) - - if build_directory.resolve() == publish_directory.resolve(): - logging.info("Local publish directory not provided or matching the build one, nothing to do.") - return - - if self.project_config["gg_publish_directory"] is None or not publish_directory.is_dir(): - raise RuntimeError("Invalid local component publish directory '{}'. Does it exist?".format(publish_directory.resolve())) - - publish_recipe_file_name = self.update_and_create_recipe_file(self.project_config["component_name"], self.project_config["component_version"]) + raise RuntimeError("Component name and version must be provided together") - logging.debug("Recipe file to be published: {}".format(publish_recipe_file_name)) - - publish_recipe_file = Path(build_recipes_directory).joinpath(publish_recipe_file_name) - - # Copy recipes - - logging.debug("Cleanup destination directory '{}' before copying new recipes.".format(publish_recipes_directory.resolve())) - shutil.rmtree(publish_recipes_directory.resolve()) - publish_recipes_directory.mkdir(parents=True, exist_ok=True) - - destination_receipe_file = Path(publish_recipes_directory.resolve()).joinpath(publish_recipe_file.name) - logging.debug("Copying recipes file '{}' into '{}'".format(publish_recipe_file, destination_receipe_file)) - - if destination_receipe_file.exists(): - os.remove(destination_receipe_file.resolve()) - - shutil.copy(publish_recipe_file.resolve(), destination_receipe_file.resolve()) - - # Copyt artifacts - - logging.debug("Cleanup destination directory '{}' before copying new artifacts.".format(publish_artifacts_directory.resolve())) - shutil.rmtree(publish_artifacts_directory.resolve()) - publish_artifacts_directory.mkdir(parents=True, exist_ok=True) + logging.debug("Project config filename : {}".format(project_config_filename)) + logging.debug("Project build directory : {}".format(project_build_directory)) + logging.debug("Project recipe filename : {}".format(project_recipe_filename)) - logging.debug("Copy artifact dir '{}' into '{}'".format(build_artifacts_directory.resolve(), publish_artifacts_directory.resolve())) - - shutil.copytree(build_artifacts_directory.resolve(), publish_artifacts_directory.resolve(), dirs_exist_ok=True) + logging.debug("Loading recipe values") + recipe_values = project_utils.get_project_recipe_values(project_recipe_filename) - logging.info("Component '{}' with version '{}' published locally.".format(component_name, component_version)) + logging.debug("Merging recipe into project configuration value") + self.project_config.update(recipe_values) - def run_remote(self): - logging.info("Publishing artifact remotely") + logging.debug("Project configuration: {}".format(self.project_config)) self.service_clients = project_utils.get_service_clients(self.project_config["region"]) + def run(self): try: self.project_config["account_number"] = self.get_account_number() if self.arguments["bucket"]: @@ -115,13 +66,13 @@ def run_remote(self): logging.warning( f"The component '{component_name}' is not built.\nSo, building the component before publishing it." ) - component.build({}) # FIXME: review according to the provided recipe file as input + component.build({}) logging.info(f"Publishing the component '{component_name}' with the given project configuration.") logging.info("Uploading the component built artifacts to s3 bucket.") self.upload_artifacts_s3(component_name, component_version) logging.info(f"Updating the component recipe {component_name}-{component_version}.") - self.update_and_create_recipe_file(component_name, component_version) + project_utils.update_and_create_recipe_file(self.project_config, component_name, component_version) logging.info(f"Creating a new greengrass component {component_name}-{component_version}") self.create_gg_component(component_name, component_version) @@ -366,101 +317,4 @@ def get_component_version_from_config(self): logging.error( "Failed to calculate the version of component '{}' based on the configuration.".format(component_name) ) - raise (e) - - def update_and_create_recipe_file(self, component_name, component_version): - """ - Updates recipe with the component version calculated and artifact URIs of the artifacts. This updated recipe is - used to create a new publish recipe file in build recipes directory. - - Parameters - ---------- - component_name(string): Name of the component. This is also used in the name of the recipe file. - component_version(string): Version of the component calculated based on the configuration. - - Returns - ------- - None - """ - logging.debug("Updating artifact URIs in the recipe...") - build_recipe = Path(self.project_config["gg_build_recipes_dir"]).joinpath( - self.project_config["component_recipe_file"].name - ) - parsed_component_recipe = project_utils.parse_recipe_file(build_recipe) - if "ComponentName" in parsed_component_recipe: - if parsed_component_recipe["ComponentName"] != component_name: - logging.error("Component '{}' is not build.".format(parsed_component_recipe["ComponentName"])) - raise Exception( - "Failed to publish the component '{}' as it is not build.\nBuild the component `gdk component" - " build` before publishing it.".format(parsed_component_recipe["ComponentName"]) - ) - gg_build_component_artifacts = self.project_config["gg_build_component_artifacts_dir"] - bucket = self.project_config["bucket"] - artifact_uri = f"{utils.s3_prefix}{bucket}/{component_name}/{component_version}" - - if "Manifests" not in parsed_component_recipe: - logging.debug("No 'Manifests' key in the recipe.") - return - for manifest in parsed_component_recipe["Manifests"]: - if "Artifacts" not in manifest: - logging.debug("No 'Artifacts' key in the recipe manifest.") - continue - for artifact in manifest["Artifacts"]: - if "URI" not in artifact: - logging.debug("No 'URI' found in the recipe artifacts.") - continue - # Skip non-s3 URIs in the recipe. Eg docker URIs - if not artifact["URI"].startswith("s3://"): - continue - artifact_file = Path(artifact["URI"]).name - # For artifact in build component artifacts folder, update its URI - build_artifact_files = list(gg_build_component_artifacts.glob(artifact_file)) - if len(build_artifact_files) == 1: - logging.debug("Updating artifact URI of '{}' in the recipe file.".format(artifact_file)) - artifact["URI"] = f"{artifact_uri}/{artifact_file}" - else: - raise Exception( - f"Could not find the artifact file specified in the recipe '{artifact_file}' inside the build folder" - f" '{gg_build_component_artifacts}'." - ) - - # Update the version of the component in the recipe - parsed_component_recipe["ComponentVersion"] = component_version - return self.create_publish_recipe_file(component_name, component_version, parsed_component_recipe) - - def create_publish_recipe_file(self, component_name, component_version, parsed_component_recipe): - """ - Creates a new recipe file(json or yaml) with anme `-.extension` in the component - recipes build directory. - - This recipe is updated with the component version calculated and artifact URIs of the artifacts. - - Parameters - ---------- - component_name(string): Name of the component. This is also used in the name of the recipe file. - component_version(string): Version of the component calculated based on the configuration. - parsed_component_recipe(dict): Updated publish recipe with component version and s3 artifact uris - Returns - ------- - None - """ - ext = self.project_config["component_recipe_file"].name.split(".")[-1] # json or yaml - publish_recipe_file_name = f"{component_name}-{component_version}.{ext}" # Eg. HelloWorld-1.0.0.yaml - publish_recipe_file = Path(self.project_config["gg_build_recipes_dir"]).joinpath(publish_recipe_file_name).resolve() - self.project_config["publish_recipe_file"] = publish_recipe_file - with open(publish_recipe_file, "w") as prf: - try: - logging.debug( - "Creating component recipe '{}' in '{}'.".format( - publish_recipe_file_name, self.project_config["gg_build_recipes_dir"] - ) - ) - - if publish_recipe_file_name.endswith(".json"): - prf.write(json.dumps(parsed_component_recipe, indent=4)) - else: - yaml.dump(parsed_component_recipe, prf) - - return publish_recipe_file_name - except Exception as e: - raise Exception("""Failed to create publish recipe file at '{}'.\n{}""".format(publish_recipe_file, e)) + raise (e) \ No newline at end of file diff --git a/gdk/static/cli_model.json b/gdk/static/cli_model.json index 5856dd63..e3190397 100644 --- a/gdk/static/cli_model.json +++ b/gdk/static/cli_model.json @@ -10,7 +10,8 @@ "init", "build", "publish", - "list" + "list", + "push" ], "help": "Initialize, build and publish GreengrassV2 components using this command." }, @@ -128,20 +129,46 @@ ], "help": "Specify alternative component build directory." }, - "publish-dir": { + "component-name": { "name": [ - "-p", - "--publish-dir" + "-n", + "--component-name" ], - "help": "Specify local component publish directory." + "help": "Specify alternative component name to be used with alternative publish location." }, - "local": { + "component-version": { "name": [ - "-l", - "--local" + "-w", + "--component-version" + ], + "help": "Specify alternative component version to be used with alternative publish location." + } + } + }, + "push": { + "help": "Push a built component into an specific location for further processing.", + "arguments": { + "gdk-config": { + "name": [ + "-c", + "--gdk-config" + ], + "help": "Specify alternative GDK configuration file." + }, + "build-dir": { + "name": [ + "-u", + "--build-dir" + ], + "help": "Specify alternative component build directory." + }, + "destination": { + "name": [ + "-e", + "--destination" ], - "action": "store_true", - "help": "Publish component locally to build directory." + "help": "Specify local component publish directory.", + "required": true } } }, From bcedee04e61e1d7100828f772a9837a8328c650e Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Wed, 15 Feb 2023 16:54:44 +0100 Subject: [PATCH 4/9] Adding new command push --- gdk/static/cli_model_schema.json | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/gdk/static/cli_model_schema.json b/gdk/static/cli_model_schema.json index e0abe636..421a97fa 100644 --- a/gdk/static/cli_model_schema.json +++ b/gdk/static/cli_model_schema.json @@ -103,6 +103,30 @@ } } }, + "push": { + "type": "object", + "description": "Sub command under 'component' command. This is one of the sub-parsers under 'component' parser.", + "required": [ + "help", + "arguments" + ], + "properties": { + "help": { + "$ref": "#/$defs/help" + }, + "arguments": { + "description": "List of all the arguments that can be passed with the publish command.", + "required": [ + "destination" + ], + "properties": { + "destination": { + "$ref": "#/$defs/argument" + } + } + } + } + }, "list": { "type": "object", "description": "Sub command under 'component' command. This is one of the sub-parsers under 'component' parser.", @@ -150,7 +174,8 @@ "init", "list", "build", - "publish" + "publish", + "push" ], "additionalProperties": false, "$defs": { From 8ac68a9db836931a2455b08e888e5647de001e06 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Wed, 15 Feb 2023 16:55:25 +0100 Subject: [PATCH 5/9] Adding new logic for push command --- gdk/commands/component/PushCommand.py | 213 ++++++++++++++++++++++++ gdk/commands/component/component.py | 8 + gdk/commands/component/project_utils.py | 184 +++++++++++++++++--- gdk/commands/methods.py | 4 + gdk/common/exceptions/error_messages.py | 5 + gdk/common/utils.py | 1 + 6 files changed, 391 insertions(+), 24 deletions(-) create mode 100644 gdk/commands/component/PushCommand.py diff --git a/gdk/commands/component/PushCommand.py b/gdk/commands/component/PushCommand.py new file mode 100644 index 00000000..8f1bba2a --- /dev/null +++ b/gdk/commands/component/PushCommand.py @@ -0,0 +1,213 @@ +import logging +from pathlib import Path +import shutil +import re +from urllib import parse + +import gdk.commands.component.project_utils as project_utils +import gdk.common.exceptions.error_messages as error_messages +from gdk.commands.Command import Command +from gdk.common import utils +from gdk import consts + + +class PushCommand(Command): + def __init__(self, command_args) -> None: + super().__init__(command_args, "push") + + logging.debug("Arguments: {}".format(command_args)) + + project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file + project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else consts.greengrass_build_dir + #project_recipe_filename = "{}/recipes/recipe.json".format(project_build_directory) + project_recipe_filename = project_utils.find_recipe_file_in_path("recipe", "{}/recipes".format(project_build_directory)) + + logging.debug("Project config filename : {}".format(project_config_filename)) + logging.debug("Project build directory : {}".format(project_build_directory)) + logging.debug("Project recipe filename : {}".format(project_recipe_filename)) + logging.debug("Component destination : {}".format(command_args["destination"])) + + logging.debug("Loading project configuration values") + self.project_config = project_utils.get_project_config_values(project_config_filename, project_build_directory) + + logging.debug("Loading recipe values") + recipe_values = project_utils.get_project_recipe_values(project_recipe_filename) + + logging.debug("Merging recipe into project configuration value") + self.project_config.update(recipe_values) + + self.project_config["component_destination"] = command_args["destination"] + self.service_clients = project_utils.get_service_clients(self.project_config["region"]) + + def run(self): + component_destination = parse.urlparse(self.project_config["component_destination"]) + + logging.info("Pushing component into destination {}".format(component_destination)) + + if len(component_destination.path) == 0: + raise Exception(error_messages.PUSH_INVALID_PATH_VALUE) + + if "s3" == component_destination.scheme: + self.push_to_s3(component_destination.netloc, component_destination.path) + elif "file" == component_destination.scheme: + self.push_to_local("{}/{}".format(component_destination.netloc, component_destination.path)) + else: + raise Exception(error_messages.PUSH_UNSUPPORTED_LOCATION_TYPE) + + def push_to_s3(self, bucket_name, bucket_path): + logging.info("S3 location selected: '{}{}'".format(bucket_name, bucket_path)) + + component_name = self.project_config["component_name"] + + bucket_path = str(bucket_path).removeprefix("/") + + s3client = self.service_clients["s3_client"] + + paginator = s3client.get_paginator('list_objects_v2') + + response_iterator = paginator.paginate( + Bucket=bucket_name, + Delimiter='/', + Prefix=bucket_path + ) + + logging.info("Searching for subfolder '{}' into bucket '{}' with prefix '{}'".format(component_name, bucket_name, bucket_path)) + logging.debug("Paginator type: {}".format(type(response_iterator))) + + search_filter = "CommonPrefixes[].Prefix".format(component_name) + + logging.debug("Filtering search with '{}'".format(search_filter)) + + filtered_iterator = response_iterator.search(search_filter) + + boolComponentFound = False + for elem in filtered_iterator: + boolComponentFound = True + + logging.debug("Item: {}".format(elem)) + + if "CommonPrefixes" in elem: + common_prefixes = elem["CommonPrefixes"] + for f in common_prefixes: + logging.debug(" >>> Folder: {}".format(f["Prefix"])) + + if "Contents" in elem: + contents = elem["Contents"] + for c in contents: + logging.debug(" >>> Object: {}".format(c["Key"])) + + logging.debug("There is an item folder alread" if boolComponentFound else "No folder found") + + # for prefix in result.search('CommonPrefixes'): + # print(prefix.get('Prefix')) + + # bucket_objects_list_request = s3client.list_objects_v2(Bucket = bucket_name) + + # for obj in bucket_objects_list_request["Contents"]: + # logging.debug("Object {}: {}".format(obj.Key, obj)) + + # if "Buckets" in bucket_list_request: + # bucket_list = bucket_list_request["Buckets"] + # for b in bucket_list: + # logging.debug("Bucket name: {}".format(b["Name"])) + # else: + # logging.info("No buckets found") + + raise RuntimeError("S3 pushin feature to be implemented") + + def push_to_local(self, location): + logging.info("Local location selected: '{}'".format(location)) + + component_name = self.project_config["component_name"] + + # Prepare target location + + location_artifacts_path = Path(location).joinpath("artifacts").resolve() + location_recipes_path = Path(location).joinpath("recipes").resolve() + + logging.debug("Destination folder for recipe : {}".format(location_recipes_path)) + logging.debug("Destination folder for artifact : {}".format(location_artifacts_path)) + + location_artifacts_path.mkdir(parents=True, exist_ok=True) + location_recipes_path.mkdir(parents=True, exist_ok=True) + location_component_artifacts_path = Path(location_artifacts_path).joinpath(component_name).resolve() + location_component_artifacts_path.mkdir(exist_ok=True) + + build_recipes_directory = Path(self.project_config["gg_build_recipes_dir"]) + build_artifacts_directory = self.project_config["gg_build_component_artifacts_dir"] + + component_version = self.get_component_version_from_local(component_name, location_component_artifacts_path) + self.project_config["component_version"] = component_version + + logging.info("Pushing artifact '{}' with version '{}' to local location '{}'".format(component_name, component_version, location)) + logging.info("Component recipe build folder : '{}'".format(build_recipes_directory)) + logging.info("Component artifact build folder : '{}'".format(build_artifacts_directory)) + + build_recipe_file_name = project_utils.update_and_create_recipe_file(self.project_config, component_name, component_version) + build_recipe_file = Path(build_recipes_directory).joinpath(build_recipe_file_name).resolve() + + logging.info("Copying recipe file '{}' into local path '{}'".format(build_recipe_file, location_recipes_path)) + + shutil.copy(build_recipe_file.resolve(), location_recipes_path) + + # Clean destination directory, if exists + location_component_and_version_artifacts_path = Path(location_component_artifacts_path).joinpath(component_version).resolve() + logging.debug("Artifact target location: '{}'".format(location_component_and_version_artifacts_path)) + if location_component_and_version_artifacts_path.exists(): + logging.debug("Target directory already exists, cleaning its content before copying") + shutil.rmtree(location_component_and_version_artifacts_path.resolve()) + location_component_and_version_artifacts_path.mkdir(exist_ok=True) + + # Copying artifact content into target directory + build_component_artifacts = list(build_artifacts_directory.iterdir()) + for file in build_component_artifacts: + logging.debug("Copying file '{}' into target location '{}'".format(file, location_component_and_version_artifacts_path)) + shutil.copy(file.resolve(), location_component_and_version_artifacts_path) + + def get_component_version_from_local(self, component_name: str, location_component_artifacts_path: Path): + if self.project_config["component_version"] == "NEXT_PATCH": + logging.debug("Component version set to 'NEXT_PATCH' in the config file. Calculating next version of the component '{}'".format(component_name)) + + if location_component_artifacts_path.exists(): + logging.debug("Target folder exists, let's compute the NEXT_PATCH") + + pattern = re.compile("^[0-9]*\.[0-9]*\.[0-9]*$") + + major = 1 + minor = 0 + patch = -1 + + for element in location_component_artifacts_path.iterdir(): + if element.is_dir() and pattern.match(element.name): + ver_split = element.name.split(".") + + curr_major = int(ver_split[0]) + curr_minor = int(ver_split[1]) + curr_patch = int(ver_split[2]) + + logging.debug("{}.{}.{} vs {}.{}.{}".format(major, minor, patch, curr_major, curr_minor, curr_patch)) + + if curr_major > major: + major = curr_major + minor = curr_minor + patch = curr_patch + elif curr_major == major: + if curr_minor < minor: + continue + + if curr_minor > minor: + minor = curr_minor + patch = curr_patch + elif curr_patch > patch: + patch = curr_patch + + logging.debug("Selected: {}.{}.{}".format(major, minor, patch)) + + return "{}.{}.{}".format(major, minor, patch + 1) + else: + logging.debug("Target component folder does not exist, return default version") + return "1.0.0" + else: + logging.info("Using the version set for the component '{}' in the config file.".format(component_name)) + return self.project_config["component_version"] + \ No newline at end of file diff --git a/gdk/commands/component/component.py b/gdk/commands/component/component.py index 8dd2b174..28bcafa0 100644 --- a/gdk/commands/component/component.py +++ b/gdk/commands/component/component.py @@ -24,6 +24,14 @@ def publish(d_args): except Exception as e: raise Exception(f"Could not publish the component due to the following error.\n{e}") +def push(d_args): + from gdk.commands.component.PushCommand import PushCommand + + try: + PushCommand(d_args).run() + except Exception as e: + raise Exception(f"Could not publish the component due to the following error.\n{e}") + def list(d_args): from gdk.commands.component.ListCommand import ListCommand diff --git a/gdk/commands/component/project_utils.py b/gdk/commands/component/project_utils.py index 1d7b6bdb..f21fd16d 100644 --- a/gdk/commands/component/project_utils.py +++ b/gdk/commands/component/project_utils.py @@ -29,8 +29,25 @@ def get_supported_component_builds(): return json.loads(supported_builds_file.read()) return None +def find_recipe_file_in_path(recipe_name, recipe_path): + logging.debug("Looking for recipe file into {}".format(recipe_path)) -def get_recipe_file(project_recipe_filename): + json_file = list(Path(recipe_path).glob("{}.json".format(recipe_name))) + yaml_file = list(Path(recipe_path).glob("{}.yaml".format(recipe_name))) + + if not json_file and not yaml_file: + logging.error("Could not find 'recipe.json' or 'recipe.yaml' into '{}' directory.".format(recipe_path)) + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + + if json_file and yaml_file: + logging.error("Found both 'recipe.json' and 'recipe.yaml' into '{}' directory.".format(recipe_path)) + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + + recipe_file = (json_file + yaml_file)[0].resolve() + + return recipe_file + +def get_recipe_file(project_recipe_filename, project_recipe_path = utils.current_directory): """ Finds recipe file based on component name and its extension. @@ -53,8 +70,8 @@ def get_recipe_file(project_recipe_filename): # In case of missing recipe filename as argument, let's look for default ones logging.debug("Looking for default recipe file name (recipe.json or recipe.yaml)") - json_file = list(Path(utils.current_directory).glob("recipe.json")) - yaml_file = list(Path(utils.current_directory).glob("recipe.yaml")) + json_file = list(Path(project_recipe_path).glob("recipe.json")) + yaml_file = list(Path(project_recipe_path).glob("recipe.yaml")) if not json_file and not yaml_file: logging.error("Could not find 'recipe.json' or 'recipe.yaml' in the project directory.") @@ -102,16 +119,34 @@ def parse_recipe_file(component_recipe_file): except Exception as e: raise Exception("""Unable to parse the recipe file - {}.\n{}""".format(component_recipe_file.name, e)) +def get_project_recipe_values(project_recipe_filename): + vars = {} + + # Get parsed recipe file + parsed_component_recipe = parse_recipe_file(project_recipe_filename) -def get_project_config_values(project_config_filename, project_recipe_filename, project_build_directory, project_publish_directory, local_deployment): + vars["component_recipe_file"] = project_recipe_filename + vars["parsed_component_recipe"] = parsed_component_recipe + + return vars + +def get_project_config_values(project_config_filename, project_build_directory, custom_component_name = None, custom_component_version = None): # Get component configuration from the greengrass project config file. logging.info("Getting project configuration from {}".format(project_config_filename)) project_config = config_actions.get_configuration(project_config_filename)["component"] # Since there's only one key in the component configuration, use next() instead of looping in. - component_name = next(iter(project_config)) + if custom_component_name is not None: + component_name = custom_component_name + else: + component_name = next(iter(project_config)) + component_config = project_config[component_name] + + if custom_component_version is not None: + component_config["version"] = custom_component_version + component_version = component_config["version"] component_author = component_config["author"] component_build_config = component_config["build"] @@ -125,21 +160,21 @@ def get_project_config_values(project_config_filename, project_recipe_filename, gg_build_recipes_dir = Path(gg_build_directory).joinpath("recipes").resolve() gg_build_component_artifacts_dir = Path(gg_build_artifacts_dir).joinpath(component_name, component_version).resolve() - # Publish directories - gg_publish_directory = Path(project_publish_directory if project_publish_directory != None else gg_build_directory).resolve() - logging.info("Publish directory: '{}'".format(gg_publish_directory)) - gg_publish_artifacts_dir = Path(gg_publish_directory).joinpath("artifacts").resolve() - gg_publish_recipes_dir = Path(gg_publish_directory).joinpath("recipes").resolve() - gg_publish_component_artifacts_dir = Path(gg_publish_artifacts_dir).joinpath(component_name, component_version).resolve() + # # Publish directories + # gg_publish_directory = Path(gg_build_directory).resolve() + # logging.info("Publish directory: '{}'".format(gg_publish_directory)) + # gg_publish_artifacts_dir = Path(gg_publish_directory).joinpath("artifacts").resolve() + # gg_publish_recipes_dir = Path(gg_publish_directory).joinpath("recipes").resolve() + # gg_publish_component_artifacts_dir = Path(gg_publish_artifacts_dir).joinpath(component_name, component_version).resolve() - # Local or remote deployment - gg_local_deployment = local_deployment + # # Local or remote deployment + # gg_local_deployment = local_deployment - # Get recipe file - component_recipe_file = get_recipe_file(project_recipe_filename) + # # Get recipe file + # component_recipe_file = get_recipe_file(project_recipe_filename) - # Get parsed recipe file - parsed_component_recipe = parse_recipe_file(component_recipe_file) + # # Get parsed recipe file + # parsed_component_recipe = parse_recipe_file(component_recipe_file) # Create dictionary with all the above values vars = {} @@ -154,13 +189,13 @@ def get_project_config_values(project_config_filename, project_recipe_filename, vars["gg_build_artifacts_dir"] = gg_build_artifacts_dir vars["gg_build_recipes_dir"] = gg_build_recipes_dir vars["gg_build_component_artifacts_dir"] = gg_build_component_artifacts_dir - vars["gg_publish_directory"] = gg_publish_directory - vars["gg_publish_artifacts_dir"] = gg_publish_artifacts_dir - vars["gg_publish_recipes_dir"] = gg_publish_recipes_dir - vars["gg_publish_component_artifacts_dir"] = gg_publish_component_artifacts_dir - vars["gg_local_deployment"] = gg_local_deployment - vars["component_recipe_file"] = component_recipe_file - vars["parsed_component_recipe"] = parsed_component_recipe + # vars["gg_publish_directory"] = gg_publish_directory + # vars["gg_publish_artifacts_dir"] = gg_publish_artifacts_dir + # vars["gg_publish_recipes_dir"] = gg_publish_recipes_dir + # vars["gg_publish_component_artifacts_dir"] = gg_publish_component_artifacts_dir + #vars["gg_local_deployment"] = gg_local_deployment + #vars["component_recipe_file"] = component_recipe_file + #vars["parsed_component_recipe"] = parsed_component_recipe return vars @@ -185,3 +220,104 @@ def create_sts_client(region=None): def create_greengrass_client(region=None): logging.debug("Creating GreengrassV2 client") return boto3.client("greengrassv2", region_name=region) + + +def update_and_create_recipe_file(project_config, component_name, component_version): + """ + Updates recipe with the component version calculated and artifact URIs of the artifacts. This updated recipe is + used to create a new publish recipe file in build recipes directory. + + Parameters + ---------- + component_name(string): Name of the component. This is also used in the name of the recipe file. + component_version(string): Version of the component calculated based on the configuration. + + Returns + ------- + None + """ + logging.debug("Updating artifact URIs in the recipe...") + build_recipe = Path(project_config["gg_build_recipes_dir"]).joinpath( + project_config["component_recipe_file"].name + ) + parsed_component_recipe = parse_recipe_file(build_recipe) + if "ComponentName" in parsed_component_recipe: + if parsed_component_recipe["ComponentName"] != component_name: + logging.error("Component '{}' is not build.".format(parsed_component_recipe["ComponentName"])) + raise Exception( + "Failed to publish the component '{}' as it is not build.\nBuild the component `gdk component" + " build` before publishing it.".format(parsed_component_recipe["ComponentName"]) + ) + gg_build_component_artifacts = project_config["gg_build_component_artifacts_dir"] + bucket = project_config["bucket"] + artifact_uri = f"{utils.s3_prefix}{bucket}/{component_name}/{component_version}" + + if "Manifests" not in parsed_component_recipe: + logging.debug("No 'Manifests' key in the recipe.") + return + for manifest in parsed_component_recipe["Manifests"]: + if "Artifacts" not in manifest: + logging.debug("No 'Artifacts' key in the recipe manifest.") + continue + for artifact in manifest["Artifacts"]: + if "URI" not in artifact: + logging.debug("No 'URI' found in the recipe artifacts.") + continue + # Skip non-s3 URIs in the recipe. Eg docker URIs + if not artifact["URI"].startswith("s3://"): + continue + artifact_file = Path(artifact["URI"]).name + # For artifact in build component artifacts folder, update its URI + build_artifact_files = list(gg_build_component_artifacts.glob(artifact_file)) + if len(build_artifact_files) == 1: + logging.debug("Updating artifact URI of '{}' in the recipe file.".format(artifact_file)) + artifact["URI"] = f"{artifact_uri}/{artifact_file}" + else: + raise Exception( + f"Could not find the artifact file specified in the recipe '{artifact_file}' inside the build folder" + f" '{gg_build_component_artifacts}'." + ) + + # Update the version of the component in the recipe + parsed_component_recipe["ComponentVersion"] = component_version + return create_publish_recipe_file(project_config, parsed_component_recipe) + +def create_publish_recipe_file(project_config, parsed_component_recipe): + """ + Creates a new recipe file(json or yaml) with anme `-.extension` in the component + recipes build directory. + + This recipe is updated with the component version calculated and artifact URIs of the artifacts. + + Parameters + ---------- + component_name(string): Name of the component. This is also used in the name of the recipe file. + component_version(string): Version of the component calculated based on the configuration. + parsed_component_recipe(dict): Updated publish recipe with component version and s3 artifact uris + Returns + ------- + None + """ + component_name = project_config["component_name"] + component_version = project_config["component_version"] + + ext = project_config["component_recipe_file"].name.split(".")[-1] # json or yaml + publish_recipe_file_name = f"{component_name}-{component_version}.{ext}" # Eg. HelloWorld-1.0.0.yaml + publish_recipe_file = Path(project_config["gg_build_recipes_dir"]).joinpath(publish_recipe_file_name).resolve() + project_config["publish_recipe_file"] = publish_recipe_file + with open(publish_recipe_file, "w") as prf: + try: + logging.debug( + "Creating component recipe '{}' in '{}'.".format( + publish_recipe_file_name, project_config["gg_build_recipes_dir"] + ) + ) + + if publish_recipe_file_name.endswith(".json"): + prf.write(json.dumps(parsed_component_recipe, indent=4)) + else: + yaml.dump(parsed_component_recipe, prf) + + return publish_recipe_file_name + except Exception as e: + raise Exception("""Failed to create publish recipe file at '{}'.\n{}""".format(publish_recipe_file, e)) \ No newline at end of file diff --git a/gdk/commands/methods.py b/gdk/commands/methods.py index 3708099a..52eee9ba 100644 --- a/gdk/commands/methods.py +++ b/gdk/commands/methods.py @@ -13,5 +13,9 @@ def _gdk_component_publish(d_args): component.publish(d_args) +def _gdk_component_push(d_args): + component.push(d_args) + + def _gdk_component_list(d_args): component.list(d_args) diff --git a/gdk/common/exceptions/error_messages.py b/gdk/common/exceptions/error_messages.py index cca777f8..20eb10f9 100644 --- a/gdk/common/exceptions/error_messages.py +++ b/gdk/common/exceptions/error_messages.py @@ -44,3 +44,8 @@ # PUBLISH COMMAND PUBLISH_FAILED = "Failed to publish new version of component with the given configuration." + +# PUSH COMMAND + +PUSH_UNSUPPORTED_LOCATION_TYPE = "Unsupported location type provided." +PUSH_INVALID_PATH_VALUE = "Invalid location value provided." diff --git a/gdk/common/utils.py b/gdk/common/utils.py index 2169576c..dbd0aa9a 100644 --- a/gdk/common/utils.py +++ b/gdk/common/utils.py @@ -135,3 +135,4 @@ def cli_version_check(): cli_version = version.__version__ latest_cli_version_file = "https://raw.githubusercontent.com/aws-greengrass/aws-greengrass-gdk-cli/main/gdk/_version.py" s3_prefix = "s3://" +file_prefix = "file://" \ No newline at end of file From 4d9777218fbf497e2ef0e35be17b668ceacb2e84 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Thu, 16 Feb 2023 00:19:49 +0100 Subject: [PATCH 6/9] Fixing bug on -r parameter for build command --- gdk/commands/component/BuildCommand.py | 2 +- gdk/commands/component/project_utils.py | 48 +++++++++---------------- 2 files changed, 17 insertions(+), 33 deletions(-) diff --git a/gdk/commands/component/BuildCommand.py b/gdk/commands/component/BuildCommand.py index 84221f91..54e3ac6a 100644 --- a/gdk/commands/component/BuildCommand.py +++ b/gdk/commands/component/BuildCommand.py @@ -22,7 +22,7 @@ def __init__(self, command_args) -> None: project_config_filename = command_args["gdk_config"] if command_args["gdk_config"] != None else consts.cli_project_config_file project_build_directory = command_args["build_dir"] if command_args["build_dir"] != None else "{}/{}".format(utils.current_directory, consts.greengrass_build_dir) - project_recipe_filename = command_args["recipe"] if command_args["recipe"] != None else project_utils.find_recipe_file_in_path("recipe", utils.current_directory) + project_recipe_filename = project_utils.find_recipe_file_in_path(command_args["recipe"] if command_args["recipe"] != None else "recipe", utils.current_directory) logging.debug("Project config filename: {}".format(project_config_filename)) logging.debug("Project build directory: {}".format(project_build_directory)) diff --git a/gdk/commands/component/project_utils.py b/gdk/commands/component/project_utils.py index f21fd16d..37262c3b 100644 --- a/gdk/commands/component/project_utils.py +++ b/gdk/commands/component/project_utils.py @@ -32,18 +32,24 @@ def get_supported_component_builds(): def find_recipe_file_in_path(recipe_name, recipe_path): logging.debug("Looking for recipe file into {}".format(recipe_path)) - json_file = list(Path(recipe_path).glob("{}.json".format(recipe_name))) - yaml_file = list(Path(recipe_path).glob("{}.yaml".format(recipe_name))) + if "." in recipe_name: + logging.debug("Seems like provided recipe name has extention, check for this file directly") + recipe_file = Path(recipe_path).joinpath(recipe_name).resolve() + else: + logging.debug("Seems like provided recipe name has no extention, check for it automatically") + + json_file = list(Path(recipe_path).glob("{}.json".format(recipe_name))) + yaml_file = list(Path(recipe_path).glob("{}.yaml".format(recipe_name))) - if not json_file and not yaml_file: - logging.error("Could not find 'recipe.json' or 'recipe.yaml' into '{}' directory.".format(recipe_path)) - raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + if not json_file and not yaml_file: + logging.error("Could not find 'recipe.json' or 'recipe.yaml' into '{}' directory.".format(recipe_path)) + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) - if json_file and yaml_file: - logging.error("Found both 'recipe.json' and 'recipe.yaml' into '{}' directory.".format(recipe_path)) - raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) + if json_file and yaml_file: + logging.error("Found both 'recipe.json' and 'recipe.yaml' into '{}' directory.".format(recipe_path)) + raise Exception(error_messages.PROJECT_RECIPE_FILE_NOT_FOUND) - recipe_file = (json_file + yaml_file)[0].resolve() + recipe_file = (json_file + yaml_file)[0].resolve() return recipe_file @@ -160,22 +166,6 @@ def get_project_config_values(project_config_filename, project_build_directory, gg_build_recipes_dir = Path(gg_build_directory).joinpath("recipes").resolve() gg_build_component_artifacts_dir = Path(gg_build_artifacts_dir).joinpath(component_name, component_version).resolve() - # # Publish directories - # gg_publish_directory = Path(gg_build_directory).resolve() - # logging.info("Publish directory: '{}'".format(gg_publish_directory)) - # gg_publish_artifacts_dir = Path(gg_publish_directory).joinpath("artifacts").resolve() - # gg_publish_recipes_dir = Path(gg_publish_directory).joinpath("recipes").resolve() - # gg_publish_component_artifacts_dir = Path(gg_publish_artifacts_dir).joinpath(component_name, component_version).resolve() - - # # Local or remote deployment - # gg_local_deployment = local_deployment - - # # Get recipe file - # component_recipe_file = get_recipe_file(project_recipe_filename) - - # # Get parsed recipe file - # parsed_component_recipe = parse_recipe_file(component_recipe_file) - # Create dictionary with all the above values vars = {} vars["project_config_filename"] = project_config_filename @@ -189,13 +179,7 @@ def get_project_config_values(project_config_filename, project_build_directory, vars["gg_build_artifacts_dir"] = gg_build_artifacts_dir vars["gg_build_recipes_dir"] = gg_build_recipes_dir vars["gg_build_component_artifacts_dir"] = gg_build_component_artifacts_dir - # vars["gg_publish_directory"] = gg_publish_directory - # vars["gg_publish_artifacts_dir"] = gg_publish_artifacts_dir - # vars["gg_publish_recipes_dir"] = gg_publish_recipes_dir - # vars["gg_publish_component_artifacts_dir"] = gg_publish_component_artifacts_dir - #vars["gg_local_deployment"] = gg_local_deployment - #vars["component_recipe_file"] = component_recipe_file - #vars["parsed_component_recipe"] = parsed_component_recipe + return vars From 00397339970676026d72846b38b64779f94e6a35 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Thu, 16 Feb 2023 00:40:57 +0100 Subject: [PATCH 7/9] Fixing bug on extention check for recipe parameter --- gdk/commands/component/project_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gdk/commands/component/project_utils.py b/gdk/commands/component/project_utils.py index 37262c3b..afabcd78 100644 --- a/gdk/commands/component/project_utils.py +++ b/gdk/commands/component/project_utils.py @@ -8,6 +8,7 @@ import gdk.common.exceptions.error_messages as error_messages import gdk.common.utils as utils import yaml +import re def get_supported_component_builds(): @@ -30,9 +31,11 @@ def get_supported_component_builds(): return None def find_recipe_file_in_path(recipe_name, recipe_path): - logging.debug("Looking for recipe file into {}".format(recipe_path)) + logging.debug("Looking for recipe '{}' file into {}".format(recipe_name, recipe_path)) - if "." in recipe_name: + extentions = re.compile("[\.json|\.yaml|\.yml]$") + + if extentions.search(recipe_name): logging.debug("Seems like provided recipe name has extention, check for this file directly") recipe_file = Path(recipe_path).joinpath(recipe_name).resolve() else: From f2f1c2dc4f600afa7f156b7c672c9215af4051f6 Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Thu, 16 Feb 2023 00:50:32 +0100 Subject: [PATCH 8/9] Updating REAMDE.md --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 66355e48..6c0c803a 100644 --- a/README.md +++ b/README.md @@ -88,3 +88,17 @@ Configure AWS CLI with your credentials as shown here - https://docs.aws.amazon. 5. Creates new version of the component in your AWS account. `gdk component publish` + +#### 4. Local deployment for local tests + +1. Build the component specifing also the gdk-config.json file, the recipe.json file and the build folder: + +`gdk component build -d -c gdk-config.json -r recipe.json -u greengrass-build` + +2. Push the component locally to a custom folder: + +`gdk component push -u greengrass-build -e file:///tmp/gg` + +1. One you have done local tests, publish component on cloud retrieving the built artifact from local folder: + +`gdk component publish -d -c gdk-config.json -u /tmp/gg -n -w `. \ No newline at end of file From 60b1a8631f4923f1a6506cf174930668bdf15d7a Mon Sep 17 00:00:00 2001 From: Iacopo Palazzi Date: Thu, 16 Feb 2023 01:27:17 +0100 Subject: [PATCH 9/9] Removing temporary code --- gdk/commands/component/PushCommand.py | 62 +-------------------------- 1 file changed, 2 insertions(+), 60 deletions(-) diff --git a/gdk/commands/component/PushCommand.py b/gdk/commands/component/PushCommand.py index 8f1bba2a..e8f88afb 100644 --- a/gdk/commands/component/PushCommand.py +++ b/gdk/commands/component/PushCommand.py @@ -55,65 +55,7 @@ def run(self): raise Exception(error_messages.PUSH_UNSUPPORTED_LOCATION_TYPE) def push_to_s3(self, bucket_name, bucket_path): - logging.info("S3 location selected: '{}{}'".format(bucket_name, bucket_path)) - - component_name = self.project_config["component_name"] - - bucket_path = str(bucket_path).removeprefix("/") - - s3client = self.service_clients["s3_client"] - - paginator = s3client.get_paginator('list_objects_v2') - - response_iterator = paginator.paginate( - Bucket=bucket_name, - Delimiter='/', - Prefix=bucket_path - ) - - logging.info("Searching for subfolder '{}' into bucket '{}' with prefix '{}'".format(component_name, bucket_name, bucket_path)) - logging.debug("Paginator type: {}".format(type(response_iterator))) - - search_filter = "CommonPrefixes[].Prefix".format(component_name) - - logging.debug("Filtering search with '{}'".format(search_filter)) - - filtered_iterator = response_iterator.search(search_filter) - - boolComponentFound = False - for elem in filtered_iterator: - boolComponentFound = True - - logging.debug("Item: {}".format(elem)) - - if "CommonPrefixes" in elem: - common_prefixes = elem["CommonPrefixes"] - for f in common_prefixes: - logging.debug(" >>> Folder: {}".format(f["Prefix"])) - - if "Contents" in elem: - contents = elem["Contents"] - for c in contents: - logging.debug(" >>> Object: {}".format(c["Key"])) - - logging.debug("There is an item folder alread" if boolComponentFound else "No folder found") - - # for prefix in result.search('CommonPrefixes'): - # print(prefix.get('Prefix')) - - # bucket_objects_list_request = s3client.list_objects_v2(Bucket = bucket_name) - - # for obj in bucket_objects_list_request["Contents"]: - # logging.debug("Object {}: {}".format(obj.Key, obj)) - - # if "Buckets" in bucket_list_request: - # bucket_list = bucket_list_request["Buckets"] - # for b in bucket_list: - # logging.debug("Bucket name: {}".format(b["Name"])) - # else: - # logging.info("No buckets found") - - raise RuntimeError("S3 pushin feature to be implemented") + raise RuntimeError("S3 pushing feature to be implemented") def push_to_local(self, location): logging.info("Local location selected: '{}'".format(location)) @@ -210,4 +152,4 @@ def get_component_version_from_local(self, component_name: str, location_compone else: logging.info("Using the version set for the component '{}' in the config file.".format(component_name)) return self.project_config["component_version"] - \ No newline at end of file +