diff --git a/pyproject.toml b/pyproject.toml index c3fa4b13..97ad00c8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "solutions-builder" -version = "1.17.15" +version = "1.17.16" description = "A solution framework to generate a project with built-in structure and modules" authors = ["Jon Chen "] license = "Apache" diff --git a/solutions_builder/cli/cli.py b/solutions_builder/cli/cli.py index 83f05981..c6ca4493 100644 --- a/solutions_builder/cli/cli.py +++ b/solutions_builder/cli/cli.py @@ -25,6 +25,7 @@ from .set import set_app from .vars import vars_app from .cli_utils import * +from .cli_constants import DEBUG __version__ = importlib.metadata.version("solutions-builder") DEFAULT_DEPLOY_PROFILE = "default-deploy" @@ -126,12 +127,14 @@ def update(solution_path: Annotated[Optional[str], # Build and deploy services. @app.command() -def deploy(profile: str = DEFAULT_DEPLOY_PROFILE, - component: str = None, - dev: Optional[bool] = False, - solution_path: Annotated[Optional[str], - typer.Argument()] = ".", - yes: Optional[bool] = False): +def deploy( + profile: Annotated[str, typer.Option("--profile", "-p")] = DEFAULT_DEPLOY_PROFILE, + component: Annotated[str, typer.Option("--component", "-c", "-m")] = None, + dev: Optional[bool] = False, + solution_path: Annotated[Optional[str], + typer.Argument()] = ".", + skaffold_args: Optional[str] = "", + yes: Optional[bool] = False): """ Build and deploy services. """ @@ -140,6 +143,9 @@ def deploy(profile: str = DEFAULT_DEPLOY_PROFILE, sb_yaml = read_yaml(f"{solution_path}/sb.yaml") project_id = sb_yaml["project_id"] terraform_gke = sb_yaml["components"].get("terraform_gke") + env_vars = { + "PROJECT_ID": project_id, + } commands = [] if component: @@ -160,15 +166,22 @@ def deploy(profile: str = DEFAULT_DEPLOY_PROFILE, ) commands.append( - f"{skaffold_command} -p {profile} {component_flag} --default-repo=\"gcr.io/{project_id}\"" + f"{skaffold_command} -p {profile} {component_flag} --default-repo=\"gcr.io/{project_id}\" {skaffold_args}" ) print("This will build and deploy all services using the command below:") for command in commands: - print_highlight(f"- {command}") + print_success(f"- {command}") + + print("\nwith the following environment variables:") + env_var_str = "" + for key, value in env_vars.items(): + print_success(f"- {key}={value}") + env_var_str += f"{key}={value} " + confirm("\nThis may take a few minutes. Continue?", skip=yes) for command in commands: - exec_shell(command, working_dir=solution_path) + exec_shell(env_var_str + command, working_dir=solution_path) # Destory deployment. @@ -236,7 +249,7 @@ def main(): print() except Exception as e: - if os.getenv("DEBUG", False): + if DEBUG: traceback.print_exc() print_error(e) diff --git a/solutions_builder/cli/cli_constants.py b/solutions_builder/cli/cli_constants.py new file mode 100644 index 00000000..e8300b5f --- /dev/null +++ b/solutions_builder/cli/cli_constants.py @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +# Global DEBUG flag for all CLI routes. +DEBUG = (os.environ.get("SB_DEBUG", "").lower() == "true") diff --git a/solutions_builder/cli/cli_utils.py b/solutions_builder/cli/cli_utils.py index bffacce9..8fcc6176 100644 --- a/solutions_builder/cli/cli_utils.py +++ b/solutions_builder/cli/cli_utils.py @@ -14,7 +14,11 @@ limitations under the License. """ -import os, yaml, typer, subprocess, re +import os +import yaml +import typer +import subprocess +import re def confirm(msg, skip=False, default=True): @@ -139,7 +143,7 @@ def list_subfolders(path): def check_git_url(url): - regex_str = "((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?" + regex_str = "((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.\\@\\:/\\-~]+)(\\.git)(/)?" regex = re.compile(regex_str) match = regex.match(url) return match is not None diff --git a/solutions_builder/cli/set.py b/solutions_builder/cli/set.py index 1423bcdb..1ffdecdb 100644 --- a/solutions_builder/cli/set.py +++ b/solutions_builder/cli/set.py @@ -21,6 +21,7 @@ from typing_extensions import Annotated from copier import run_auto from .cli_utils import * +from .vars import set_var set_app = typer.Typer() @@ -38,8 +39,10 @@ def project_id( ): validate_solution_folder(solution_path) root_st_yaml = read_yaml(f"{solution_path}/sb.yaml") - old_project_id = root_st_yaml.get("project_id") - old_project_number = root_st_yaml.get("project_number") + global_variables = root_st_yaml.get("global_variables", {}) + + old_project_id = global_variables.get("project_id") + old_project_number = global_variables.get("project_number") assert old_project_id, "project_id does not exist in sb.yaml" confirm( @@ -51,8 +54,9 @@ def project_id( new_project_number = int(get_project_number(new_project_id)) assert new_project_number, "Unable to receive project number for project '{new_project_id}'" - root_st_yaml["project_id"] = new_project_id - root_st_yaml["project_number"] = new_project_number + global_variables["project_id"] = new_project_id + global_variables["project_number"] = new_project_number + root_st_yaml["global_variables"] = global_variables write_yaml(f"{solution_path}/sb.yaml", root_st_yaml) # Update copier answers @@ -61,31 +65,10 @@ def project_id( copier_yaml["project_number"] = int(get_project_number(new_project_id)) write_yaml(f"{solution_path}/.copier-answers.yml", copier_yaml) - file_set = set() - # Adding includes. - for pattern in INCLUDE_PATTERNS: - file_list = pathlib.Path(solution_path).rglob(f"{pattern}") - file_set.update(set([str(x) for x in file_list])) - - # Removing excludes. - for pattern in EXCLUDE_PATTERNS: - file_list = pathlib.Path(solution_path).rglob(f"{pattern}") - file_set = file_set - set([str(x) for x in file_list]) - - for filename in list(file_set): - with open(filename, "r") as file: - filedata = file.read() - # Replace project_id - filedata = re.sub(old_project_id, new_project_id, filedata) - # Replace project_number - if old_project_number and new_project_number: - filedata = re.sub(str(old_project_number), str(new_project_number), filedata) - - # Write back to the original file. - with open(filename, "w") as file: - file.write(filedata) + set_var("project_id", new_project_id) + set_var("project_number", new_project_number) print( f"\nReplaced project_id from '{old_project_id}' to '{new_project_id}'.") print( - f"Replaced project_number from '{old_project_number}' to '{new_project_number}'.") + f"Replaced project_number from '{old_project_number}' to '{new_project_number}'.\n") diff --git a/solutions_builder/cli/vars.py b/solutions_builder/cli/vars.py index 2edb32bf..4f635c11 100644 --- a/solutions_builder/cli/vars.py +++ b/solutions_builder/cli/vars.py @@ -20,39 +20,40 @@ from typing_extensions import Annotated from copier import run_auto from .cli_utils import * +from .cli_constants import DEBUG + vars_app = typer.Typer() INCLUDE_PATTERNS = [ - "*.yaml", "*.yml", "*.env", "*.tfvars", "*.tf", "*.sh", "*.md" + "*.yaml", "*.yml", "*.env", "*.tfvars", "*.tf", "*.sh" ] EXCLUDE_PATTERNS = ["**/.terraform/**/*.*", "**/node_modules/**/*.*", "**/.venv/**/*.*"] # Replace a variable with a given text content. -def replace_var_to_template(var_name, text, custom_template=False, debug=False): - # This pattern matches lines with sb-var anchor in the comment at the end. +def replace_var_to_template(var_name, text, custom_template=False): + # Regex test: https://regex101.com/r/XtnJQI/4 + # match_pattern matches lines with sb-var anchor in the comment at the end. # For example: # PROJECT_ID: 12345 # sb-var:project_id # GCP_REGION = "us-central1" # sb-var:gcp_region - match_pattern = f"^([^\\r]*[:|=][\\s\-]*)([\"\']?)([^\"^\']*)([\"\']?)\\s*#\\s*sb-var:{var_name}" + match_pattern = f"(\\s*[\":=-][ ]*)(-[ ]*)?([\"\']?)([^\"^\'^\r^\n]*)([\"\']?)\\s*#\\s*sb-var:{var_name}" - # This output patterh print the jinja2 template for the specific variable name. + # output_pattern prints the jinja2 template for the specific variable name. # For example: # PROJECT_ID: {{project_id}} # sb-var:project_id - output_pattern = f"\\1\\2{{{{{var_name}}}}}\\4 # sb-var:{var_name}" + output_pattern = f"\\1\\2\\3{{{{{var_name}}}}}\\5 # sb-var:{var_name}" # In addition, if custom_template is true, the pattern will extend to the custom # template string at the end of the anchor. For example: # BUCKET_NAME: my-project-bucket # sb-var:project_id:{{project_id}}-bucket if custom_template: match_pattern = match_pattern + ":(.*)" - output_pattern = f"\\1\\2\\5\\4 # sb-var:{var_name}:\\5" - - if debug: - print(f"match_pattern = {match_pattern}") + output_pattern = f"\\1\\2\\3\\6\\5 # sb-var:{var_name}:\\6" # Replace with regex pattern and returns new text and count of changes. text, count = re.subn(match_pattern, output_pattern, text) + return (text, count) def restore_template_in_comment(var_name, var_value, text): @@ -109,6 +110,9 @@ def apply_var_to_folder(solution_path, var_name, var_value): modified_files_list = [] for filename in list(file_set): + if DEBUG: + print(filename) + with open(filename, "r") as file: # Replace variable filedata = file.read() @@ -133,6 +137,7 @@ def set_var( solution_path: Annotated[Optional[str], typer.Argument()] = ".", ): validate_solution_folder(solution_path) + print(f"Setting {var_name} to '{var_value}'...") # Update to the root sb.yaml root_st_yaml = read_yaml(f"{solution_path}/sb.yaml") diff --git a/solutions_builder/cli/vars_test.py b/solutions_builder/cli/vars_test.py index d8ae75a2..f977cc8e 100644 --- a/solutions_builder/cli/vars_test.py +++ b/solutions_builder/cli/vars_test.py @@ -74,3 +74,43 @@ def test_replace_var_to_value_custom_template(): # FIXME: it double-counted the changes, as both simple template and custom # template both counts. assert count == 2 + +def test_replace_with_multiple_occurances(): + text = """ + env: + PROJECT_ID: old-value-1 # sb-var:project_id + PROJECT_ID_2: old-value-2 # sb-var:project_id + """ + text, count = replace_var_to_value("project_id", "fake-id", text) + assert text == """ + env: + PROJECT_ID: fake-id # sb-var:project_id + PROJECT_ID_2: fake-id # sb-var:project_id + """ + assert count == 2 + + +def test_replace_with_yaml_arrays(): + text = """ + array: + - not-replaced-yet # sb-var:project_id + """ + text, count = replace_var_to_value("project_id", "fake-id", text) + assert text == """ + array: + - fake-id # sb-var:project_id + """ + assert count == 1 + + text = """ + array: + - "not-replaced-yet" # sb-var:project_id + - "not-replaced-yet" # sb-var:project_id + """ + text, count = replace_var_to_value("project_id", "fake-id", text) + assert text == """ + array: + - "fake-id" # sb-var:project_id + - "fake-id" # sb-var:project_id + """ + assert count == 2