Merge pull request #139 from GoogleCloudPlatform/fix-sb-vars

fix vars set on yaml arrays

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "solutions-builder"
-version = "1.17.15"
+version = "1.17.16"
 description = "A solution framework to generate a project with built-in structure and modules"
 authors = ["Jon Chen <[email protected]>"]
 license = "Apache"

solutions_builder/cli/cli.py

@@ -25,6 +25,7 @@
 from .set import set_app
 from .vars import vars_app
 from .cli_utils import *
+from .cli_constants import DEBUG
 
 __version__ = importlib.metadata.version("solutions-builder")
 DEFAULT_DEPLOY_PROFILE = "default-deploy"
@@ -126,12 +127,14 @@ def update(solution_path: Annotated[Optional[str],
 
 # Build and deploy services.
 @app.command()
-def deploy(profile: str = DEFAULT_DEPLOY_PROFILE,
-           component: str = None,
-           dev: Optional[bool] = False,
-           solution_path: Annotated[Optional[str],
-                                    typer.Argument()] = ".",
-           yes: Optional[bool] = False):
+def deploy(
+    profile: Annotated[str, typer.Option("--profile", "-p")] = DEFAULT_DEPLOY_PROFILE,
+    component: Annotated[str, typer.Option("--component", "-c", "-m")] = None,
+    dev: Optional[bool] = False,
+    solution_path: Annotated[Optional[str],
+                            typer.Argument()] = ".",
+    skaffold_args: Optional[str] = "",
+    yes: Optional[bool] = False):
   """
   Build and deploy services.
   """
@@ -140,6 +143,9 @@ def deploy(profile: str = DEFAULT_DEPLOY_PROFILE,
   sb_yaml = read_yaml(f"{solution_path}/sb.yaml")
   project_id = sb_yaml["project_id"]
   terraform_gke = sb_yaml["components"].get("terraform_gke")
+  env_vars = {
+    "PROJECT_ID": project_id,
+  }
   commands = []
 
   if component:
@@ -160,15 +166,22 @@ def deploy(profile: str = DEFAULT_DEPLOY_PROFILE,
     )
 
   commands.append(
-      f"{skaffold_command} -p {profile} {component_flag} --default-repo=\"gcr.io/{project_id}\""
+      f"{skaffold_command} -p {profile} {component_flag} --default-repo=\"gcr.io/{project_id}\" {skaffold_args}"
   )
   print("This will build and deploy all services using the command below:")
   for command in commands:
-    print_highlight(f"- {command}")
+    print_success(f"- {command}")
+
+  print("\nwith the following environment variables:")
+  env_var_str = ""
+  for key, value in env_vars.items():
+    print_success(f"- {key}={value}")
+    env_var_str += f"{key}={value} "
+
   confirm("\nThis may take a few minutes. Continue?", skip=yes)
 
   for command in commands:
-    exec_shell(command, working_dir=solution_path)
+    exec_shell(env_var_str + command, working_dir=solution_path)
 
 
 # Destory deployment.
@@ -236,7 +249,7 @@ def main():
     print()
 
   except Exception as e:
-    if os.getenv("DEBUG", False):
+    if DEBUG:
       traceback.print_exc()
     print_error(e)
 

solutions_builder/cli/cli_constants.py

@@ -0,0 +1,18 @@
+# Copyright 2023 Google LLC
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+# Global DEBUG flag for all CLI routes.
+DEBUG = (os.environ.get("SB_DEBUG", "").lower() == "true")

solutions_builder/cli/cli_utils.py

@@ -14,7 +14,11 @@
 limitations under the License.
 """
 
-import os, yaml, typer, subprocess, re
+import os
+import yaml
+import typer
+import subprocess
+import re
 
 
 def confirm(msg, skip=False, default=True):
@@ -139,7 +143,7 @@ def list_subfolders(path):
 
 
 def check_git_url(url):
-  regex_str = "((git|ssh|http(s)?)|(git@[\w\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?"
+  regex_str = "((git|ssh|http(s)?)|(git@[\\w\\.]+))(:(//)?)([\\w\\.\\@\\:/\\-~]+)(\\.git)(/)?"
   regex = re.compile(regex_str)
   match = regex.match(url)
   return match is not None

solutions_builder/cli/set.py

@@ -21,6 +21,7 @@
 from typing_extensions import Annotated
 from copier import run_auto
 from .cli_utils import *
+from .vars import set_var
 
 set_app = typer.Typer()
 
@@ -38,8 +39,10 @@ def project_id(
 ):
   validate_solution_folder(solution_path)
   root_st_yaml = read_yaml(f"{solution_path}/sb.yaml")
-  old_project_id = root_st_yaml.get("project_id")
-  old_project_number = root_st_yaml.get("project_number")
+  global_variables = root_st_yaml.get("global_variables", {})
+
+  old_project_id = global_variables.get("project_id")
+  old_project_number = global_variables.get("project_number")
   assert old_project_id, "project_id does not exist in sb.yaml"
 
   confirm(
@@ -51,8 +54,9 @@ def project_id(
   new_project_number = int(get_project_number(new_project_id))
   assert new_project_number, "Unable to receive project number for project '{new_project_id}'"
 
-  root_st_yaml["project_id"] = new_project_id
-  root_st_yaml["project_number"] = new_project_number
+  global_variables["project_id"] = new_project_id
+  global_variables["project_number"] = new_project_number
+  root_st_yaml["global_variables"] = global_variables
   write_yaml(f"{solution_path}/sb.yaml", root_st_yaml)
 
   # Update copier answers
@@ -61,31 +65,10 @@ def project_id(
   copier_yaml["project_number"] = int(get_project_number(new_project_id))
   write_yaml(f"{solution_path}/.copier-answers.yml", copier_yaml)
 
-  file_set = set()
-  # Adding includes.
-  for pattern in INCLUDE_PATTERNS:
-    file_list = pathlib.Path(solution_path).rglob(f"{pattern}")
-    file_set.update(set([str(x) for x in file_list]))
-
-  # Removing excludes.
-  for pattern in EXCLUDE_PATTERNS:
-    file_list = pathlib.Path(solution_path).rglob(f"{pattern}")
-    file_set = file_set - set([str(x) for x in file_list])
-
-  for filename in list(file_set):
-    with open(filename, "r") as file:
-      filedata = file.read()
-      # Replace project_id
-      filedata = re.sub(old_project_id, new_project_id, filedata)
-      # Replace project_number
-      if old_project_number and new_project_number:
-        filedata = re.sub(str(old_project_number), str(new_project_number), filedata)
-
-    # Write back to the original file.
-    with open(filename, "w") as file:
-      file.write(filedata)
+  set_var("project_id", new_project_id)
+  set_var("project_number", new_project_number)
 
   print(
       f"\nReplaced project_id from '{old_project_id}' to '{new_project_id}'.")
   print(
-      f"Replaced project_number from '{old_project_number}' to '{new_project_number}'.")
+      f"Replaced project_number from '{old_project_number}' to '{new_project_number}'.\n")

solutions_builder/cli/vars.py

@@ -20,39 +20,40 @@
 from typing_extensions import Annotated
 from copier import run_auto
 from .cli_utils import *
+from .cli_constants import DEBUG
+
 
 vars_app = typer.Typer()
 
 INCLUDE_PATTERNS = [
-    "*.yaml", "*.yml", "*.env", "*.tfvars", "*.tf", "*.sh", "*.md"
+    "*.yaml", "*.yml", "*.env", "*.tfvars", "*.tf", "*.sh"
 ]
 EXCLUDE_PATTERNS = ["**/.terraform/**/*.*", "**/node_modules/**/*.*", "**/.venv/**/*.*"]
 
 # Replace a variable with a given text content.
-def replace_var_to_template(var_name, text, custom_template=False, debug=False):
-  # This pattern matches lines with sb-var anchor in the comment at the end.
+def replace_var_to_template(var_name, text, custom_template=False):
+  # Regex test: https://regex101.com/r/XtnJQI/4
+  # match_pattern matches lines with sb-var anchor in the comment at the end.
   # For example:
   #   PROJECT_ID: 12345          # sb-var:project_id
   #   GCP_REGION = "us-central1" # sb-var:gcp_region
-  match_pattern = f"^([^\\r]*[:|=][\\s\-]*)([\"\']?)([^\"^\']*)([\"\']?)\\s*#\\s*sb-var:{var_name}"
+  match_pattern = f"(\\s*[\":=-][ ]*)(-[ ]*)?([\"\']?)([^\"^\'^\r^\n]*)([\"\']?)\\s*#\\s*sb-var:{var_name}"
 
-  # This output patterh print the jinja2 template for the specific variable name.
+  # output_pattern prints the jinja2 template for the specific variable name.
   # For example:
   #   PROJECT_ID: {{project_id}} # sb-var:project_id
-  output_pattern = f"\\1\\2{{{{{var_name}}}}}\\4 # sb-var:{var_name}"
+  output_pattern = f"\\1\\2\\3{{{{{var_name}}}}}\\5 # sb-var:{var_name}"
 
   # In addition, if custom_template is true, the pattern will extend to the custom
   # template string at the end of the anchor. For example:
   #   BUCKET_NAME: my-project-bucket # sb-var:project_id:{{project_id}}-bucket
   if custom_template:
     match_pattern = match_pattern + ":(.*)"
-    output_pattern = f"\\1\\2\\5\\4 # sb-var:{var_name}:\\5"
-
-  if debug:
-    print(f"match_pattern = {match_pattern}")
+    output_pattern = f"\\1\\2\\3\\6\\5 # sb-var:{var_name}:\\6"
 
   # Replace with regex pattern and returns new text and count of changes.
   text, count = re.subn(match_pattern, output_pattern, text)
+
   return (text, count)
 
 def restore_template_in_comment(var_name, var_value, text):
@@ -109,6 +110,9 @@ def apply_var_to_folder(solution_path, var_name, var_value):
 
   modified_files_list = []
   for filename in list(file_set):
+    if DEBUG:
+      print(filename)
+
     with open(filename, "r") as file:
       # Replace variable
       filedata = file.read()
@@ -133,6 +137,7 @@ def set_var(
     solution_path: Annotated[Optional[str], typer.Argument()] = ".",
 ):
   validate_solution_folder(solution_path)
+  print(f"Setting {var_name} to '{var_value}'...")
 
   # Update to the root sb.yaml
   root_st_yaml = read_yaml(f"{solution_path}/sb.yaml")

solutions_builder/cli/vars_test.py

@@ -74,3 +74,43 @@ def test_replace_var_to_value_custom_template():
   # FIXME: it double-counted the changes, as both simple template and custom
   # template both counts.
   assert count == 2
+
+def test_replace_with_multiple_occurances():
+  text = """
+    env:
+      PROJECT_ID: old-value-1 # sb-var:project_id
+      PROJECT_ID_2: old-value-2 # sb-var:project_id
+    """
+  text, count = replace_var_to_value("project_id", "fake-id", text)
+  assert text == """
+    env:
+      PROJECT_ID: fake-id # sb-var:project_id
+      PROJECT_ID_2: fake-id # sb-var:project_id
+    """
+  assert count == 2
+
+
+def test_replace_with_yaml_arrays():
+  text = """
+    array:
+      - not-replaced-yet # sb-var:project_id
+    """
+  text, count = replace_var_to_value("project_id", "fake-id", text)
+  assert text == """
+    array:
+      - fake-id # sb-var:project_id
+    """
+  assert count == 1
+
+  text = """
+    array:
+      - "not-replaced-yet" # sb-var:project_id
+      - "not-replaced-yet" # sb-var:project_id
+    """
+  text, count = replace_var_to_value("project_id", "fake-id", text)
+  assert text == """
+    array:
+      - "fake-id" # sb-var:project_id
+      - "fake-id" # sb-var:project_id
+    """
+  assert count == 2