Issue 141 - all-layers, s3 key multi region support and others (#149)

* ISSUE-141 Added multi options for s3 key checking

* ISSUE-141 Improved way to check DR

* ISSUE-1 added multi region support, tfautomv, s3 key autogeneration, layer region autodetection, region fallback system (layer dir, then account tfvars, then project tfvars)

* ISSUE-141 adding path to deploy beta and alpha images

* ISSUE-141 fixed python version

* ISSUE-141 added region var to tf command line; fixed var name in credentials

* ISSUE-141 set toolbox version

* ISSUE-141 imported tests from master

* ISSUE-141 fix to _find_region function

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Update leverage/modules/terraform.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Issue 123 all layers option (#151)

* ISSUE-123 added logic for init-multi command

* ISSUE-123 all terraform commands keeps running on layers but accept --layers list to run from account level

* ISSUE-141 persist s3 key when dynamically generated

* ISSUE-123 added logic for recording the s3 key in the tf file when automagically generated; removed hidden commands from tf commands

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* ISSUE-123 changed internal functions from click commands to regular functions; code refactor for sake of readability

* ISSUE-123 fixed test

* ISSUE-123 fixed makefile for testing

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Update leverage/container.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* Update leverage/modules/terraform.py

Co-authored-by: Angelo Fenoglio <[email protected]>

* ISSUE-123 fixed comments for PR 151

* ISSUE-123 removed comments

* ISSUE-123 fixed condition

* ISSUE-123 changed value sublayer to layers_group

Co-authored-by: Angelo Fenoglio <[email protected]>

* ISSUE-141 toolbox version updated

Co-authored-by: Angelo Fenoglio <[email protected]>

.github/workflows/build-push-test-package.yml

@@ -17,6 +17,8 @@ jobs:
       - uses: actions/checkout@v2
 
       - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
 
       - name: install_dependencies
         run: |
@@ -36,7 +38,7 @@ jobs:
             RELEASE_VERSION="$(echo $LATEST_VERSION | awk 'BEGIN{FS="."; OFS="."} {print $1,$2,$3+1}')rc.1"
 
           echo "[INFO] Checking Release Version (template 9.9.9-rc9)..."
-          ([ $(echo $RELEASE_VERSION | grep -oP "([0-9]*\.[0-9]*\.[0-9]*\-rc[0-9]+)") ] && echo "[INFO] Version ok" ) || (echo "[ERROR] Version is wrong" && exit 1)
+          ([ $(echo $RELEASE_VERSION | grep -oP "([0-9]*\.[0-9]*\.[0-9]*\-(rc|alpha|beta)[0-9]+)") ] && echo "[INFO] Version ok" ) || (echo "[ERROR] Version is wrong" && exit 1)
           echo "[INFO] Bump version to $RELEASE_VERSION"
           sed -i 's/'$CURRENT_VERSION'/'$RELEASE_VERSION'/' $INIT_FILE
         env:

.github/workflows/run-tests.yml

@@ -190,7 +190,7 @@ jobs:
           EOF
           echo "[INFO] Disable MFA\n"
           sed -i "s/^\(MFA_ENABLED=\)true/\1false/" build.env
-          sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.2.7-latest/' build.env;
+          sed -E -i 's/^TERRAFORM_IMAGE_TAG=.+$/TERRAFORM_IMAGE_TAG=1.2.7-0.0.5/' build.env;
         working-directory: ../theblairwitchproject
 
       - name: Test Testing Reference Architecture
@@ -213,3 +213,4 @@ jobs:
           printf "[INFO] Destroying all generated created resources\n"
           leverage tf destroy -auto-approve
         working-directory: ../theblairwitchproject/apps-devstg/global/cli-test-layer
+

Makefile

@@ -1,7 +1,7 @@
 .PHONY: help build
 LEVERAGE_TESTING_IMAGE := binbash/leverage-cli-testing
 LEVERAGE_TESTING_TAG   := 2.5.0
-LEVERAGE_IMAGE_TAG     := 1.2.7-latest
+LEVERAGE_IMAGE_TAG     := 1.2.7-0.0.5
 
 help:
 	@echo 'Available Commands:'

README.md

@@ -39,7 +39,7 @@ PROJECT=bb
 MFA_ENABLED=false
 
 # Terraform
-TERRAFORM_IMAGE_TAG=1.2.7-0.0.5
+TERRAFORM_IMAGE_TAG=1.2.7-0.1.0
 ```
 
 So, if you have created a project with version <1.8.0 and want to use it with version >=1.8.0 you should:

leverage/__init__.py

@@ -4,7 +4,7 @@
 #pylint: disable=wrong-import-position
 
 __version__ = "0.0.0"
-__toolbox_version__ = "1.2.7-0.0.5"
+__toolbox_version__ = "1.2.7-0.1.0"
 
 import sys
 from shutil import which

leverage/container.py

@@ -310,6 +310,29 @@ def exec(self, command="", *arguments):
         """
         return self._exec(command, *arguments)
 
+    def get_location_type(self):
+        """
+        Returns the location type:
+        - root
+        - account
+        - config
+        - layer
+        - sublayer
+        - not a project
+        """
+        if self.cwd == self.root_dir:
+            return 'root'
+        elif self.cwd == self.account_dir:
+            return 'account'
+        elif self.cwd in (self.common_config_dir, self.account_config_dir):
+            return 'config'
+        elif (self.cwd.as_posix().find(self.account_dir.as_posix()) >= 0) and list(self.cwd.glob("*.tf")):
+            return 'layer'
+        elif (self.cwd.as_posix().find(self.account_dir.as_posix()) >= 0) and not list(self.cwd.glob("*.tf")):
+            return 'layers-group'
+        else:
+            return 'not a project'
+
 
 class AWSCLIContainer(LeverageContainer):
     """ Leverage Container specially tailored to run AWS CLI commands. """
@@ -404,6 +427,8 @@ def __init__(self, client):
             Mount(source=(self.home / ".gitconfig").as_posix(), target="/etc/gitconfig", type="bind")
         ]
 
+        self._backend_key = None
+
         logger.debug(f"[bold cyan]Container configuration:[/bold cyan]\n{json.dumps(self.container_config, indent=2)}")
 
     def _guest_config_file(self, file):
@@ -435,7 +460,8 @@ def tf_default_args(self):
                                for common_file in self.common_config_dir.glob("*.tfvars")]
         account_config_files = [f"-var-file={self._guest_config_file(account_file)}"
                                 for account_file in self.account_config_dir.glob("*.tfvars")]
-        return common_config_files + account_config_files
+        region_settings      = [f"-var=\"region={self.region}\""]
+        return common_config_files + account_config_files + region_settings
 
     def enable_mfa(self):
         """ Enable Multi-Factor Authentication. """
@@ -452,6 +478,10 @@ def disable_authentication(self):
 
     def _check_sso_token(self):
         """ Check for the existence and validity of the SSO token to be used to get credentials. """
+
+        # Adding `token` file name to this function in order to
+        # meet the requirement regarding to have just one
+        # token file in the sso/cache
         sso_role = self.account_conf.get("sso_role")
         token_file = self.sso_cache / sso_role
 
@@ -461,9 +491,12 @@ def _check_sso_token(self):
             raise Exit(1)
 
         if token_file not in token_files:
-            logger.error("No valid AWS SSO token found for current account.\n"
-                         "Please log out and reconfigure SSO before proceeding.")
-            raise Exit(1)
+            sso_role = 'token'
+            token_file = self.sso_cache / sso_role
+            if token_file not in token_files:
+                logger.error("No valid AWS SSO token found for current account.\n"
+                            "Please log out and reconfigure SSO before proceeding.")
+                raise Exit(1)
 
         entrypoint = self.entrypoint
         self.entrypoint = ""
@@ -528,6 +561,16 @@ def exec(self, command, *arguments):
 
         return self._exec(command, *arguments)
 
+    def system_exec(self, command):
+        """ Momentarily override the container's default entrypoint. To run arbitrary commands and not only AWS CLI ones. """
+        original_entrypoint = self.entrypoint
+        self.entrypoint = ""
+        exit_code, output = self._exec(command)
+
+        self.entrypoint = original_entrypoint
+        return exit_code, output
+
+
     def start_shell(self):
         """ Launch a shell in the container. """
         if self.mfa_enabled or self.sso_enabled:
@@ -536,3 +579,124 @@ def start_shell(self):
         self.entrypoint = ""
         self._prepare_container()
         self._start()
+
+    @property
+    def region(self):
+        """ Determine the region based on PATH, account tfvars or project tfvars. Returns a string with the region name. """
+        region = self._find_region(self.cwd.relative_to(self.account_dir))
+
+        if not region is None:
+            return region
+
+        logger.error("No region found")
+        raise Exit(1)
+
+    @property
+    def terraform_backend(self):
+        """ Determine the terraform backend for the account. Returns an object with backend information. """
+        # These are the possible backend layer names for Refarch v1 and v2
+        possible_layer_names = ['terraform-backend', 'base-tf-backend']
+        for layer_name in possible_layer_names:
+            for directory in Path(self.account_dir).glob(f"**/{layer_name}"):
+
+                region = self._find_region(Path(directory).relative_to(self.account_dir))
+
+                if not region is None:
+                    return {'directory': directory , 'region': region}
+                else:
+                    logger.error("No region found for backend layer")
+                    raise Exit(1)
+
+        logger.error("No backend layer found")
+        raise Exit(1)
+
+    def _find_region(self, local_directory):
+        local_directory = local_directory.as_posix().split('/')
+        if len(local_directory) > 1:
+            if local_directory[0] != 'global':
+                return local_directory[0]
+
+        if 'region' in self.account_conf:
+            return self.account_conf.get('region')
+        if 'region' in self.common_conf:
+            return self.common_conf.get('region')
+        if 'region_primary' in self.account_conf:
+            return self.account_conf.get('region_primary')
+        if 'region_primary' in self.common_conf:
+            return self.common_conf.get('region_primary')
+
+        return None
+
+    def set_backend_key(self, skip_validation=False):
+        # Scenarios:
+        #
+        # scenario    |  s3 backend set   |  s3 key set  |  skip_validation  |  result
+        # 0           |  false            |  false       |  false            |  fail
+        # 1           |  false            |  false       |  true             |  ok
+        # 2           |  true             |  false       |  false/true       |  set the key
+        # 3           |  true             |  true        |  false/true       |  read the key
+        try:
+            config_tf_file = self.cwd / "config.tf"
+            config_tf = hcl2.loads(config_tf_file.read_text()) if config_tf_file.exists() else {}
+            if 'terraform' in config_tf and 'backend' in config_tf["terraform"][0] and 's3' in config_tf["terraform"][0]["backend"][0]:
+                if 'key' in config_tf["terraform"][0]["backend"][0]["s3"]:
+                    backend_key = config_tf["terraform"][0]["backend"][0]["s3"]["key"]
+                    self._backend_key = backend_key
+                else:
+                    self._backend_key = f"{self.cwd.relative_to(self.root_dir).as_posix()}/terraform.tfstate".replace('/base-','/').replace('/tools-','/')
+
+                    in_container_file_path = f"{self.guest_base_path}/{config_tf_file.relative_to(self.root_dir).as_posix()}"
+                    resp = self.system_exec("hcledit "
+                                             f"-f {in_container_file_path} -u"
+                                             f" attribute append terraform.backend.key \"\\\"{self._backend_key}\\\"\"")
+            else:
+                if not skip_validation:
+                    raise KeyError()
+        except (KeyError, IndexError):
+            logger.error("[red]✘[/red] Malformed [bold]config.tf[/bold] file. Missing Terraform backend bucket key.")
+            raise Exit(1)
+        except Exception as e:
+            logger.error("[red]✘[/red] Malformed [bold]config.tf[/bold] file. Unable to parse.")
+            logger.debug(e)
+            raise Exit(1)
+
+
+    @property
+    def backend_key(self):
+        return self._backend_key
+
+    @backend_key.setter
+    def backend_key(self, backend_key):
+        self._backend_key = backend_key
+
+
+class TFautomvContainer(TerraformContainer):
+    """ Leverage Container tailored to run general commands. """
+    TFAUTOMV_CLI_BINARY = '/usr/local/bin/tfautomv'
+
+    def __init__(self, client):
+        super().__init__(client)
+
+        self.environment['TF_CLI_ARGS_init'] = ' '.join(self.tf_default_args)
+        self.environment['TF_CLI_ARGS_plan'] = ' '.join(self.tf_default_args)
+
+        self.entrypoint = self.TFAUTOMV_CLI_BINARY
+
+        logger.debug(f"[bold cyan]Container configuration:[/bold cyan]\n{json.dumps(self.container_config, indent=2)}")
+
+    def start(self, *arguments):
+        self._prepare_container()
+
+        return self._start('', *arguments)
+
+    def start_in_layer(self, *arguments):
+        """ Run a command that can only be performed in layer level. """
+        self.check_for_layer_location()
+
+        return self.start(*arguments)
+
+    def exec(self, command, *arguments):
+        self._prepare_container()
+
+        return self._exec(command, *arguments)
+

leverage/leverage.py

@@ -13,6 +13,7 @@
 from leverage.modules import project
 from leverage.modules import terraform
 from leverage.modules import credentials
+from leverage.modules import tfautomv
 
 @click.group(invoke_without_command=True)
 @click.option("--filename", "-f",
@@ -53,3 +54,4 @@ def leverage(context, state, filename, list_tasks, verbose):
 leverage.add_command(terraform, name="tf")
 leverage.add_command(credentials)
 leverage.add_command(aws)
+leverage.add_command(tfautomv)

leverage/modules/__init__.py

@@ -3,3 +3,4 @@
 from .project import project
 from .terraform import terraform
 from .credentials import credentials
+from .tfautomv import tfautomv

leverage/modules/aws.py

@@ -87,7 +87,12 @@ def _sso(context, cli):
         logger.error("SSO configuration can only be performed at [bold]layer[/bold] level.")
         raise Exit(1)
 
-    default_region = cli.common_conf.get("region_primary", cli.common_conf.get("sso_region"))
+    # region_primary was added in refarch v1
+    # for v2 it was replaced by region at project level
+    region_primary = 'region_primary'
+    if not 'region_primary' in cli.common_conf:
+        region_primary = 'region'
+    default_region = cli.common_conf.get(region_primary, cli.common_conf.get("sso_region"))
     if default_region is None:
         logger.error("No primary region configured in global config file.")
         raise Exit(1)
@@ -111,18 +116,35 @@ def _sso(context, cli):
 
     current_account = cli.account_conf.get("environment")
     try:
+        # this is for refarch v1
         account_id = cli.common_conf.get("accounts").get(current_account).get("id")
     except AttributeError:
-        logger.error("Missing environment configuration in global config file.")
-        raise Exit(1)
+        # this is for refarch v2
+        try:
+            # this is for accounts with no org unit on top of it
+            account_id = cli.common_conf.get("organization").get("accounts").get(current_account).get("id")
+        except AttributeError:
+            try:
+                # this is for accounts with no org unit on top of it
+                found = False
+                for ou in cli.common_conf.get("organization").get("organizational_units"):
+                    if current_account in cli.common_conf.get("organization").get("organizational_units").get(ou).get("accounts"):
+                        account_id = cli.common_conf.get("organization").get("organizational_units").get(ou).get("accounts").get(current_account).get("id")
+                        found = True
+                        break
+                if not found:
+                    raise AttributeError
+            except AttributeError:
+                logger.error(f"Missing account configuration for [bold]{current_account}[/bold] in global config file.")
+                raise Exit(1)
     if not account_id:
         logger.error(f"Missing id for account [bold]{current_account}[/bold].")
         raise Exit(1)
 
     logger.info(f"Configuring [bold]{cli.project}-sso[/bold] profile.")
     sso_profile = {
         "sso_start_url": cli.common_conf.get("sso_start_url"),
-        "sso_region": cli.common_conf.get("sso_region", cli.common_conf.get("region_primary")),
+        "sso_region": cli.common_conf.get("sso_region", cli.common_conf.get(region_primary)),
         "sso_account_id": account_id,
         "sso_role_name": sso_role
     }
@@ -152,10 +174,13 @@ def sso(context, cli, args):
 @pass_container
 def login(cli):
     """ Login """
-    if (cli.cwd in (cli.root_dir, cli.account_dir) or
-        cli.account_dir.parent != cli.root_dir or
-        not list(cli.cwd.glob("*.tf"))):
-        logger.error("SSO configuration can only be performed at [bold]layer[/bold] level.")
+    # only from account or layer directories
+    # when to fail:
+    # - when this cond meets:
+    #   - no account dir
+    #   - no layer dir
+    if not cli.get_location_type() in ['account', 'layer', 'layers-group']:
+        logger.error("SSO configuration can only be performed at [bold]layer[/bold] or [bold]account[/bold] level.")
         raise Exit(1)
 
     exit_code, region = cli.exec(f"configure get sso_region --profile {cli.project}-sso")

leverage/modules/credentials.py

@@ -319,8 +319,13 @@ def _load_configs_for_credentials():
     config_values["project_name"] = (project_config.get("project_name")
                                          or terraform_config.get("project_long"))
 
-    config_values["primary_region"] = (project_config.get("primary_region")
-                                           or terraform_config.get("region_primary")
+    # region_primary was added in refarch v1
+    # for v2 it was replaced by region at project level
+    region_primary = 'region_primary'
+    if not 'region_primary' in project_config and not 'region_primary' in terraform_config:
+        region_primary = 'region'
+    config_values["primary_region"] = (project_config.get(region_primary)
+                                           or terraform_config.get(region_primary)
                                            or _ask_for_region())
     config_values["secondary_region"] = terraform_config.get("region_secondary")