[v2.0] Added cluster-wide tagging for Azure and AWS. (#3440)

hitachienergy · Feb 22, 2024 · b4d40c2 · b4d40c2
1 parent ee2f04f
commit b4d40c2
Show file tree

Hide file tree

Showing 46 changed files with 269 additions and 50 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -64,7 +64,7 @@ RUN : INSTALL PIP REQUIREMENTS \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
     --requirement /requirements.txt \
  && pip install --disable-pip-version-check --no-cache-dir --default-timeout=100 \
-    poetry pylint pylint_junit ansible-lint==6.5.0 ansible-lint-to-junit-xml yamllint pytest pytest_mock setuptools twine wheel
+    poetry pylint==2.* pylint_junit ansible-lint==6.5.0 ansible-lint-to-junit-xml yamllint pytest pytest_mock setuptools twine wheel
 
 RUN : SETUP USER AND OTHERS \
  && groupadd --gid $USER_GID $USERNAME \

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,39 +1,61 @@
 {
     "name": "epicli",
     "dockerFile": "Dockerfile",
-    "extensions": [
-        // Ansible
-        "redhat.ansible",
-        // Black Duck Scan
-        "synopsyscodesight.vscode-codesight",
-        // Common
-        "shardulm94.trailing-spaces",
-        // Git
-        "eamodio.gitlens",
-        // Jinja
-        "samuelcolvin.jinjahtml",
-        // Kubernetes
-        "ms-kubernetes-tools.vscode-kubernetes-tools",
-        // Markdown
-        "davidanson.vscode-markdownlint",
-        // Python
-        "ms-python.python",
-        // Ruby
-        "rebornix.Ruby",
-        "castwide.solargraph",
-        // Shell
-        "timonwong.shellcheck",
-        // Terraform
-        "hashicorp.terraform",
-        // YAML
-        "redhat.vscode-yaml",
-    ],
+    "customizations":{
+        "vscode": {
+            "extensions": [
+                // Ansible
+                "redhat.ansible",
+                // Common
+                "shardulm94.trailing-spaces",
+                // Jinja
+                "samuelcolvin.jinjahtml",
+                // Kubernetes
+                "ms-kubernetes-tools.vscode-kubernetes-tools",
+                // Markdown
+                "davidanson.vscode-markdownlint",
+                // Python
+                "ms-python.python",
+                "ms-python.pylint",
+                // Ruby
+                "rebornix.Ruby",
+                "castwide.solargraph",
+                // Shell
+                "timonwong.shellcheck",
+                // Terraform
+                "hashicorp.terraform",
+                // YAML
+                "redhat.vscode-yaml"
+            ],
+			"settings": {
+                // Common
+                "trailing-spaces.highlightCurrentLine": false, // highlighting currently edited line can be annoying
+                "trailing-spaces.syntaxIgnore": [
+                    "markdown" // "two spaces" syntax is for line breaks
+                ],
+
+                // Python
+                "pylint.enabled": true,
+                "pylint.args": [
+                    "--rcfile=${workspaceFolder}/.pylintrc"
+                ],
+                "python.testing.pytestEnabled": true,
+                "python.testing.unittestEnabled": false,
+                "python.envFile": "${workspaceFolder}/.devcontainer/python.env",
+                "python.pythonPath": "/usr/local/bin/python",
+
+                // Ruby
+                "ruby.format": false,
+                "ruby.lint": {
+                    "rubocop": true
+                },
+                "solargraph.diagnostics": false,
+                "solargraph.formatting": true
+			}
+        }
+    },
     "mounts": [
-        "source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh,target=/home/vscode/.ssh,type=bind",
+        "source=${localEnv:HOME}${localEnv:USERPROFILE}/.ssh,target=/home/vscode/.ssh,type=bind"
     ],
-    "settings": {
-        "python.envFile": "${workspaceFolder}/.devcontainer/python.env",
-        "python.pythonPath": "/usr/local/bin/python",
-    },
-    "remoteUser": "vscode",
+    "remoteUser": "vscode"
 }
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -6,7 +6,7 @@
     "configurations": [
         {
             "name": "epicli",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "${workspaceFolder}/cli/epicli.py",
             "cwd": "${workspaceFolder}",

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -6,11 +6,9 @@
     ],
 
     // Python
-    "python.linting.enabled": true,
-    "python.linting.pylintEnabled": true,
-    "python.linting.pylintArgs": [
-        "--rcfile",
-        "${workspaceFolder}/.pylintrc"
+    "pylint.enabled": true,
+    "pylint.args": [
+        "--rcfile=${workspaceFolder}/.pylintrc"
     ],
     "python.testing.pytestEnabled": true,
     "python.testing.unittestEnabled": false,

diff --git a/cli/src/providers/aws/InfrastructureBuilder.py b/cli/src/providers/aws/InfrastructureBuilder.py
@@ -23,6 +23,7 @@ def __init__(self, docs, manifest_docs=[]):
         self.cluster_prefix = self.cluster_model.specification.prefix.lower()
         self.use_network_security_groups = self.cluster_model.specification.cloud.network.use_network_security_groups
         self.use_public_ips = self.cluster_model.specification.cloud.use_public_ips
+        self.global_tags = self.cluster_model.specification.cloud.tags
         self.docs = docs
         self.manifest_docs = manifest_docs
 
@@ -143,6 +144,7 @@ def get_vm(self, component_key, vm_config, subnet, public_key_config, security_g
         vm.specification.key_name = public_key_config.specification.key_name
         vm.specification.use_network_security_groups = self.use_network_security_groups
         vm.specification.availability_zone = subnet.specification.availability_zone
+        vm.specification.tags.extend(self.global_tags)
         if self.use_network_security_groups:
             vm.specification.security_groups = [security_group.specification.name]
         vm.specification.associate_public_ip = self.cluster_model.specification.cloud.use_public_ips

diff --git a/cli/src/providers/azure/InfrastructureBuilder.py b/cli/src/providers/azure/InfrastructureBuilder.py
@@ -25,6 +25,7 @@ def __init__(self, docs, manifest_docs=[]):
         self.region = self.cluster_model.specification.cloud.region
         self.use_network_security_groups = self.cluster_model.specification.cloud.network.use_network_security_groups
         self.use_public_ips = self.cluster_model.specification.cloud.use_public_ips
+        self.global_tags = self.cluster_model.specification.cloud.tags
         self.docs = docs
         self.manifest_docs = manifest_docs
 
@@ -155,18 +156,21 @@ def get_resource_group(self):
         resource_group = self.get_config_or_default(self.docs, 'infrastructure/resource-group')
         resource_group.specification.name = self.resource_group_name
         resource_group.specification.region = self.cluster_model.specification.cloud.region
+        resource_group.specification.tags.extend(self.global_tags)
         return resource_group
 
     def get_virtual_network(self):
         vnet = self.get_config_or_default(self.docs, 'infrastructure/vnet')
         vnet.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'vnet')
         vnet.specification.address_space = self.cluster_model.specification.cloud.vnet_address_pool
+        vnet.specification.tags.extend(self.global_tags)
         return vnet
 
     def get_network_security_group(self, component_key, security_rules,  index):
         security_group = self.get_config_or_default(self.docs, 'infrastructure/network-security-group')
         security_group.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'nsg' + '-' + str(index), component_key)
         security_group.specification.rules = security_rules
+        security_group.specification.tags.extend(self.global_tags)
         return security_group
 
     def get_subnet(self, subnet_definition, component_key, index):
@@ -183,6 +187,7 @@ def get_availability_set(self, availability_set_name):
         )
         if availability_set is not None:
             availability_set.specification.name = resource_name(self.cluster_prefix, self.cluster_name, availability_set_name + '-' + 'aset')
+            availability_set.specification.tags.extend(self.global_tags)
         return availability_set
 
     def get_subnet_network_security_group_association(self, component_key, subnet_name, security_group_name, index):
@@ -209,6 +214,7 @@ def get_network_interface(self, component_key, vm_config, subnet_name, public_ip
         network_interface.specification.use_public_ip = self.cluster_model.specification.cloud.use_public_ips
         network_interface.specification.public_ip_name = public_ip_name
         network_interface.specification.enable_accelerated_networking = vm_config.specification.network_interface.enable_accelerated_networking
+        network_interface.specification.tags.extend(self.global_tags)
         return network_interface
 
     def get_public_ip(self, component_key, vm_config, index):
@@ -217,12 +223,14 @@ def get_public_ip(self, component_key, vm_config, index):
         public_ip.specification.allocation_method = vm_config.specification.network_interface.public_ip.allocation_method
         public_ip.specification.idle_timeout_in_minutes = vm_config.specification.network_interface.public_ip.idle_timeout_in_minutes
         public_ip.specification.sku = vm_config.specification.network_interface.public_ip.sku
+        public_ip.specification.tags.extend(self.global_tags)
         return public_ip
 
     def get_storage_share_config(self):
         storage_share = self.get_config_or_default(self.docs, 'infrastructure/storage-share')
         storage_share.specification.name = resource_name(self.cluster_prefix, self.cluster_name, 'k8s-ss')
         storage_share.specification.storage_account_name = storage_account_name(self.cluster_prefix, self.cluster_name, 'k8s')
+        storage_share.specification.tags.extend(self.global_tags)
         return storage_share
 
     def get_vm(self, component_key, alt_component_name, vm_config, availability_set, network_interface_name, security_group_association_name, index):
@@ -241,6 +249,7 @@ def get_vm(self, component_key, alt_component_name, vm_config, availability_set,
         vm.specification.security_group_association_name = security_group_association_name
         vm.specification.tags.append({'cluster': cluster_tag(self.cluster_prefix, self.cluster_name)})
         vm.specification.tags.append({component_key: ''})
+        vm.specification.tags.extend(self.global_tags)
         if vm_config.specification.os_type == 'windows':
             raise NotImplementedError('Windows VMs not supported jet.')
         pub_key_path = self.cluster_model.specification.admin_user.key_path + '.pub'

diff --git a/docs/changelogs/CHANGELOG-2.0.md b/docs/changelogs/CHANGELOG-2.0.md
@@ -1,6 +1,12 @@
 <!-- markdownlint-disable-file no-duplicate-header -->
 # Changelog 2.0
 
+## [2.0.12] 2024-xx-xx
+
+### Added
+
+- [#3437](https://github.com/hitachienergy/epiphany/issues/3437) - Cluster-wide tagging for all cloud resources
+
 ## [2.0.11] 2024-01-11
 
 ### Fixed

diff --git a/docs/home/HOWTO.md b/docs/home/HOWTO.md
@@ -27,6 +27,7 @@
   - [How to use Azure availability sets](./howto/CLUSTER.md#how-to-use-azure-availability-sets)
   - [Downloading offline requirements with a Docker container](./howto/CLUSTER.md#downloading-offline-requirements-with-a-docker-container)
   - [How to additional custom Terraform templates](./howto/CLUSTER.md#how-to-additional-custom-terraform-templates)
+  - [How to add cluster-wide tagging](./howto/CLUSTER.md#how-to-add-cluster-wide-tagging)
 
 - [Monitoring](./howto/MONITORING.md)
   - [How to enable provided Prometheus rules](./howto/MONITORING.md#how-to-enable-provided-prometheus-rules)

diff --git a/docs/home/howto/CLUSTER.md b/docs/home/howto/CLUSTER.md
@@ -1120,3 +1120,42 @@ And will be placed in the following folder:
 When Epicli is run/re-run any Terraform scripts which will start with the ```xxx_*.tf``` filter  will be removed and regenerated. The user can make custom Terraform scripts and place them allongside the Epicli generated ones and these will be applied/re-applied during the Epicli run.
 
 If you need to define any additional security rules for component subnets for custom infrastructure you can check the documentation [here](./SECURITY_GROUPS.md).
+
+## How to add cluster-wide tagging
+
+Epiphany supports the use of cluster-wide tagging of all resources that are beeing created in `AWS` and `Azure`. These can used to keep track of resources, billing and cost managment or other automation that involves tags.
+
+Cluster-wide tags can be added to the `cloud` section of the main Epiphany cluster document like so:
+
+```yaml
+---
+kind: epiphany-cluster
+title: Epiphany cluster Config
+name: default
+specification:
+  ...
+  cloud:
+    tags:
+      tag1: 'First-tag'
+      tag2: 'Second-tag'
+      tag3: 'Third-tag'
+      ...
+  ...
+```
+
+A few tags should not be used as these are used by Epiphany internally. 
+
+For Azure:
+- `cluster` as key with cluster name as value
+- `<component-name>` as key with empty value
+
+For AWS:
+- `cluster_name` as key with cluster name as value
+- `component-key` as key with componenent name as value
+
+Currently there are some open issues with both the `AWS` and `Azure` providers which prevent updating/changes of tags on certain VM attached storages:
+
+- [Azure](https://github.com/hashicorp/terraform-provider-azurerm/issues/2568)
+- [AWS](https://github.com/hashicorp/terraform-provider-aws/issues/19406)
+
+*Note: Duplicate tags are overwritten with the latest one defined in the configuration*
diff --git a/schema/aws/defaults/configuration/minimal-cluster-config.yml b/schema/aws/defaults/configuration/minimal-cluster-config.yml
@@ -16,6 +16,7 @@ specification:
       access_key_id: XXXX-XXXX-XXXX
       secret_access_key: XXXXXXXXXXXXXXXX
     default_os_image: default
+    tags: []
   components:
     repository:
       count: 1

diff --git a/schema/aws/defaults/epiphany-cluster.yml b/schema/aws/defaults/epiphany-cluster.yml
@@ -21,6 +21,7 @@ specification:
     network:
       use_network_security_groups: True
     default_os_image: default
+    tags: []
   components:
     kubernetes_master:
       count: 1

diff --git a/schema/aws/defaults/infrastructure/virtual-machine.yml b/schema/aws/defaults/infrastructure/virtual-machine.yml
@@ -20,6 +20,7 @@ specification:
   os_full_name: ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20231004
   os_type: linux
   ebs_optimized: false
+  tags: []
   disks:
     root: # Updating this section on existing machine will cause resource replacement
       volume_type: gp2

diff --git a/schema/aws/validation/infrastructure/virtual-machine.yml b/schema/aws/validation/infrastructure/virtual-machine.yml
@@ -38,6 +38,10 @@ properties:
     type: string
   ebs_optimized:
     type: boolean
+  tags:
+    type: array
+    items:
+      items: {}
   disks:
     type: object
     properties:

diff --git a/schema/azure/defaults/configuration/minimal-cluster-config.yml b/schema/azure/defaults/configuration/minimal-cluster-config.yml
@@ -20,6 +20,7 @@ specification:
       subscriptionId: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxx"
     use_public_ips: False # When not using public IPs you have to provide connectivity via private IPs (VPN)
     default_os_image: default
+    tags: [] # Tags that will be added to every resource
   components:
     repository:
       count: 1

diff --git a/schema/azure/defaults/epiphany-cluster.yml b/schema/azure/defaults/epiphany-cluster.yml
@@ -25,6 +25,7 @@ specification:
       use_network_security_groups: True
     default_os_image: default
     hostname_domain_extension: '' # Domain name that will be added to every machines hostname with a . seperator
+    tags: [] # Tags that will be added to every resource
   components:
     kubernetes_master:
       count: 1

diff --git a/schema/azure/defaults/infrastructure/availability-set.yml b/schema/azure/defaults/infrastructure/availability-set.yml
@@ -8,3 +8,4 @@ specification:
     platform_fault_domain_count: 2
     platform_update_domain_count: 5
     managed: true
+    tags: []
diff --git a/schema/azure/defaults/infrastructure/network-interface.yml b/schema/azure/defaults/infrastructure/network-interface.yml
@@ -12,3 +12,4 @@ specification:
     use_public_ip: SET_BY_AUTOMATION
     public_ip_name: SET_BY_AUTOMATION
     enable_accelerated_networking: SET_BY_AUTOMATION
+    tags: []
diff --git a/schema/azure/defaults/infrastructure/network-security-group.yml b/schema/azure/defaults/infrastructure/network-security-group.yml
@@ -6,3 +6,4 @@ name: default
 specification:
   name: SET_BY_AUTOMATION
   rules: []
+  tags: []
diff --git a/schema/azure/defaults/infrastructure/public-ip.yml b/schema/azure/defaults/infrastructure/public-ip.yml
@@ -8,3 +8,4 @@ specification:
   allocation_method: SET_BY_AUTOMATION
   idle_timeout_in_minutes: SET_BY_AUTOMATION
   sku: SET_BY_AUTOMATION
+  tags: []
diff --git a/schema/azure/defaults/infrastructure/resource-group.yml b/schema/azure/defaults/infrastructure/resource-group.yml
@@ -6,3 +6,4 @@ name: default
 specification:
   name: SET_BY_AUTOMATION
   region: SET_BY_AUTOMATION
+  tags: []
diff --git a/schema/azure/defaults/infrastructure/storage-share.yml b/schema/azure/defaults/infrastructure/storage-share.yml
@@ -7,3 +7,4 @@ specification:
   name: SET_BY_AUTOMATION
   storage_account_name: SET_BY_AUTOMATION
   quota: 50
+  tags: []
diff --git a/schema/azure/defaults/infrastructure/vnet.yml b/schema/azure/defaults/infrastructure/vnet.yml
@@ -6,3 +6,4 @@ name: default
 specification:
   name: SET_BY_AUTOMATION
   address_space: SET_BY_AUTOMATION
+  tags: []
diff --git a/schema/azure/validation/infrastructure/availability-set.yml b/schema/azure/validation/infrastructure/availability-set.yml
@@ -19,3 +19,7 @@ properties:
   managed:
     "$id": "#/properties/specification/properties/managed"
     type: boolean
+  tags:
+    type: array
+    items:
+      items: {}
diff --git a/schema/azure/validation/infrastructure/network-interface.yml b/schema/azure/validation/infrastructure/network-interface.yml
@@ -20,3 +20,7 @@ properties:
     type: string
   enable_accelerated_networking:
     type: boolean
+  tags:
+    type: array
+    items:
+      items: {}
diff --git a/schema/azure/validation/infrastructure/network-security-group.yml b/schema/azure/validation/infrastructure/network-security-group.yml
@@ -31,3 +31,7 @@ properties:
           type: string
         destination_address_prefix:
           type: string
+  tags:
+    type: array
+    items:
+      items: {}
diff --git a/schema/azure/validation/infrastructure/public-ip.yml b/schema/azure/validation/infrastructure/public-ip.yml
@@ -12,3 +12,7 @@ properties:
     type: integer
   sku:
     type: string
+  tags:
+    type: array
+    items:
+      items: {}