[Test] Added Templates to test cluster upgrade

Added new aws and azure clustertemplates to test
managed cluster upgrades.

templates/cluster/aws-standalone-cp-0-0-2/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

templates/cluster/aws-standalone-cp-0-0-2/Chart.yaml

@@ -0,0 +1,23 @@
+apiVersion: v2
+name: aws-standalone-cp
+description: |
+  An HMC template to deploy a k0s cluster on AWS with bootstrapped control plane nodes.
+  Upgrade sequences: 
+  * 0.0.1 -> 0.0.2
+    Diff:
+    * aws-ebs-csi-driver: bumped 2.33.0 -> 2.35.1
+    * aws-cloud-controller-manager: enabled more verbose logging (v=2 -> v=10)
+type: application
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.2
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.31.1+k0s.0"
+annotations:
+  hmc.mirantis.com/infrastructure-providers: aws
+  hmc.mirantis.com/control-plane-providers: k0s
+  hmc.mirantis.com/bootstrap-providers: k0s

templates/cluster/aws-standalone-cp-0-0-2/README.md

@@ -0,0 +1,11 @@
+## Install applications into Target Cluster
+
+To install applications into the target cluster created using Cluster API (CAPI) upon creation, a Flux `HelmRelease` object is to be made such that its `.spec.KubeConfig` references the kubeconfig of the target cluster.
+
+**Reference:** https://fluxcd.io/flux/components/helm/helmreleases/#remote-clusters--cluster-api
+
+This chart/template already defines the following applications under `templates/beachheadservices` which can be be installed into the target cluster by setting `.Values.installBeachHeadServices=true`:
+1. cert-manager
+2. nginx-ingress
+
+**Important:** The Flux objects added to `templates/beachheadservices` to install custom applications must have the `hmc.mirantis.com/managed: "true"` label to be reconciled by HMC.

templates/cluster/aws-standalone-cp-0-0-2/templates/_helpers.tpl

@@ -0,0 +1,23 @@
+{{- define "cluster.name" -}}
+    {{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{- define "awsmachinetemplate.controlplane.name" -}}
+    {{- include "cluster.name" . }}-cp-mt
+{{- end }}
+
+{{- define "awsmachinetemplate.worker.name" -}}
+    {{- include "cluster.name" . }}-worker-mt
+{{- end }}
+
+{{- define "k0scontrolplane.name" -}}
+    {{- include "cluster.name" . }}-cp
+{{- end }}
+
+{{- define "k0sworkerconfigtemplate.name" -}}
+    {{- include "cluster.name" . }}-machine-config
+{{- end }}
+
+{{- define "machinedeployment.name" -}}
+    {{- include "cluster.name" . }}-md
+{{- end }}

templates/cluster/aws-standalone-cp-0-0-2/templates/awscluster.yaml

@@ -0,0 +1,26 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+kind: AWSCluster
+metadata:
+  name: {{ include "cluster.name" . }}
+  annotations:
+    aws.cluster.x-k8s.io/external-resource-gc: "true"
+spec:
+  region: {{ .Values.region }}
+  identityRef:
+    kind: {{ .Values.clusterIdentity.kind }}
+    name: {{ .Values.clusterIdentity.name }}
+  controlPlaneLoadBalancer:
+    healthCheckProtocol: TCP
+  network:
+    additionalControlPlaneIngressRules:
+      - description: "k0s controller join API"
+        protocol: tcp
+        fromPort: 9443
+        toPort: 9443
+  {{- if not (quote .Values.sshKeyName | empty) }}
+  sshKeyName: {{ .Values.sshKeyName | quote }}
+  {{- end }}
+  {{- with .Values.bastion }}
+  bastion:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}

templates/cluster/aws-standalone-cp-0-0-2/templates/awsmachinetemplate-controlplane.yaml

@@ -0,0 +1,24 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+kind: AWSMachineTemplate
+metadata:
+  name: {{ include "awsmachinetemplate.controlplane.name" . }}
+spec:
+  template:
+    spec:
+      {{- if not (quote .Values.controlPlane.amiID | empty) }}
+      ami:
+        id: {{ .Values.controlPlane.amiID }}
+      {{- end }}
+      imageLookupFormat: {{ .Values.controlPlane.imageLookup.format }}
+      imageLookupOrg: "{{ .Values.controlPlane.imageLookup.org }}"
+      imageLookupBaseOS: {{ .Values.controlPlane.imageLookup.baseOS }}
+      instanceType: {{ .Values.controlPlane.instanceType }}
+      # Instance Profile created by `clusterawsadm bootstrap iam create-cloudformation-stack`
+      iamInstanceProfile: {{ .Values.controlPlane.iamInstanceProfile }}
+      cloudInit:
+        # Makes CAPA use k0s bootstrap cloud-init directly and not via SSM
+        # Simplifies the VPC setup as we do not need custom SSM endpoints etc.
+        insecureSkipSecretsManager: true
+      publicIP: {{ .Values.publicIP }}
+      rootVolume:
+        size: {{ .Values.controlPlane.rootVolumeSize }}

templates/cluster/aws-standalone-cp-0-0-2/templates/awsmachinetemplate-worker.yaml

@@ -0,0 +1,24 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+kind: AWSMachineTemplate
+metadata:
+  name: {{ include "awsmachinetemplate.worker.name" . }}
+spec:
+  template:
+    spec:
+      {{- if not (quote .Values.worker.amiID | empty) }}
+      ami:
+        id: {{ .Values.worker.amiID }}
+      {{- end }}
+      imageLookupFormat: {{ .Values.worker.imageLookup.format }}
+      imageLookupOrg: "{{ .Values.worker.imageLookup.org }}"
+      imageLookupBaseOS: {{ .Values.worker.imageLookup.baseOS }}
+      instanceType: {{ .Values.worker.instanceType }}
+      # Instance Profile created by `clusterawsadm bootstrap iam create-cloudformation-stack`
+      iamInstanceProfile: {{ .Values.worker.iamInstanceProfile }}
+      cloudInit:
+        # Makes CAPA use k0s bootstrap cloud-init directly and not via SSM
+        # Simplifies the VPC setup as we do not need custom SSM endpoints etc.
+        insecureSkipSecretsManager: true
+      publicIP: {{ .Values.publicIP }}
+      rootVolume:
+        size: {{ .Values.worker.rootVolumeSize }}

templates/cluster/aws-standalone-cp-0-0-2/templates/beachheadservices/cert-manager.yaml

@@ -0,0 +1,42 @@
+{{- if .Values.installBeachHeadServices }}
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: cert-manager
+  labels:
+    hmc.mirantis.com/managed: "true"
+spec:
+  interval: 24h
+  url: https://charts.jetstack.io
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ include "cluster.name" . }}-cert-manager
+  labels:
+    hmc.mirantis.com/managed: "true"
+spec:
+  chart:
+    metadata:
+      labels:
+        hmc.mirantis.com/managed: "true"
+    spec:
+      chart: cert-manager
+      version: "v1.12.3"
+      sourceRef:
+        kind: HelmRepository
+        name: cert-manager
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  interval: 10m
+  kubeConfig:
+    secretRef:
+      name: {{ include "cluster.name" . }}-kubeconfig
+  releaseName: cert-manager
+  targetNamespace: cert-manager
+  storageNamespace: cert-manager
+  values:
+    installCRDs: true
+{{- end }}

templates/cluster/aws-standalone-cp-0-0-2/templates/beachheadservices/nginx-ingress.yaml

@@ -0,0 +1,36 @@
+{{- if .Values.installBeachHeadServices }}
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: OCIRepository
+metadata:
+  name: nginx-ingress
+  labels:
+    hmc.mirantis.com/managed: "true"
+spec:
+  interval: 24h
+  url: oci://ghcr.io/nginxinc/charts/nginx-ingress
+  ref:
+    semver: "1.3.2"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: {{ include "cluster.name" . }}-nginx-ingress
+  labels:
+    hmc.mirantis.com/managed: "true"
+spec:
+  targetNamespace: nginx-ingress
+  storageNamespace: nginx-ingress
+  kubeConfig:
+    secretRef:
+      name: {{ include "cluster.name" . }}-kubeconfig
+  interval: 10m
+  chartRef:
+    kind: OCIRepository
+    name: nginx-ingress
+  install:
+    createNamespace: true
+    remediation:
+      retries: -1
+  values:
+    fullnameOverride: nginx-ingress
+{{- end }}

templates/cluster/aws-standalone-cp-0-0-2/templates/cluster.yaml

@@ -0,0 +1,17 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  name: {{ include "cluster.name" . }}
+spec:
+  {{- with .Values.clusterNetwork }}
+  clusterNetwork:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  controlPlaneRef:
+    apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+    kind: K0sControlPlane
+    name: {{ include "k0scontrolplane.name" .  }}
+  infrastructureRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+    kind: AWSCluster
+    name: {{ include "cluster.name" . }}

templates/cluster/aws-standalone-cp-0-0-2/templates/k0scontrolplane.yaml

@@ -0,0 +1,65 @@
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: K0sControlPlane
+metadata:
+  name: {{ include "k0scontrolplane.name" . }}
+spec:
+  replicas: {{ .Values.controlPlaneNumber }}
+  version: {{ .Values.k0s.version }}
+  updateStrategy: {{ .Values.k0s.updateStrategy }}
+  k0sConfigSpec:
+    args:
+      - --enable-worker
+      - --enable-cloud-provider
+      - --kubelet-extra-args="--cloud-provider=external"
+      - --disable-components=konnectivity-server
+    k0s:
+      apiVersion: k0s.k0sproject.io/v1beta1
+      kind: ClusterConfig
+      metadata:
+        name: k0s
+      spec:
+        api:
+          extraArgs:
+            anonymous-auth: "true"
+        network:
+          provider: calico
+          calico:
+            mode: ipip
+        extensions:
+          helm:
+            repositories:
+              - name: aws-cloud-controller-manager
+                url: https://kubernetes.github.io/cloud-provider-aws
+              - name: aws-ebs-csi-driver
+                url: https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+            charts:
+              - name: aws-cloud-controller-manager
+                namespace: kube-system
+                chartname: aws-cloud-controller-manager/aws-cloud-controller-manager
+                version: "0.0.8"
+                values: |
+                  nodeSelector:
+                    node-role.kubernetes.io/control-plane: "true"
+                  image:
+                    tag: v1.30.3
+                  args:
+                    - --v=10
+                    - --cloud-provider=aws
+                    - --cluster-cidr={{ first .Values.clusterNetwork.pods.cidrBlocks }}
+                    - --allocate-node-cidrs=true
+                    - --cluster-name={{ include "cluster.name" . }}
+              - name: aws-ebs-csi-driver
+                namespace: kube-system
+                chartname: aws-ebs-csi-driver/aws-ebs-csi-driver
+                version: 2.35.1
+                values: |
+                  defaultStorageClass:
+                    enabled: true
+                  node:
+                    kubeletPath: /var/lib/k0s/kubelet
+  machineTemplate:
+    infrastructureRef:
+      apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+      kind: AWSMachineTemplate
+      name: {{ include "awsmachinetemplate.controlplane.name" . }}
+      namespace: {{ .Release.Namespace }}

templates/cluster/aws-standalone-cp-0-0-2/templates/k0sworkerconfigtemplate.yaml

@@ -0,0 +1,11 @@
+apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+kind: K0sWorkerConfigTemplate
+metadata:
+  name: {{ include "k0sworkerconfigtemplate.name" . }}
+spec:
+  template:
+    spec:
+      version: {{ .Values.k0s.version }}
+      args:
+      - --enable-cloud-provider
+      - --kubelet-extra-args="--cloud-provider=external"

templates/cluster/aws-standalone-cp-0-0-2/templates/machinedeployment.yaml

@@ -0,0 +1,26 @@
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  name: {{ include "machinedeployment.name" . }}
+spec:
+  clusterName: {{ include "cluster.name" . }}
+  replicas: {{ .Values.workersNumber }}
+  selector:
+    matchLabels:
+      cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }}
+  template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/cluster-name: {{ include "cluster.name" . }}
+    spec:
+      version: {{ regexReplaceAll "\\+k0s.+$" .Values.k0s.version "" }}
+      clusterName: {{ include "cluster.name" . }}
+      bootstrap:
+        configRef:
+          apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
+          kind: K0sWorkerConfigTemplate
+          name: {{ include "k0sworkerconfigtemplate.name" . }}
+      infrastructureRef:
+        apiVersion: infrastructure.cluster.x-k8s.io/v1beta2
+        kind: AWSMachineTemplate
+        name: {{ include "awsmachinetemplate.worker.name" . }}