Merge pull request #301 from wireapp/release_2020_06_26

release 2020-06-26

CHANGELOG.md

@@ -1,3 +1,22 @@
+# 2020-06-26
+
+## Features
+
+* [charts] introduce cert-manager support in `nginx-ingress-services` to automate TLS certificate 
+  issuing. Please refer to the [docs](https://docs.wire.com/how-to/install/helm.html#how-to-direct-traffic-to-your-cluster)
+  or the issue [#280](https://github.com/wireapp/wire-server-deploy/pull/280) for more details.
+
+## Bug Fixes
+
+* [charts] Update frontend apps version: webapp, team-settings, due to a broken team-settings version (#300)  
+
+## Internal Changes
+
+* cleanup scripts used in automation (#300)
+* ongoing work in several Terraform modules: ingress, CORS, cargohold
+
+For more information, please refer to the [diff](https://github.com/wireapp/wire-server-deploy/compare/v2020-06-19...v2020-06-26) 
+
 # 2020-06-19
 
 ## Features

bin/sync.sh

@@ -49,21 +49,13 @@ else
     charts=( $(find $CHART_DIR/ -maxdepth 1 -type d | sed -n "s=$CHART_DIR/\(.\+\)=\1 =p") )
 fi
 
-# install s3 plugin
+# install s3 plugin if not present
 # See https://github.com/hypnoglow/helm-s3/pull/56 for reason to use fork
 s3_plugin_version=$(helm plugin list | grep "^s3 " | awk '{print $2}' || true)
 if [[ $s3_plugin_version != "0.9.0" ]]; then
     echo "not version 0.9.0 from steven-sheehy fork, upgrading or installing plugin..."
     helm plugin remove s3 || true
     helm plugin install https://github.com/steven-sheehy/helm-s3.git --version v0.9.0
-else
-    # double check we have the right version of the s3 plugin
-    plugin_sha=$(cat $HOME/.helm/plugins/helm-s3.git/.git/HEAD)
-    if [[ $plugin_sha != "f7ab4a8818f11380807da45a6c738faf98106d62" ]]; then
-        echo "git hash doesn't match forked s3-plugin version (or maybe there is a path issue and your plugins are installed elsewhere? Attempting to re-install..."
-        helm plugin remove s3
-        helm plugin install https://github.com/steven-sheehy/helm-s3.git --version v0.9.0
-    fi
 fi
 
 # index/sync charts to S3

bin/update.sh

@@ -6,9 +6,6 @@ USAGE="download and bundle dependent helm charts: $0 <chart-directory>"
 dir=${1:?$USAGE}
 
 
-# nothing serves on localhost, remove that repo
-helm repo remove local 2&> /dev/null || true
-
 # hacky workaround for helm's lack of recursive dependency update
 # See https://github.com/helm/helm/issues/2247
 helmDepUp () {

charts/nginx-ingress-services/README.md

@@ -18,12 +18,42 @@ secrets:
     -----END PRIVATE KEY-----
 ```
 
-or encrypted with `sops` and then use `helm-wrapper`.
+or encrypted with `sops` and then use `helm-wrapper`. As an alternative, it is possible to use `cert-manager` (see further down below).
 
 Have a look at the [values file](values.yaml) for different configuration options.
 
-# Common issues
+### Common issues
 
 Q: My ingress keeps serving "Kubernetes Ingress Controller Fake Certificate"!!
 
 A: Ensure that your certificate is _valid_ and has _not expired_; trying to serve expired certificates will silently fail and the nginx ingress will simply fallback to the default certificate.
+
+
+## About cert-manager
+
+### Prerequisites
+
+* `cert-manager` and its CRDs have to be installed upfront, 
+   e.g. `helm upgrade --install -n cert-manager-ns --set 'installCRDs=true' cert-manager jetstack/cert-manager`,
+   because upstream decided that this is the way (https://github.com/jetstack/cert-manager/pull/2964)
+
+
+### What does this chart do?
+
+* define `Ingress` for various services and their corresponding FQDNS
+* do TLS termination either by explicitly providing a wildcard certificate or letting
+  *cert-manager* take care of this
+* [optional] configure an *Issuer* to issue ACME HTTP01 certificates provided by Letsencrypt
+* [optional] define a *Certificate* representation that causes *cert-manager* to issue a
+  certificate that is then used by `Ingress` 
+
+
+### Todo when introducing support for K8s >= 1.15 
+
+* the `apiVersion` of all resources based on cert-manager's CRDs, namely `./templates/issuer.yaml` and 
+  `./templates/certificate.yaml`, has to be changed to `cert-manager.io/v1alpha3`
+
+
+### Monitoring
+
+__FUTUREWORK:__ When `wire-server-metrics` is ready, expiration & renewal should be integrated into monitoring.

charts/nginx-ingress-services/templates/_helpers.tpl

@@ -14,3 +14,38 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
 {{- $name := default .Chart.Name .Values.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
+
+{{/*
+Determine DNS zone based on one of the given FQDNs
+*/}}
+{{- define "nginx-ingress-services.zone" -}}
+{{- $zones := splitList "." .Values.config.dns.https -}}
+{{- slice $zones 1 | join "." -}}
+{{- end -}}
+
+{{/*
+Generate the secrate name in a conistent way, since it's referred to in multiple places, while
+at the same time being used for distinct scenarios
+*/}}
+{{- define "nginx-ingress-services.getCertificateSecretName" -}}
+{{- $nameParts := list (include "nginx-ingress-services.fullname" .) -}}
+{{- if .Values.tls.useCertManager -}}
+    {{- $nameParts = append $nameParts "managed" -}}
+{{- else -}}
+    {{- $nameParts = append $nameParts "wildcard" -}}
+{{- end -}}
+{{- $nameParts = append $nameParts "tls-certificate" -}}
+{{- join "-" $nameParts -}}
+{{- end -}}
+
+{{/*
+Returns the Letsencrypt API server URL based on whether testMode is enabled or disabled
+*/}}
+{{- define "certificate-manager.apiServerURL" -}}
+{{- $hostnameParts := list "acme" -}}
+{{- if .Values.certManager.inTestMode -}}
+    {{- $hostnameParts = append $hostnameParts "staging" -}}
+{{- end -}}
+{{- $hostnameParts = append $hostnameParts "v02" -}}
+{{- join "-" $hostnameParts | printf "https://%s.api.letsencrypt.org/directory" -}}
+{{- end -}}

charts/nginx-ingress-services/templates/certificate.yaml

@@ -0,0 +1,40 @@
+{{- if and .Values.tls.enabled .Values.tls.useCertManager -}}
+apiVersion: cert-manager.io/v1alpha2
+kind: Certificate
+metadata:
+  name: "{{ include "nginx-ingress-services.zone" . | replace "." "-" }}-csr"
+  namespace: {{ .Release.Namespace }}
+  docs: "https://cert-manager.io/docs/usage/certificate"
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  issuerRef:
+    name: letsencrypt-http01
+    kind: Issuer
+  usages:
+    - server auth
+  duration: 2160h     # 90d, Letsencrypt default; NOTE: changes are ignored by Letsencrypt
+  renewBefore: 360h   # 15d
+  isCA: false
+  keyAlgorithm: ecdsa
+  keySize: 384        # 521 is not supported by Letsencrypt
+  keyEncoding: pkcs1
+  secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
+  # NOTE: disabled due to https://github.com/jetstack/cert-manager/issues/2978
+  # TODO: enable when fixed (probably when cert-manager:v0.16 released)
+  #privateKey:
+  #  rotationPolicy: Always
+  dnsNames:
+    - {{ .Values.config.dns.https }}
+    - {{ .Values.config.dns.ssl }}
+    - {{ .Values.config.dns.webapp }}
+    - {{ .Values.config.dns.fakeS3 }}
+    {{- if .Values.teamSettings.enabled }}
+    - {{ .Values.config.dns.teamSettings }}
+    {{- end }}
+    {{- if .Values.accountPages.enabled }}
+    - {{ .Values.config.dns.accountPages }}
+    {{- end }}
+{{- end -}}

charts/nginx-ingress-services/templates/ingress.yaml

@@ -20,7 +20,7 @@ spec:
 {{- if .Values.accountPages.enabled }}
       - {{ .Values.config.dns.accountPages }}
 {{- end }}
-    secretName: nginx-ingress-services-wildcard-tls-certificate
+    secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
 {{- end }}
   rules:
     - host: {{ .Values.config.dns.https }}

charts/nginx-ingress-services/templates/issuer.yaml

@@ -0,0 +1,23 @@
+{{- if and .Values.tls.enabled .Values.tls.useCertManager -}}
+apiVersion: cert-manager.io/v1alpha2
+kind: Issuer
+metadata:
+  name: letsencrypt-http01
+  namespace: {{ .Release.Namespace }}
+  docs: "https://cert-manager.io/docs/configuration/acme/"
+  labels:
+    chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+    release: "{{ .Release.Name }}"
+    heritage: "{{ .Release.Service }}"
+spec:
+  acme:
+    server: {{ include "certificate-manager.apiServerURL" . | quote }}
+    email: {{ required "Missing value: certmasterEmail" .Values.certManager.certmasterEmail | quote }}
+    # NOTE: this secret doesnt need to be created, it only gets a name with this
+    privateKeySecretRef:
+      name: letsencrypt-http01-account-key
+    solvers:
+      - http01:
+          ingress:
+            class: nginx
+{{- end -}}

charts/nginx-ingress-services/templates/secret.yaml

@@ -1,17 +1,25 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: nginx-ingress-services-wildcard-tls-certificate
+  name: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
   labels:
     chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
     release: "{{ .Release.Name }}"
     heritage: "{{ .Release.Service }}"
 type: kubernetes.io/tls
+{{ if and .Values.tls.enabled .Values.tls.useCertManager -}}
+{{- /* NOTE: providing `data` (and empty strings) allows to manage this secret resource with Helm if cert-manager is user */ -}}
 data:
-  {{/* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
+  tls.crt: ""
+  tls.key: ""
+{{- end -}}
+{{- if and .Values.tls.enabled (not .Values.tls.useCertManager) -}}
+data:
+  {{- /* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
   for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}
 
   {{- with .Values.secrets }}
   tls.crt: {{ .tlsWildcardCert | b64enc | quote }}
   tls.key: {{ .tlsWildcardKey | b64enc | quote }}
   {{- end }}
+{{- end -}}

charts/nginx-ingress-services/values.yaml

@@ -20,6 +20,22 @@ accountPages:
 # for an example)
 tls:
   enabled: true
+  # NOTE:
+  #  (1) enables automated certificate renewal provided by https://github.com/jetstack/cert-manager
+  #  (2) supersedes wildcard certificate configuration mentioned above
+  # IMPORTANT: requires cert-manager to be installed on the cluster beforehand, e.g.
+  #            `helm upgrade --install -n cert-manager-ns --set 'installCRDs=true' cert-manager jetstack/cert-manager`
+  #
+  useCertManager: false
+
+certManager:
+  # Indicates whether Letsencrypt's staging API server is used and therefore certificates are NOT trusted
+  # default: production API server is used and certificates are trusted
+  inTestMode: false
+  # NOTE:
+  #  (1) required to be set in values.yaml
+  #  (2) used by Letsencrypt to send expiration notices and alike
+  certmasterEmail:
 
 service:
   nginz:

charts/team-settings/values.yaml

@@ -9,7 +9,7 @@ resources:
     cpu: "1"
 image:
   repository: quay.io/wire/team-settings
-  tag: 14965-2.12.0-9c23a0-v0.24.68-production
+  tag: 15522-2.13.0-45540f-v0.24.79-production
 service:
   https:
     externalPort: 443

charts/webapp/values.yaml

@@ -9,7 +9,7 @@ resources:
     cpu: "1"
 image:
   repository: quay.io/wire/webapp
-  tag: 52663-0.1.0-b524bd-v0.24.72-production
+  tag: 53287-0.1.0-a268bc-v0.24.78-production
 service:
   https:
     externalPort: 443

terraform/modules/aws-cargohold-asset-storage/resources.s3.tf

@@ -2,17 +2,25 @@ resource "aws_s3_bucket" "asset_storage" {
   bucket = "${random_string.bucket.keepers.env}-${random_string.bucket.keepers.name}-cargohold-${random_string.bucket.result}"
   acl    = "private"
   region = var.region
+
+  cors_rule {
+    allowed_headers = ["*"]
+    allowed_methods = ["GET", "HEAD"]
+    allowed_origins = ["*"]
+    max_age_seconds = 3000
+  }
+
 }
 
 resource "random_string" "bucket" {
-  length = 8
-  lower = true
-  upper = false
-  number = true
+  length  = 8
+  lower   = true
+  upper   = false
+  number  = true
   special = false
 
   keepers = {
-    env = var.environment
+    env  = var.environment
     name = var.bucket_name
   }
 }

terraform/modules/aws-vpc-security-groups/main.tf

@@ -178,19 +178,28 @@ resource "aws_security_group" "has_assets" {
   }
 }
 
-# A security group for access to kubernetes nodes. should be added to the admin host only.
+# A security group for administrative access to kubernetes nodes. should be added to the admin host only.
 resource "aws_security_group" "talk_to_k8s" {
   name        = "talk_to_k8s"
   description = "hosts that are allowed to speak to kubernetes."
   vpc_id      = var.vpc_id
 
+  # kubectl
   egress {
     from_port   = 6443
     to_port     = 6443
     protocol    = "tcp"
     cidr_blocks = ["172.17.0.0/20"]
   }
 
+  # the application itsself.
+  egress {
+    from_port   = 31772
+    to_port     = 31773
+    protocol    = "tcp"
+    cidr_blocks = ["172.17.0.0/20"]
+  }
+
   tags = {
     Name = "talk_to_k8s"
   }
@@ -226,6 +235,16 @@ resource "aws_security_group" "k8s_node" {
     security_groups = ["${aws_security_group.k8s_private.id}"]
   }
 
+  # incoming traffic to the application.
+  ingress {
+    from_port   = 31772
+    to_port     = 31773
+    protocol    = "tcp"
+    # NOTE: NLBs dont allow security groups to be set on them, which is why
+    # we go with the CIDR for now, which is hard-coded and needs fixing
+    cidr_blocks = ["172.17.0.0/20"]
+  }
+
   tags = {
     Name = "k8s_node"
   }

terraform/modules/aws-vpc/main.tf

@@ -24,7 +24,7 @@ module "vpc" {
   private_subnets = ["172.17.0.0/22", "172.17.4.0/22", "172.17.8.0/22"]
   public_subnets  = ["172.17.12.0/24", "172.17.13.0/24", "172.17.14.0/24"]
 
-  enable_dns_hostnames = true
+  enable_dns_hostnames = false
   enable_dns_support   = true
 
   enable_dhcp_options      = true
@@ -35,11 +35,7 @@ module "vpc" {
   # VPC endpoint for DynamoDB
   enable_dynamodb_endpoint = true
 
-  # In case we run terraform from within the environment.
-  # VPC Endpoint for EC2
-  enable_ec2_endpoint              = true
-  ec2_endpoint_private_dns_enabled = true
-  ec2_endpoint_security_group_ids  = [data.aws_security_group.default.id]
+  enable_s3_endpoint = true
 
   enable_nat_gateway     = true
   one_nat_gateway_per_az = false
@@ -54,4 +50,10 @@ module "vpc" {
     Owner = "Backend Team"
     Name  = var.name
   }
+  private_subnet_tags = {
+    Routability = "private"
+  }
+  public_subnet_tags = {
+    Routability = "public"
+  }
 }