Merge pull request #320 from wireapp/release_2020_07_29

CHANGELOG.md

@@ -1,3 +1,13 @@
+# 2020-07-39
+
+## Features
+
+* [tf-module:dns-records] Add output for FQDNs (#315)
+* README.md: stop explicitly referring to the "develop" branch (#318)
+* nginz redirect /teams/invitations/by-email to brig (#317)
+* S3 support (#311, #316)
+* Provide AWS_REGION variable to cargohold (#314)
+
 # 2020-07-13
 
 ## Features

charts/cargohold/templates/deployment.yaml

@@ -54,6 +54,8 @@ spec:
               secretKeyRef:
                 name: cargohold
                 key: awsSecretKey
+          - name: AWS_REGION
+            value: "{{ .Values.config.aws.region }}"
           {{- with .Values.config.proxy }}
           {{- if .httpProxy }}
           - name: http_proxy

charts/cargohold/templates/tests/cargohold-integration.yaml

@@ -17,4 +17,12 @@ spec:
     volumeMounts:
     - name: "cargohold-integration"
       mountPath: "/etc/wire/integration"
+    env:
+    # these dummy values are necessary for Amazonka's "Discover"
+    - name: AWS_ACCESS_KEY_ID
+      value: "dummy"
+    - name: AWS_SECRET_ACCESS_KEY
+      value: "dummy"
+    - name: AWS_REGION
+      value: "eu-west-1"
   restartPolicy: Never

charts/cargohold/values.yaml

@@ -15,5 +15,6 @@ resources:
 config:
   logLevel: Info
   aws:
+    region: "eu-west-1"
     s3Bucket: assets
   proxy: {}

charts/nginz/values.yaml

@@ -225,6 +225,10 @@ nginx_conf:
       envs:
       - all
       disable_zauth: true
+    - path: ~* ^/teams/invitations/by-email$
+      envs:
+      - all
+      disable_zauth: true
     - path: /i/teams/invitation-code
       envs:
       - staging

terraform/modules/aws-brig-prekey-lock-event-queue-email-sending/README.md

@@ -1,4 +1,4 @@
-Terraform module: Brig pre-key locking, event queue (optional: email sending) 
+Terraform module: Brig pre-key locking, event queue (optional: email sending)
 =============================================================================
 
 State: __experimental__
@@ -9,7 +9,7 @@ in cassandra to avoid race conditions), and (B) to establish a message queue
 for internal events (used e.g. during user deletions).
 
 [Optional] Wire-server's "brig" components needs to send emails. This can either
-be done by configuring an SMTP server (Option 1), or by using AWS resources (Option 2).           
+be done by configuring an SMTP server (Option 1), or by using AWS resources (Option 2).
 This terraform module can enable brig to send emails using option 2. In addition, it
 configures *MAIL FROM* for outgoing emails, but does not enable incoming emails
 (possible solution: `aws_ses_receipt_rule`).
@@ -29,8 +29,8 @@ mandatory.
 
 ```hcl
 module "brig_prekey_lock_and_event_queue_emailing" {
-  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-brig-prekey-lock-event-queue-email-sending?ref=develop"
-  
+  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-brig-prekey-lock-event-queue-email-sending?ref=CHANGE-ME"
+
   environment = "staging"
 
   zone_id = "Z12345678SQWERTYU"
@@ -42,8 +42,8 @@ module "brig_prekey_lock_and_event_queue_emailing" {
 
 ```hcl
 module "brig_prekey_lock_and_event_queue" {
-  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-brig-prekey-lock-event-queue-email-sending?ref=develop"
-  
+  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-brig-prekey-lock-event-queue-email-sending?ref=CHANGE-ME"
+
   environment = "staging"
   enable_email_sending = false    # default: true
 }

terraform/modules/aws-cargohold-asset-storage/README.md

@@ -23,8 +23,8 @@ mandatory.
 
 ```hcl
 module "cargohold_asset_storage" {
-  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-cargohold-asset-storage?ref=develop"
-  
+  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-cargohold-asset-storage?ref=CHANGE-ME"
+
   environment = "staging"
 }
 ```

terraform/modules/aws-cargohold-asset-storage/outputs.tf

@@ -4,10 +4,22 @@ output "bucket_name" {
   value = aws_s3_bucket.asset_storage.bucket
 }
 
+output "bucket_id" {
+  value = aws_s3_bucket.asset_storage.id
+}
+
 output "s3_endpoint" {
   value = "https://s3.${aws_s3_bucket.asset_storage.region}.amazonaws.com"
 }
 
+output "s3_endpoint_CIDRs" {
+  value = aws_vpc_endpoint.s3.cidr_blocks
+}
+
+output "s3_endpoint_id" {
+  value = aws_vpc_endpoint.s3.id
+}
+
 output "cargohold_access_key" {
   value = aws_iam_access_key.cargohold.id
 }

terraform/modules/aws-cargohold-asset-storage/resources.iam.tf

@@ -34,3 +34,54 @@ resource "aws_iam_user_policy" "cargohold" {
   }
   EOP
 }
+
+# Create a policy that can be applied to a role, and can be used to access the bucket and the files within.
+resource "aws_iam_policy" "cargohold-s3" {
+  name = "${var.environment}-cargohold-s3"
+  policy = <<-EOP
+  {
+      "Version": "2012-10-17",
+      "Statement": [
+          {
+              "Effect": "Allow",
+              "Action": "s3:*",
+              "Resource": "*"
+          }
+      ]
+  }
+  EOP
+}
+
+# Create an IAM role that can be applied to an instance, and can be used to access the bucket and the files within.
+resource "aws_iam_role" "cargohold-s3" {
+  name = "${var.environment}-cargohold-s3"
+  description = "provide access to s3, for cargohold."
+  assume_role_policy = <<-EOP
+  {
+     "Version": "2012-10-17",
+     "Statement": [
+       {
+         "Action": "sts:AssumeRole",
+         "Principal": {
+           "Service": "ec2.amazonaws.com"
+         },
+         "Effect": "Allow",
+         "Sid": ""
+       }
+     ]
+  }
+  EOP
+  tags = {
+    Name        = "${var.environment}-cargohold-s3",
+    Environment = "${var.environment}"
+    Gateway     = "cargohold-s3"
+  }
+}
+
+
+# attach our IAM policy to our IAM role.
+resource "aws_iam_policy_attachment" "cargohold-s3-attach" {
+  name       = "${var.environment}-cargohold-s3"
+  roles       = [aws_iam_role.cargohold-s3.name]
+  policy_arn = aws_iam_policy.cargohold-s3.arn
+}

terraform/modules/aws-cargohold-asset-storage/resources.s3.tf

@@ -24,3 +24,29 @@ resource "random_string" "bucket" {
     name = var.bucket_name
   }
 }
+
+resource "aws_vpc_endpoint" "s3" {
+  vpc_id       = var.vpc_id
+  service_name = "com.amazonaws.${var.region}.s3"
+
+  tags = {
+    Environment = var.environment
+  }
+}
+
+data "aws_route_tables" "private" {
+  vpc_id = var.vpc_id
+
+  filter {
+    name   = "association.subnet-id"
+    values = var.subnet_ids
+  }
+}
+
+# the routing table association that allows nodes to route traffic to the S3 endpoint.
+resource "aws_vpc_endpoint_route_table_association" "private_s3" {
+  for_each = { for k, v in data.aws_route_tables.private.ids : v => v }
+
+  route_table_id  = each.value
+  vpc_endpoint_id = aws_vpc_endpoint.s3.id
+}

terraform/modules/aws-cargohold-asset-storage/variables.tf

@@ -15,3 +15,13 @@ variable "bucket_name" {
   description = "Name of the bucket that cargohold uses to store files (default: 'assets'; prefix: $environment) "
   default     = "assets"
 }
+
+variable "vpc_id" {
+  type        = string
+  description = "the ID of the VPC to add an S3 endpoint to"
+}
+
+variable "subnet_ids" {
+  type        = list(string)
+  description = "list of the subnet IDs of the subnets to make the S3 endpoint available to."
+}

terraform/modules/aws-dns-records/README.md

@@ -15,8 +15,8 @@ AWS resources: route53
 
 ```hcl
 module "dns_records" {
-  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-dns-records?ref=develop"
-  
+  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-dns-records?ref=CHANGE-ME"
+
   environment = "staging"
 
   zone_fqdn = "example.com"

terraform/modules/aws-dns-records/outputs.tf

@@ -0,0 +1,6 @@
+output "fqdns" {
+  value = concat(
+    [for record in aws_route53_record.a : record.fqdn],
+    [for record in aws_route53_record.cname : record.fqdn]
+  )
+}

terraform/modules/aws-gundeck-push-notifications/README.md

@@ -18,8 +18,8 @@ mandatory.
 
 ```hcl
 module "gundeck-push-notification" {
-  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-gundeck-push-notifications?ref=develop"
-  
+  source = "github.com/wireapp/wire-server-deploy.git//terraform/modules/aws-gundeck-push-notifications?ref=CHANGE-ME"
+
   environment = "dev"
   apns_application_id = "myapp.tld"
   apns_voip_key = file("path/to/app-credentials/key.pem")

terraform/modules/aws-vpc-security-groups/main.tf

@@ -184,8 +184,20 @@ resource "aws_security_group" "talk_to_k8s" {
   description = "hosts that are allowed to speak to kubernetes."
   vpc_id      = var.vpc_id
 
+  # HACK: running out of security groups per instance.
+  #       adding this here since the admin node needs to talk to S3.
+  # S3
+  egress {
+    description = ""
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = var.s3_CIDRs
+  }
+
   # kubectl
   egress {
+    description = ""
     from_port   = 6443
     to_port     = 6443
     protocol    = "tcp"
@@ -194,6 +206,7 @@ resource "aws_security_group" "talk_to_k8s" {
 
   # the application itsself.
   egress {
+    description = ""
     from_port   = 31772
     to_port     = 31773
     protocol    = "tcp"
@@ -213,6 +226,7 @@ resource "aws_security_group" "k8s_node" {
 
   # incoming from the admin node (kubectl)
   ingress {
+    description     = ""
     from_port       = 6443
     to_port         = 6443
     protocol        = "tcp"
@@ -221,6 +235,7 @@ resource "aws_security_group" "k8s_node" {
 
   # FIXME: tighten this up.
   ingress {
+    description     = ""
     from_port       = 0
     to_port         = 65535
     protocol        = "tcp"
@@ -229,6 +244,7 @@ resource "aws_security_group" "k8s_node" {
 
   # FIXME: tighten this up. need UDP for flannel.
   ingress {
+    description     = ""
     from_port       = 0
     to_port         = 65535
     protocol        = "udp"
@@ -272,6 +288,16 @@ resource "aws_security_group" "k8s_private" {
     cidr_blocks = ["172.17.0.0/20"]
   }
 
+  # HACK: running out of security groups, adding this here since all k8s nodes need to talk to S3.
+  # S3
+  egress {
+    description = ""
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = var.s3_CIDRs
+  }
+
   tags = {
     Name = "k8s_private"
   }

terraform/modules/aws-vpc-security-groups/variables.tf

@@ -1,5 +1,10 @@
 variable "vpc_id" {
   type        = string
-  description = "ID of VPC these security groups are for"
+  description = "ID of VPC these security groups are for."
+}
+
+variable "s3_CIDRs" {
+  type        = list(string)
+  description = "subnets that S3 gateways we are using exist in."
 }
 

terraform/modules/aws-vpc/main.tf

@@ -7,12 +7,6 @@ provider "aws" {
   region = "eu-central-1"
 }
 
-# Used for the in-VPC EC2 endpoint.
-data "aws_security_group" "default" {
-  name   = "default"
-  vpc_id = module.vpc.vpc_id
-}
-
 module "vpc" {
   source = "github.com/terraform-aws-modules/terraform-aws-vpc?ref=v2.33.0"
 
@@ -35,8 +29,6 @@ module "vpc" {
   # VPC endpoint for DynamoDB
   enable_dynamodb_endpoint = true
 
-  enable_s3_endpoint = true
-
   enable_nat_gateway     = true
   one_nat_gateway_per_az = false
   # Use this only in productionish environments.

terraform/modules/aws-vpc/outputs.tf

@@ -9,3 +9,4 @@ output "public_subnets" {
 output "private_subnets" {
   value = module.vpc.private_subnets
 }
+