Merge pull request #237 from wireapp/release-2020-04-24

Release 2020-04-24
wireapp · Apr 27, 2020 · 6b95c3f · 6b95c3f
2 parents a55d17a + e174bf6
commit 6b95c3f
Show file tree

Hide file tree

Showing 43 changed files with 969 additions and 69 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,9 +1,28 @@
-# [ 2020-04-15 ]
+# 2020-04-24
+
+## Features
+
+- Add missing galley route to nginz. (#223)
+- Add maxFanoutSize to galley's options (#231)
+- move to helm 3. (#236)
+- terraform configuration from the offline environment. (#230)
+- terraform module to initialize state sharing on AWS (#234)
+- add a commented out block for specifying a non-default elasticsearch apt mirror. (#225)
+
+## Bug fixes
+
+- Fix commented out example value for HTTPS proxy environment variable
+- All to set HTTP proxy environment vars for brig, cargohold, galley, gundeck, proxy, spar (#217)
+- skip memorizing the IPs of redis nodes if there are not any. (#224)
+- Add missing cassandra host value for elasticsearch-index chart (#227)
+- Remove unused table (#222)
+
+# 2020-04-15
 
 ## Release Notes
 
 - This version adds a new migration to the elasticsearch index, it will go through all users in
-  cassandra and (re-)create all users in elasticsearch. So, it could take a long time to finish 
+  cassandra and (re-)create all users in elasticsearch. So, it could take a long time to finish
   depending on the number of users in the system.
 
 ## Features

diff --git a/ansible/download_cli_binaries.yml b/ansible/download_cli_binaries.yml
@@ -11,7 +11,7 @@
   vars:
     # Ensure these versions match the server-side ones (or are compatible)
     # Also see download_kubespray.yml
-    kubernetes_helm_ver: v2.13.1
+    kubernetes_helm_ver: v3.1.1
     kubectl_version: "1.14.2"
     # default directories for a global installation
     kubectl_bin_directory: "/usr/local/bin"
@@ -49,6 +49,3 @@
     - ansible-kubectl
     - ansible-helm
 
-  tasks:
-    - name: initialize helm
-      shell: helm init --client-only
diff --git a/ansible/helm_external.yml b/ansible/helm_external.yml
@@ -23,24 +23,28 @@
         external_dir_name: elasticsearch-external
         server_type: elasticsearch
         network_interface: "{{ elasticsearch_network_interface | default('') }}"
+      when: '"elasticsearch" in groups'
 
     - name: Generate minio IPs for helm
       include_tasks: tasks/helm_external.yml
       vars:
         external_dir_name: minio-external
         server_type: minio
         network_interface: "{{ minio_network_interface | default('') }}"
+      when: '"minio" in groups'
 
     - name: Generate cassandra IPs for helm
       include_tasks: tasks/helm_external.yml
       vars:
         external_dir_name: cassandra-external
         server_type: cassandra
         network_interface: "{{ cassandra_network_interface | default('') }}"
+      when: '"cassandra" in groups'
 
     - name: Generate redis IPs for helm
       include_tasks: tasks/helm_external.yml
       vars:
         external_dir_name: redis-external
         server_type: redis
         network_interface: "{{ redis_network_interface | default('') }}"
+      when: '"redis" in groups'
diff --git a/ansible/hosts.example-demo.ini b/ansible/hosts.example-demo.ini
@@ -40,6 +40,5 @@ bootstrap_os = ubuntu
 docker_dns_servers_strict = False
 
 [k8s-cluster:vars]
-helm_enabled = True
 kube_network_plugin = flannel
 kubeconfig_localhost = True
diff --git a/ansible/hosts.example.ini b/ansible/hosts.example.ini
@@ -126,6 +126,10 @@ is_aws_environment = False
 ## see comments in elasticsearch.yml for details.
 # elasticsearch_network_interface = "ens123"
 
+## Set these in order to use an APT mirror other than the default.
+# es_apt_key = "https://<mymirror>/linux/ubuntu/gpg"
+# es_apt_url = "deb [trusted=yes] https://<mymirror>/apt bionic stable"
+
 ### MINIO section ###
 
 ## Set this to a name of a network interface (e.g. 'eth0'), on which you wish minio processes to talk to each other.
@@ -141,8 +145,6 @@ bootstrap_os = ubuntu
 # docker_dns_servers_strict = false
 
 [k8s-cluster:vars]
-# gotcha: use capital True or False (otherwise ansible parses as string)
-helm_enabled = True
 # 'flannel' is preferred on bare-metal setups, in case you wish to use `metallb`
 kube_network_plugin = flannel
 

diff --git a/charts/brig/templates/configmap.yaml b/charts/brig/templates/configmap.yaml
@@ -43,7 +43,6 @@ data:
 
     {{- with .aws }}
     aws:
-      blacklistTable: {{ .blacklistTable }}
       prekeyTable: {{ .prekeyTable }}
       sqsEndpoint: {{ .sqsEndpoint | quote }}
       dynamoDBEndpoint: {{ .dynamoDBEndpoint | quote }}

diff --git a/charts/brig/templates/deployment.yaml b/charts/brig/templates/deployment.yaml
@@ -66,6 +66,26 @@ spec:
           # TODO: Is this the best way to do this?
           - name: AWS_REGION
             value: "{{ .Values.config.aws.region }}"
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/brig/values.yaml b/charts/brig/values.yaml
@@ -73,6 +73,7 @@ config:
     # setUserMaxPermClients: <int>
   smtp:
     passwordFile: /etc/wire/brig/secrets/smtp-password.txt
+  proxy: {}
 turnStatic:
   v1:
   - turn:localhost:3478

diff --git a/charts/cargohold/templates/deployment.yaml b/charts/cargohold/templates/deployment.yaml
@@ -54,6 +54,26 @@ spec:
               secretKeyRef:
                 name: cargohold
                 key: awsSecretKey
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/cargohold/values.yaml b/charts/cargohold/values.yaml
@@ -16,3 +16,4 @@ config:
   logLevel: Info
   aws:
     s3Bucket: assets
+  proxy: {}
diff --git a/charts/fake-aws-dynamodb/templates/deployment.yaml b/charts/fake-aws-dynamodb/templates/deployment.yaml
@@ -50,11 +50,6 @@ spec:
           while true
           do
               # Recreate resources if needed
-              TABLE=$(table_exists "{{ $.Values.tables.brigUserkeyBlacklist }}")
-              if [ "$TABLE" == "1" ]
-                then echo "Table {{ $.Values.tables.brigUserkeyBlacklist }} exists, no need to re-create"
-                else exec_until_ready "aws --endpoint-url=http://localhost:{{ $.Values.service.internalPort }} dynamodb create-table --table-name {{ $.Values.tables.brigUserkeyBlacklist }} --attribute-definitions AttributeName=key,AttributeType=S --key-schema AttributeName=key,KeyType=HASH --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5"
-              fi
               TABLE=$(table_exists "{{ $.Values.tables.brigPrekeys }}")
               if [ "$TABLE" == "1" ]
                 then echo "Table {{ $.Values.tables.brigPrekeys }} exists, no need to re-create"

diff --git a/charts/fake-aws-dynamodb/values.yaml b/charts/fake-aws-dynamodb/values.yaml
@@ -8,7 +8,6 @@ service:
 
 tables:
   brigPrekeys: integration-brig-prekeys
-  brigUserkeyBlacklist: integration-brig-userkey-blacklist
 
 resources:
   limits:

diff --git a/charts/galley/templates/configmap.yaml b/charts/galley/templates/configmap.yaml
@@ -43,6 +43,9 @@ data:
       intraListing: false
       maxTeamSize: {{ .settings.maxTeamSize }}
       maxConvSize: {{ .settings.maxConvSize }}
+      {{- if .settings.maxFanoutSize }}
+      maxFanoutSize: {{ .settings.maxFanoutSize }}
+      {{- end }}
       conversationCodeURI: {{ .settings.conversationCodeURI | quote }}
       {{- if .settings.featureFlags }}
       featureFlags:

diff --git a/charts/galley/templates/deployment.yaml b/charts/galley/templates/deployment.yaml
@@ -56,6 +56,26 @@ spec:
                 key: awsSecretKey
           - name: AWS_REGION
             value: "{{ .Values.config.aws.region }}"
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/galley/values.yaml b/charts/galley/values.yaml
@@ -27,3 +27,4 @@ config:
       legalhold: disabled-by-default
   aws:
     region: "eu-west-1"
+  proxy: {}
diff --git a/charts/gundeck/templates/deployment.yaml b/charts/gundeck/templates/deployment.yaml
@@ -56,6 +56,26 @@ spec:
                 key: awsSecretKey
           - name: AWS_REGION
             value: "{{ .Values.config.aws.region }}"
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/gundeck/values.yaml b/charts/gundeck/values.yaml
@@ -22,3 +22,4 @@ config:
   bulkPush: false
   aws:
     region: "eu-west-1"
+  proxy: {}
diff --git a/charts/nginz/values.yaml b/charts/nginz/values.yaml
@@ -269,6 +269,9 @@ nginx_conf:
     - path: ~* ^/teams/([^/]*)/members(.*)
       envs:
       - all
+    - path: ~* ^/teams/([^/]*)/get-members-by-ids-using-post(.*)
+      envs:
+      - all
     - path: ~* ^/teams/([^/]*)/conversations(.*)
       envs:
       - all

diff --git a/charts/proxy/templates/deployment.yaml b/charts/proxy/templates/deployment.yaml
@@ -43,6 +43,27 @@ spec:
             mountPath: "/etc/wire/proxy/secrets"
           - name: "proxy-config"
             mountPath: "/etc/wire/proxy/conf"
+          env:
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/proxy/values.yaml b/charts/proxy/values.yaml
@@ -14,3 +14,4 @@ resources:
     cpu: "500m"
 config:
   logLevel: Debug
+  proxy: {}
diff --git a/charts/spar/templates/deployment.yaml b/charts/spar/templates/deployment.yaml
@@ -37,6 +37,27 @@ spec:
           volumeMounts:
           - name: "spar-config"
             mountPath: "/etc/wire/spar/conf"
+          env:
+          {{- with .Values.config.proxy }}
+          {{- if .httpProxy }}
+          - name: http_proxy
+            value: {{ .httpProxy | quote }}
+          - name: HTTP_PROXY
+            value: {{ .httpProxy | quote }}
+          {{- end }}
+          {{- if .httpsProxy }}
+          - name: https_proxy
+            value: {{ .httpsProxy | quote }}
+          - name: HTTPS_PROXY
+            value: {{ .httpsProxy | quote }}
+          {{- end }}
+          {{- if .noProxyList }}
+          - name: no_proxy
+            value: {{ join "," .noProxyList | quote }}
+          - name: NO_PROXY
+            value: {{ join "," .noProxyList | quote }}
+          {{- end }}
+          {{- end }} 
           ports:
             - containerPort: {{ .Values.service.internalPort }}
           livenessProbe:

diff --git a/charts/spar/values.yaml b/charts/spar/values.yaml
@@ -20,3 +20,4 @@ config:
   logLevel: Info
   maxttlAuthreq: 7200
   maxttlAuthresp: 7200
+  proxy: {}
diff --git a/terraform/.gitignore b/terraform/.gitignore
@@ -0,0 +1 @@
+.terraform
diff --git a/terraform/examples/README.md b/terraform/examples/README.md
@@ -1,5 +1,7 @@
 # Example terraform scripts
 
-This example terraform script can be used to create a few virtual machines on the hetzner cloud provider, and generate an inventory file to use with ansible (see wire-server-deploy/ansible/ )
-
 Adapt to your needs as necessary.
+
+## create-infrastructure.tf
+This terraform script can be used to create a few virtual machines on the hetzner cloud provider, and generate an inventory file to use with ansible. (see: wire-server-deploy/ansible/ )
+
diff --git a/terraform/examples/inventory.tpl b/terraform/examples/inventory.tpl
@@ -91,7 +91,6 @@ bootstrap_os = ubuntu
 docker_dns_servers_strict = False
 
 [k8s-cluster:vars]
-helm_enabled = True
 # flannel is preferred on bare-metal setups, in case you wish to use metallb
 kube_network_plugin = flannel
 ## download the kubeconfig after installing to localhost

diff --git a/terraform/modules/README.md b/terraform/modules/README.md
@@ -0,0 +1,5 @@
+# terraform modules
+
+HERE BE DRAGONS
+
+These terraform modules are work-in-progress in active development. DO NOT rely on these for a production environment.
diff --git a/terraform/modules/aws_ami_ubuntu_search/README.md b/terraform/modules/aws_ami_ubuntu_search/README.md
@@ -0,0 +1,5 @@
+# terraform modules
+
+HERE BE DRAGONS
+
+These terraform modules are work-in-progress in active development. DO NOT rely on these for a production environment.
-Original file line number
+Diff line change
@@ Expand Up / @@ -8,7 +8,6 @@ service: @@
     tables:
       brigPrekeys: integration-brig-prekeys
-      brigUserkeyBlacklist: integration-brig-userkey-blacklist
     resources:
       limits:
@@ Expand Down @@