From c93d012b5cd782bf8fcb177cec18b7db13c3efa7 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 2 Nov 2023 12:26:10 +0200 Subject: [PATCH] aks/: Update Azure deployment files Update Azure deployment files, to automate deployment. Signed-off-by: Denys Fedoryshchenko --- kube/aks/README.md | 7 +- kube/aks/api-initial-deploy.sh | 207 +++++++++++++++++++++++++ kube/aks/{ => deploy}/api.yaml | 18 ++- kube/aks/deploy/configmap.yaml.example | 13 ++ kube/aks/{ => deploy}/ingress.yaml | 28 ++-- kube/aks/{ => deploy}/redis.yaml | 4 +- 6 files changed, 248 insertions(+), 29 deletions(-) create mode 100755 kube/aks/api-initial-deploy.sh rename kube/aks/{ => deploy}/api.yaml (70%) create mode 100644 kube/aks/deploy/configmap.yaml.example rename kube/aks/{ => deploy}/ingress.yaml (60%) rename kube/aks/{ => deploy}/redis.yaml (89%) diff --git a/kube/aks/README.md b/kube/aks/README.md index 53426e0bf..c7b16e88c 100644 --- a/kube/aks/README.md +++ b/kube/aks/README.md @@ -29,11 +29,12 @@ portal](https://portal.azure.com/#create/Microsoft.AKS). The standard settings are fine for this use-case although you might need to adjust a few things to match the scale of your particular deployment. + ## Atlas MongoDB account -This AKS reference deployment relies on an Atlas account for MongoDB in order -to keep everything in the Cloud and application setup to the bare minimal. As -such, please [create an Atlas +You can use also Atlas account for MongoDB in order +to keep everything in the Cloud and keep application setup to the bare minimal. +As such, please [create an Atlas account](https://www.mongodb.com/cloud/atlas/register) if you don't already have one and set up a database. The MongoDB service string will be needed later on to let the API service connect to it. diff --git a/kube/aks/api-initial-deploy.sh b/kube/aks/api-initial-deploy.sh new file mode 100755 index 000000000..6f0bb1cf1 --- /dev/null +++ b/kube/aks/api-initial-deploy.sh @@ -0,0 +1,207 @@ +#!/bin/bash + +# Azure specific variables, unset if you are not using Azure +AZURE_RG="kernelci-api-staging" +LOCATION="eastus" + +CONTEXT="kernelci-api-staging-1-admin" +CLUSTER_NAME="kernelci-api-staging-1" +NS="kernelci-api-testns" +SECRET=$(openssl rand -hex 32) +DNSLABEL="kernelci-api-staging" +# mongodb+srv://username:password@customname.mongodb.net +MONGO="" + +# This might contain IP variable +if [ -f api-initial-deploy.cfg ]; then + source api-initial-deploy.cfg +fi + +function fetch_static_ip { + # az network public-ip create --resource-group MC_myResourceGroup_myAKSCluster_eastus --name myAKSPublicIP --sku Standard --allocation-method static --query publicIp.ipAddress -o tsv + if [ -z "$AZURE_RG" ]; then + echo "AZURE_RG not set, not an Azure deployment" + echo "You need to retrieve the static IP address of the ingress controller and set the IP variable in api-initial-deploy.cfg" + exit 1 + fi + # TODO: specify zone + az network public-ip create --resource-group $AZURE_RG --name kernelci-api-staging-ip --sku Standard --allocation-method static --query publicIp.ipAddress -o tsv + IP=$(az network public-ip show --resource-group $AZURE_RG --name kernelci-api-staging-ip --query ipAddress -o tsv) + echo "export IP=\"${IP}\"" >> api-initial-deploy.cfg +} + +function azure_ip_permissions { + echo "Assign permissions for cluster to read/retrieve IPs" + echo "Retrieving cluster resource group..." + RG_SCOPE=$(az group show --name $AZURE_RG --query id -o tsv) + echo "Retrieving cluster client id..." + CLIENT_ID=$(az aks show --name $CLUSTER_NAME --resource-group $AZURE_RG --query identity.principalId -o tsv) + echo "Assigning permissions..." + az role assignment create \ + --assignee ${CLIENT_ID} \ + --role "Network Contributor" \ + --scope ${RG_SCOPE} +} + +function local_setup { + # obviously, do we have working kubectl? + if ! command -v kubectl &> /dev/null + then + echo "kubectl not found, exiting" + exit + fi + if ! kubectl config get-contexts -o name | grep -q ${CONTEXT} + then + echo "Context ${CONTEXT} not found, exiting" + exit + fi + + # helm + if ! command -v helm &> /dev/null + then + echo "helm not found, downloading...(https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3)" + echo "Do you want to proceed or you will install helm using your own means?" + select yn in "Yes" "No"; do + case $yn in + Yes ) break;; + No ) exit;; + esac + done + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod 700 get_helm.sh + ./get_helm.sh + # Clean up poodle :) + rm get_helm.sh + echo "helm installed" + fi +} + +# function for namespace +function recreate_ns { + # Delete namespace + echo "Deleting namespace ${NS}..." + kubectl --context=${CONTEXT} delete namespace ${NS} + sleep 10 + + # Create namespace + echo "Creating namespace ${NS}..."2 + kubectl --context=${CONTEXT} create namespace ${NS} +} + +function update_fqdn { + echo "Getting public ip ResourceID for ip ${IP}..." + PUBLICIPID=$(az network public-ip list --query "[?ipAddress!=null]|[?contains(ipAddress, '$IP')].[id]" --output tsv) + if [ "$PUBLICIPID" == "" ]; then + echo "FATAL: IP ResourceID not found" + exit 1 + fi + + # Update public IP address with DNS name + az network public-ip update --ids $PUBLICIPID --dns-name $DNSLABEL + + # Display the FQDN + az network public-ip show --ids $PUBLICIPID --query "[dnsSettings.fqdn]" --output tsv +} + +function deploy_once { + # Set secret + echo "Setting secret..." + kubectl --context=${CONTEXT} create secret generic kernelci-api-secret --from-literal=secret-key=${SECRET} --namespace=${NS} + echo "Secret: ${SECRET}" >> .api-secret.txt + + # replace MONGOCONNECTSTRING in deploy/configmap.yaml.example to ${MONGO} and save to deploy/configmap.yaml + sed "s|MONGOCONNECTSTRING|${MONGO}|g" deploy/configmap.yaml.example > deploy/configmap.yaml + + # Deploy configmap + echo "Deploying configmap..." + kubectl --context=${CONTEXT} create -f deploy/configmap.yaml --namespace=${NS} + + # HELM misc stuff to prepare + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo add stable https://charts.helm.sh/stable + helm repo add jetstack https://charts.jetstack.io + helm repo update + # helm show values ingress-nginx/ingress-nginx + +} + +function deploy_update_nginx { + echo "Deploying ingress-nginx..." + helm uninstall ingress-nginx --namespace=kernelci-api-testns + helm install ingress-nginx ingress-nginx/ingress-nginx \ + -n ${NS} \ + --set controller.replicaCount=1 \ + --set controller.nodeSelector."beta\.kubernetes\.io/os"=linux \ + --set defaultBackend.nodeSelector."beta\.kubernetes\.io/os"=linux \ + --set controller.service.externalTrafficPolicy=Local \ + --set controller.service.loadBalancerIP="${IP}" \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-dns-label-name"="${DNSLABEL}" \ + --set controller.publishService.enabled=true \ + --set controller.service.annotations."service\.beta\.kubernetes\.io/azure-load-balancer-resource-group"="${AZURE_RG}" + + # You might spot the following warning in the output: + # "It may take a few minutes for the LoadBalancer IP to be available." + # You can watch the status by running 'kubectl --namespace kernelci-api-testns get services -o wide -w ingress-nginx-controller' + # Other diagnostic commands: + # kubectl --namespace=kernelci-api-testns logs svc/ingress-nginx-controller + # kubectl describe service ingress-nginx-controller --namespace=kernelci-api-testns + # kubectl --namespace=kernelci-api-testns get svc +} + +function deploy_update_cert_manager { + # Deploy cert-manager + echo "Deploying cert-manager..." + helm install cert-manager jetstack/cert-manager --namespace=${NS} --set installCRDs=true +} + +function deploy_update { + # Deploy redis + echo "Deploying redis..." + kubectl --context=${CONTEXT} apply -f deploy/redis.yaml --namespace=${NS} + + # Deploy API + echo "Deploying API Deployment..." + kubectl --context=${CONTEXT} apply -f deploy/api.yaml --namespace=${NS} + + # TODO: Deploy only once, we rarely need to update them + deploy_update_nginx + deploy_update_cert_manager + + # Deploy API + echo "Deploying API Ingress..." + kubectl --context=${CONTEXT} apply -f deploy/ingress.yaml --namespace=${NS} +} + +function setup_admin { + # metadata: labels: app: "api" + API_POD_NAME=$(kubectl get pods --namespace=${NS} -l "app=api" -o jsonpath="{.items[0].metadata.name}") + echo "Setting up admin user..." + kubectl exec --namespace=${NS} -it $API_POD_NAME -- python3 -m api.admin --mongo ${MONGO} --email bot@kernelci.org +} + +echo "This script will deploy kernelci API to your cluster" + +# Local toolset setup +local_setup + +# if IP not set, initial set, allocate static ip and update fqdn +if [ -z "$IP" ]; then + echo "Likely you are running script first time" + echo "It will assign static IP and update DNS" + echo "Then give your cluster permission to use public IPs" + fetch_static_ip + update_fqdn + azure_ip_permissions + echo "Waiting IP to propagate, 30 seconds..." + sleep 30 +fi + +recreate_ns +deploy_once +deploy_update +setup_admin + +echo "----------------------------------------" +echo "Done" +echo "Test API availability by curl https://${DNSLABEL}.${LOCATION}.cloudapp.azure.com/latest/" + diff --git a/kube/aks/api.yaml b/kube/aks/deploy/api.yaml similarity index 70% rename from kube/aks/api.yaml rename to kube/aks/deploy/api.yaml index b026be006..99adf6ab1 100644 --- a/kube/aks/api.yaml +++ b/kube/aks/deploy/api.yaml @@ -7,7 +7,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: api - namespace: kernelci-api + namespace: kernelci-api-testns spec: replicas: 1 selector: @@ -30,21 +30,25 @@ spec: - name: SECRET_KEY valueFrom: secretKeyRef: - name: kernelci-api-secret-key + name: kernelci-api-secret key: secret-key - name: MONGO_SERVICE valueFrom: - secretKeyRef: - name: kernelci-api-mongo-service - key: mongo-service + configMapKeyRef: +# this one should be actually secret if atlas is used + name: kernelci-api-config + key: mongo_service - name: REDIS_HOST - value: redis + valueFrom: + configMapKeyRef: + name: kernelci-api-config + key: redis_host --- apiVersion: v1 kind: Service metadata: name: api - namespace: kernelci-api + namespace: kernelci-api-testns spec: ports: - port: 80 diff --git a/kube/aks/deploy/configmap.yaml.example b/kube/aks/deploy/configmap.yaml.example new file mode 100644 index 000000000..4f6462d5b --- /dev/null +++ b/kube/aks/deploy/configmap.yaml.example @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# +# Copyright (C) 2023 Collabora Limited +# Author: Jeny Sadadia + +apiVersion: v1 +kind: ConfigMap +metadata: + name: kernelci-api-config + namespace: kernelci-api-testns +data: + redis_host: "redis.kernelci-api-testns.svc.cluster.local" + mongo_service: "MONGOCONNECTSTRING" diff --git a/kube/aks/ingress.yaml b/kube/aks/deploy/ingress.yaml similarity index 60% rename from kube/aks/ingress.yaml rename to kube/aks/deploy/ingress.yaml index c73355280..197c2189f 100644 --- a/kube/aks/ingress.yaml +++ b/kube/aks/deploy/ingress.yaml @@ -1,25 +1,18 @@ -# SPDX-License-Identifier: LGPL-2.1-or-later -# -# Copyright (C) 2023 Collabora Limited -# Author: Guillaume Tucker - -# Nginx ingress - apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: api - namespace: kernelci-api + name: kernelci-api-ingress + namespace: kernelci-api-testns annotations: cert-manager.io/cluster-issuer: letsencrypt spec: ingressClassName: nginx tls: - hosts: - - kernelci-api.eastus.cloudapp.azure.com - secretName: tls-secret + - kernelci-api-staging.eastus.cloudapp.azure.com + secretName: kernelci-api-staging-tls rules: - - host: kernelci-api.eastus.cloudapp.azure.com + - host: kernelci-api-staging.eastus.cloudapp.azure.com http: paths: - path: / @@ -28,16 +21,13 @@ spec: service: name: api port: - number: 80 + number: 8000 --- - -# SSL certificate issuer - apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: letsencrypt - namespace: kernelci-api + namespace: kernelci-api-testns spec: acme: server: https://acme-v02.api.letsencrypt.org/directory @@ -48,3 +38,7 @@ spec: - http01: ingress: class: nginx + podTemplate: + spec: + nodeSelector: + "kubernetes.io/os": linux diff --git a/kube/aks/redis.yaml b/kube/aks/deploy/redis.yaml similarity index 89% rename from kube/aks/redis.yaml rename to kube/aks/deploy/redis.yaml index 147a7a04d..c9f168296 100644 --- a/kube/aks/redis.yaml +++ b/kube/aks/deploy/redis.yaml @@ -7,7 +7,7 @@ apiVersion: apps/v1 kind: Deployment metadata: name: redis - namespace: kernelci-api + namespace: kernelci-api-testns spec: replicas: 1 selector: @@ -28,7 +28,7 @@ apiVersion: v1 kind: Service metadata: name: redis - namespace: kernelci-api + namespace: kernelci-api-testns spec: ports: - port: 6379