From 9cc91989f471159675bc8fc9db9f054c91d26d5e Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Tue, 20 Feb 2024 16:07:44 +0000 Subject: [PATCH 01/12] traefik persistent volume --- .../kubernetes_ingress/template/main.tf | 40 +- .../modules/kubernetes/ingress/main.tf | 750 ++++++++++-------- .../modules/kubernetes/ingress/outputs.tf | 24 +- .../modules/kubernetes/ingress/variables.tf | 179 +++-- .../kubernetes_ingress/template/outputs.tf | 9 +- .../kubernetes_ingress/template/variables.tf | 168 ++-- 6 files changed, 644 insertions(+), 526 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/main.tf b/src/_nebari/stages/kubernetes_ingress/template/main.tf index 6e275cb39a..f1df788bf7 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/main.tf @@ -1,17 +1,23 @@ -module "kubernetes-ingress" { - source = "./modules/kubernetes/ingress" - - namespace = var.environment - - node-group = var.node_groups.general - - traefik-image = var.traefik-image - - certificate-service = var.certificate-service - acme-email = var.acme-email - acme-server = var.acme-server - certificate-secret-name = var.certificate-secret-name - load-balancer-annotations = var.load-balancer-annotations - load-balancer-ip = var.load-balancer-ip - additional-arguments = var.additional-arguments -} +module "kubernetes-ingress" { + source = "./modules/kubernetes/ingress" + + namespace = var.environment + + node-group = var.node_groups.general + + traefik-image = var.traefik-image + + certificate-service = var.certificate-service + acme-email = var.acme-email + acme-server = var.acme-server + certificate-secret-name = var.certificate-secret-name + load-balancer-annotations = var.load-balancer-annotations + load-balancer-ip = var.load-balancer-ip + additional-arguments = var.additional-arguments + + storage_size = var.storage_size + access_modes = var.access_modes + storage_type = var.storage_type + iops = var.iops + reclaim_policy = var.reclaim_policy +} diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 08bb5b295d..6bdb8c0331 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -1,350 +1,400 @@ -locals { - default_cert = [ - "--entrypoints.websecure.http.tls.certResolver=default", - "--entrypoints.minio.http.tls.certResolver=default", - ] - certificate-settings = { - lets-encrypt = [ - "--entrypoints.websecure.http.tls.certResolver=letsencrypt", - "--entrypoints.minio.http.tls.certResolver=letsencrypt", - "--certificatesresolvers.letsencrypt.acme.tlschallenge", - "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", - "--certificatesresolvers.letsencrypt.acme.storage=acme.json", - "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", - ] - self-signed = local.default_cert - existing = local.default_cert - disabled = [] - } - add-certificate = local.certificate-settings[var.certificate-service] -} - - -resource "kubernetes_service_account" "main" { - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - } -} - - -resource "kubernetes_cluster_role" "main" { - metadata { - name = "${var.name}-traefik-ingress" - } - - rule { - api_groups = [""] - resources = ["services", "endpoints", "secrets"] - verbs = ["get", "list", "watch"] - } - - rule { - api_groups = ["extensions", "networking.k8s.io"] - resources = ["ingresses", "ingressclasses"] - verbs = ["get", "list", "watch"] - } - - rule { - api_groups = ["extensions"] - resources = ["ingresses/status"] - verbs = ["update"] - } - - rule { - api_groups = ["traefik.containo.us"] - resources = ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "middlewaretcps", "tlsoptions", "tlsstores", "traefikservices", "serverstransports"] - verbs = ["get", "list", "watch"] - } -} - - -resource "kubernetes_cluster_role_binding" "main" { - metadata { - name = "${var.name}-traefik-ingress" - } - - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = kubernetes_cluster_role.main.metadata.0.name - } - subject { - kind = "ServiceAccount" - name = kubernetes_service_account.main.metadata.0.name - namespace = var.namespace - } -} - - -resource "kubernetes_service" "main" { - wait_for_load_balancer = true - - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - annotations = var.load-balancer-annotations - } - - spec { - selector = { - "app.kubernetes.io/component" = "traefik-ingress" - } - - port { - name = "http" - protocol = "TCP" - port = 80 - target_port = 80 - } - - port { - name = "https" - protocol = "TCP" - port = 443 - target_port = 443 - } - - port { - name = "ssh" - protocol = "TCP" - port = 8022 - target_port = 8022 - } - - port { - name = "sftp" - protocol = "TCP" - port = 8023 - target_port = 8023 - } - - port { - name = "minio" - protocol = "TCP" - port = 9080 - target_port = 9080 - } - - port { - name = "tcp" - protocol = "TCP" - port = 8786 - target_port = 8786 - } - - type = "LoadBalancer" - load_balancer_ip = var.load-balancer-ip - } -} - -resource "kubernetes_service" "traefik_internal" { - wait_for_load_balancer = true - - metadata { - name = "${var.name}-traefik-internal" - namespace = var.namespace - annotations = { - "prometheus.io/scrape" = "true" - "prometheus.io/path" = "/metrics" - "prometheus.io/port" = 9000 - } - labels = { - "app.kubernetes.io/component" = "traefik-internal-service" - "app.kubernetes.io/part-of" = "traefik-ingress" - } - } - - spec { - selector = { - "app.kubernetes.io/component" = "traefik-ingress" - } - - port { - name = "http" - protocol = "TCP" - port = 9000 - target_port = 9000 - } - - type = "ClusterIP" - } -} - -resource "kubernetes_deployment" "main" { - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - } - - spec { - replicas = 1 - - selector { - match_labels = { - "app.kubernetes.io/component" = "traefik-ingress" - } - } - - template { - metadata { - labels = { - "app.kubernetes.io/component" = "traefik-ingress" - } - } - - spec { - service_account_name = kubernetes_service_account.main.metadata.0.name - termination_grace_period_seconds = 60 - - affinity { - node_affinity { - required_during_scheduling_ignored_during_execution { - node_selector_term { - match_expressions { - key = var.node-group.key - operator = "In" - values = [var.node-group.value] - } - } - } - } - } - - container { - image = "${var.traefik-image.image}:${var.traefik-image.tag}" - name = var.name - - security_context { - capabilities { - drop = ["ALL"] - add = ["NET_BIND_SERVICE"] - } - } - - args = concat([ - # Do not send usage stats - "--global.checknewversion=false", - "--global.sendanonymoususage=false", - # allow access to the dashboard directly through the port - # TODO: eventually needs to be tied into traefik middle - # security possibly using jupyterhub auth this is not a - # security risk at the moment since this port is not - # externally accessible - "--api.insecure=true", - "--api.dashboard=true", - "--ping=true", - # Start the Traefik Kubernetes Ingress Controller - "--providers.kubernetesingress=true", - "--providers.kubernetesingress.namespaces=${var.namespace}", - "--providers.kubernetesingress.ingressclass=traefik", - # Start the Traefik Kubernetes CRD Controller Provider - "--providers.kubernetescrd", - "--providers.kubernetescrd.namespaces=${var.namespace}", - "--providers.kubernetescrd.throttleduration=2s", - "--providers.kubernetescrd.allowcrossnamespace=false", - # Define two entrypoint ports, and setup a redirect from HTTP to HTTPS. - "--entryPoints.web.address=:80", - "--entryPoints.websecure.address=:443", - "--entrypoints.ssh.address=:8022", - "--entrypoints.sftp.address=:8023", - "--entryPoints.tcp.address=:8786", - "--entryPoints.traefik.address=:9000", - # Define the entrypoint port for Minio - "--entryPoints.minio.address=:9080", - # Redirect http -> https - "--entrypoints.web.http.redirections.entryPoint.to=websecure", - "--entrypoints.web.http.redirections.entryPoint.scheme=https", - # Enable Prometheus Monitoring of Traefik - "--metrics.prometheus=true", - # Enable debug logging. Useful to work out why something might not be - # working. Fetch logs of the pod. - "--log.level=${var.loglevel}", - ], - local.add-certificate, - var.additional-arguments, - ) - - port { - name = "http" - container_port = 80 - } - - port { - name = "https" - container_port = 443 - } - - port { - name = "ssh" - container_port = 8022 - } - - port { - name = "sftp" - container_port = 8023 - } - - port { - name = "tcp" - container_port = 8786 - } - - port { - name = "traefik" - container_port = 9000 - } - - port { - name = "minio" - container_port = 9080 - } - - liveness_probe { - http_get { - path = "/ping" - port = "traefik" - } - - initial_delay_seconds = 10 - timeout_seconds = 2 - period_seconds = 10 - failure_threshold = 3 - success_threshold = 1 - } - - readiness_probe { - http_get { - path = "/ping" - port = "traefik" - } - - initial_delay_seconds = 10 - timeout_seconds = 2 - period_seconds = 10 - failure_threshold = 1 - success_threshold = 1 - } - } - } - } - } -} - - -resource "kubernetes_manifest" "tlsstore_default" { - count = var.certificate-secret-name != null ? 1 : 0 - manifest = { - "apiVersion" = "traefik.containo.us/v1alpha1" - "kind" = "TLSStore" - "metadata" = { - "name" = "default" - "namespace" = var.namespace - } - "spec" = { - "defaultCertificate" = { - "secretName" = var.certificate-secret-name - } - } - } -} +locals { + default_cert = [ + "--entrypoints.websecure.http.tls.certResolver=default", + "--entrypoints.minio.http.tls.certResolver=default", + ] + certificate-settings = { + lets-encrypt = [ + "--entrypoints.websecure.http.tls.certResolver=letsencrypt", + "--entrypoints.minio.http.tls.certResolver=letsencrypt", + "--certificatesresolvers.letsencrypt.acme.tlschallenge", + "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", + "--certificatesresolvers.letsencrypt.acme.storage=acme.json", + "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", + ] + self-signed = local.default_cert + existing = local.default_cert + disabled = [] + } + add-certificate = local.certificate-settings[var.certificate-service] +} + + +resource "kubernetes_service_account" "main" { + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + } +} + + +resource "kubernetes_cluster_role" "main" { + metadata { + name = "${var.name}-traefik-ingress" + } + + rule { + api_groups = [""] + resources = ["services", "endpoints", "secrets"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["extensions", "networking.k8s.io"] + resources = ["ingresses", "ingressclasses"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["extensions"] + resources = ["ingresses/status"] + verbs = ["update"] + } + + rule { + api_groups = ["traefik.containo.us"] + resources = ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "middlewaretcps", "tlsoptions", "tlsstores", "traefikservices", "serverstransports"] + verbs = ["get", "list", "watch"] + } +} + + +resource "kubernetes_cluster_role_binding" "main" { + metadata { + name = "${var.name}-traefik-ingress" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.main.metadata.0.name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.main.metadata.0.name + namespace = var.namespace + } +} + + +resource "kubernetes_service" "main" { + wait_for_load_balancer = true + + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + annotations = var.load-balancer-annotations + } + + spec { + selector = { + "app.kubernetes.io/component" = "traefik-ingress" + } + + port { + name = "http" + protocol = "TCP" + port = 80 + target_port = 80 + } + + port { + name = "https" + protocol = "TCP" + port = 443 + target_port = 443 + } + + port { + name = "ssh" + protocol = "TCP" + port = 8022 + target_port = 8022 + } + + port { + name = "sftp" + protocol = "TCP" + port = 8023 + target_port = 8023 + } + + port { + name = "minio" + protocol = "TCP" + port = 9080 + target_port = 9080 + } + + port { + name = "tcp" + protocol = "TCP" + port = 8786 + target_port = 8786 + } + + type = "LoadBalancer" + load_balancer_ip = var.load-balancer-ip + } +} + +resource "kubernetes_service" "traefik_internal" { + wait_for_load_balancer = true + + metadata { + name = "${var.name}-traefik-internal" + namespace = var.namespace + annotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/path" = "/metrics" + "prometheus.io/port" = 9000 + } + labels = { + "app.kubernetes.io/component" = "traefik-internal-service" + "app.kubernetes.io/part-of" = "traefik-ingress" + } + } + + spec { + selector = { + "app.kubernetes.io/component" = "traefik-ingress" + } + + port { + name = "http" + protocol = "TCP" + port = 9000 + target_port = 9000 + } + + type = "ClusterIP" + } +} + +resource "kubernetes_deployment" "main" { + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + } + + spec { + replicas = 1 + + selector { + match_labels = { + "app.kubernetes.io/component" = "traefik-ingress" + } + } + + template { + metadata { + labels = { + "app.kubernetes.io/component" = "traefik-ingress" + } + } + + spec { + service_account_name = kubernetes_service_account.main.metadata.0.name + termination_grace_period_seconds = 60 + + affinity { + node_affinity { + required_during_scheduling_ignored_during_execution { + node_selector_term { + match_expressions { + key = var.node-group.key + operator = "In" + values = [var.node-group.value] + } + } + } + } + } + + container { + image = "${var.traefik-image.image}:${var.traefik-image.tag}" + name = var.name + + security_context { + capabilities { + drop = ["ALL"] + add = ["NET_BIND_SERVICE"] + } + } + + args = concat([ + # Do not send usage stats + "--global.checknewversion=false", + "--global.sendanonymoususage=false", + # allow access to the dashboard directly through the port + # TODO: eventually needs to be tied into traefik middle + # security possibly using jupyterhub auth this is not a + # security risk at the moment since this port is not + # externally accessible + "--api.insecure=true", + "--api.dashboard=true", + "--ping=true", + # Start the Traefik Kubernetes Ingress Controller + "--providers.kubernetesingress=true", + "--providers.kubernetesingress.namespaces=${var.namespace}", + "--providers.kubernetesingress.ingressclass=traefik", + # Start the Traefik Kubernetes CRD Controller Provider + "--providers.kubernetescrd", + "--providers.kubernetescrd.namespaces=${var.namespace}", + "--providers.kubernetescrd.throttleduration=2s", + "--providers.kubernetescrd.allowcrossnamespace=false", + # Define two entrypoint ports, and setup a redirect from HTTP to HTTPS. + "--entryPoints.web.address=:80", + "--entryPoints.websecure.address=:443", + "--entrypoints.ssh.address=:8022", + "--entrypoints.sftp.address=:8023", + "--entryPoints.tcp.address=:8786", + "--entryPoints.traefik.address=:9000", + # Define the entrypoint port for Minio + "--entryPoints.minio.address=:9080", + # Redirect http -> https + "--entrypoints.web.http.redirections.entryPoint.to=websecure", + "--entrypoints.web.http.redirections.entryPoint.scheme=https", + # Enable Prometheus Monitoring of Traefik + "--metrics.prometheus=true", + # Enable debug logging. Useful to work out why something might not be + # working. Fetch logs of the pod. + "--log.level=${var.loglevel}", + ], + local.add-certificate, + var.additional-arguments, + ) + + port { + name = "http" + container_port = 80 + } + + port { + name = "https" + container_port = 443 + } + + port { + name = "ssh" + container_port = 8022 + } + + port { + name = "sftp" + container_port = 8023 + } + + port { + name = "tcp" + container_port = 8786 + } + + port { + name = "traefik" + container_port = 9000 + } + + port { + name = "minio" + container_port = 9080 + } + + liveness_probe { + http_get { + path = "/ping" + port = "traefik" + } + + initial_delay_seconds = 10 + timeout_seconds = 2 + period_seconds = 10 + failure_threshold = 3 + success_threshold = 1 + } + + readiness_probe { + http_get { + path = "/ping" + port = "traefik" + } + + initial_delay_seconds = 10 + timeout_seconds = 2 + period_seconds = 10 + failure_threshold = 1 + success_threshold = 1 + } + } + } + } + } +} + + +resource "kubernetes_manifest" "tlsstore_default" { + count = var.certificate-secret-name != null ? 1 : 0 + manifest = { + "apiVersion" = "traefik.containo.us/v1alpha1" + "kind" = "TLSStore" + "metadata" = { + "name" = "default" + "namespace" = var.namespace + } + "spec" = { + "defaultCertificate" = { + "secretName" = var.certificate-secret-name + } + } + } +} + +resource "kubernetes_persistent_volume_claim" "traefik_certs_claim" { + metadata { + name = "traefik-public-certificates-claim" + namespace = var.namespace + } + + spec { + access_modes = var.access_modes + resources { + requests = { + storage = var.storage_size + } + } + } +} + +resource "kubernetes_storage_class" "traefik_certs_storage_class" { + metadata { + name = "traefik-certs-storage-class" + + } + + storage_provisioner = "kubernetes.io/aws-ebs" + parameters = { + type = var.storage_type + iops = var.iops + } + reclaim_policy = var.reclaim_policy +} + +resource "kubernetes_persistent_volume" "traefik_certs" { + metadata { + name = "traefik-public-certificates" + + } + + spec { + capacity = { + storage = var.storage_size + } + access_modes = var.access_modes + persistent_volume_source { + vsphere_volume { + volume_path = "/data" + } + } + storage_class_name = kubernetes_storage_class.traefik_certs_storage_class.metadata.0.name + } +} \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf index ac6b517922..43ee0ea083 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf @@ -1,9 +1,15 @@ -locals { - ingress = kubernetes_service.main.status.0.load_balancer.0.ingress -} - -output "endpoint" { - description = "traefik load balancer endpoint" - // handles the case when ingress is empty list - value = length(local.ingress) == 0 ? null : local.ingress.0 -} +locals { + ingress = kubernetes_service.main.status.0.load_balancer.0.ingress +} + +output "endpoint" { + description = "traefik load balancer endpoint" + // handles the case when ingress is empty list + value = length(local.ingress) == 0 ? null : local.ingress.0 +} + + + +output "traefik_certs_pvc_name" { + value = kubernetes_persistent_volume_claim.traefik_certs_claim.metadata.0.name +} \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index 20e869a13d..2d63406cc1 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -1,75 +1,104 @@ -variable "name" { - description = "name prefix to assign to traefik" - type = string - default = "nebari" -} - -variable "namespace" { - description = "namespace to deploy traefik" - type = string -} - -variable "node-group" { - description = "Node group to associate ingress deployment" - type = object({ - key = string - value = string - }) - -} - -variable "traefik-image" { - description = "traefik image to use" - type = object({ - image = string - tag = string - }) -} - -variable "loglevel" { - description = "traefik log level" - default = "WARN" -} - -variable "acme-email" { - description = "ACME server email" - default = "costrouchov@quansight.com" -} - -variable "acme-server" { - description = "ACME server" - # for testing use the letencrypt staging server - # - staging: https://acme-staging-v02.api.letsencrypt.org/directory - # - production: https://acme-v02.api.letsencrypt.org/directory - default = "https://acme-staging-v02.api.letsencrypt.org/directory" -} - -variable "certificate-secret-name" { - description = "Kubernetes secret used for certificate" - type = string - default = null -} - -variable "load-balancer-ip" { - description = "IP Address of the load balancer" - type = string - default = null -} - -variable "load-balancer-annotations" { - description = "Annotations for the load balancer" - type = map(string) - default = null -} - -variable "certificate-service" { - description = "The certificate service to use" - type = string - default = "self-signed" -} - -variable "additional-arguments" { - description = "Additional command line arguments to supply to traefik ingress" - type = list(string) - default = [] -} +variable "name" { + description = "name prefix to assign to traefik" + type = string + default = "nebari" +} + +variable "namespace" { + description = "namespace to deploy traefik" + type = string +} + +variable "node-group" { + description = "Node group to associate ingress deployment" + type = object({ + key = string + value = string + }) + +} + +variable "traefik-image" { + description = "traefik image to use" + type = object({ + image = string + tag = string + }) +} + +variable "loglevel" { + description = "traefik log level" + default = "WARN" +} + +variable "acme-email" { + description = "ACME server email" + default = "costrouchov@quansight.com" +} + +variable "acme-server" { + description = "ACME server" + # for testing use the letencrypt staging server + # - staging: https://acme-staging-v02.api.letsencrypt.org/directory + # - production: https://acme-v02.api.letsencrypt.org/directory + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} + +variable "certificate-secret-name" { + description = "Kubernetes secret used for certificate" + type = string + default = null +} + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(string) + default = null +} + +variable "certificate-service" { + description = "The certificate service to use" + type = string + default = "self-signed" +} + +variable "additional-arguments" { + description = "Additional command line arguments to supply to traefik ingress" + type = list(string) + default = [] +} + +variable "storage_size" { + type = string + default = "1Gi" +} + +variable "access_modes" { + type = list(string) + default = ["ReadWriteOnce"] +} + +variable "storage_type" { + type = string + default = "gp3" +} + +variable "iops" { + type = number + default = 100 +} + +variable "reclaim_policy" { + type = string + default = "Retain" +} +variable "path" { + type = string + default ="./" +} \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/outputs.tf index 33cff35c36..8de01ef97f 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/outputs.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/outputs.tf @@ -1,4 +1,5 @@ -output "load_balancer_address" { - description = "traefik load balancer address" - value = module.kubernetes-ingress.endpoint -} +output "load_balancer_address" { + description = "traefik load balancer address" + value = module.kubernetes-ingress.endpoint +} + diff --git a/src/_nebari/stages/kubernetes_ingress/template/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf index ebafdac2f7..f22b30f875 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/variables.tf @@ -1,71 +1,97 @@ -variable "name" { - description = "Prefix name to assign to ingress kubernetes resources" - type = string -} - -variable "environment" { - description = "Kubernetes namespace to deploy ingress resources" - type = string -} - -variable "node_groups" { - description = "Node group selectors for kubernetes resources" - type = map(object({ - key = string - value = string - })) -} - -variable "traefik-image" { - description = "traefik image to use" - type = object({ - image = string - tag = string - }) -} - -variable "acme-email" { - description = "ACME server email" - default = "nebari@example.com" -} - -variable "acme-server" { - description = "ACME server" - # for testing use the letencrypt staging server - # - staging: https://acme-staging-v02.api.letsencrypt.org/directory - # - production: https://acme-v02.api.letsencrypt.org/directory - default = "https://acme-staging-v02.api.letsencrypt.org/directory" -} - -variable "certificate-secret-name" { - description = "Kubernetes secret used for certificate" - default = "" -} - - -variable "load-balancer-ip" { - description = "IP Address of the load balancer" - type = string - default = null -} - - -variable "load-balancer-annotations" { - description = "Annotations for the load balancer" - type = map(string) - default = null -} - - -variable "certificate-service" { - description = "The certificate service to use" - type = string - default = "self-signed" -} - - -variable "additional-arguments" { - description = "Additional command line arguments to supply to traefik ingress" - type = list(string) - default = [] -} +variable "name" { + description = "Prefix name to assign to ingress kubernetes resources" + type = string +} + +variable "environment" { + description = "Kubernetes namespace to deploy ingress resources" + type = string +} + +variable "node_groups" { + description = "Node group selectors for kubernetes resources" + type = map(object({ + key = string + value = string + })) +} + +variable "traefik-image" { + description = "traefik image to use" + type = object({ + image = string + tag = string + }) +} + +variable "acme-email" { + description = "ACME server email" + default = "nebari@example.com" +} + +variable "acme-server" { + description = "ACME server" + # for testing use the letencrypt staging server + # - staging: https://acme-staging-v02.api.letsencrypt.org/directory + # - production: https://acme-v02.api.letsencrypt.org/directory + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} + +variable "certificate-secret-name" { + description = "Kubernetes secret used for certificate" + default = "" +} + + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(string) + default = null +} + + +variable "certificate-service" { + description = "The certificate service to use" + type = string + default = "self-signed" +} + + +variable "additional-arguments" { + description = "Additional command line arguments to supply to traefik ingress" + type = list(string) + default = [] +} + + +variable "storage_size" { + type = string + default = "1Gi" +} + +variable "access_modes" { + type = list(string) + default = ["ReadWriteOnce"] +} + +variable "storage_type" { + type = string + default = "gp3" +} + +variable "iops" { + type = number + default = 100 +} + +variable "reclaim_policy" { + type = string + default = "Retain" +} \ No newline at end of file From 19daf93335be782d35a2b7f7bf4b49c943ac35ff Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 21 Feb 2024 19:12:55 +0000 Subject: [PATCH 02/12] pv and pvc manifest yaml added and volume attached to pod --- .../modules/kubernetes/ingress/main.tf | 44 ++++++++++++------- .../modules/kubernetes/ingress/outputs.tf | 8 ++-- .../modules/kubernetes/ingress/pv.yaml | 13 ++++++ .../modules/kubernetes/ingress/pv_claim.yaml | 13 ++++++ .../modules/kubernetes/ingress/variables.tf | 2 +- .../kubernetes_ingress/template/variables.tf | 2 +- 6 files changed, 60 insertions(+), 22 deletions(-) create mode 100644 src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml create mode 100644 src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 6bdb8c0331..886171a067 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -214,7 +214,10 @@ resource "kubernetes_deployment" "main" { container { image = "${var.traefik-image.image}:${var.traefik-image.tag}" name = var.name - + volume_mount { + name = "traefik-certs" + mount_path = "/tmp/traefik-persistent-volume" + } security_context { capabilities { drop = ["ALL"] @@ -326,6 +329,12 @@ resource "kubernetes_deployment" "main" { success_threshold = 1 } } + volume { + name = "traefik-certs" + persistent_volume_claim { + claim_name = "persistent-volume-claim" + } + } } } } @@ -349,21 +358,7 @@ resource "kubernetes_manifest" "tlsstore_default" { } } -resource "kubernetes_persistent_volume_claim" "traefik_certs_claim" { - metadata { - name = "traefik-public-certificates-claim" - namespace = var.namespace - } - spec { - access_modes = var.access_modes - resources { - requests = { - storage = var.storage_size - } - } - } -} resource "kubernetes_storage_class" "traefik_certs_storage_class" { metadata { @@ -397,4 +392,21 @@ resource "kubernetes_persistent_volume" "traefik_certs" { } storage_class_name = kubernetes_storage_class.traefik_certs_storage_class.metadata.0.name } -} \ No newline at end of file +} + +# resource "kubernetes_persistent_volume_claim" "traefik_certs_claim" { + # metadata { + # name = "traefik-public-certificates-claim" + # namespace = var.namespace + # } + + # spec { + # access_modes = var.access_modes + # resources { + # requests = { + # storage = var.storage_size + # } + # } + # } +# } + diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf index 43ee0ea083..f7186db304 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf @@ -9,7 +9,7 @@ output "endpoint" { } - -output "traefik_certs_pvc_name" { - value = kubernetes_persistent_volume_claim.traefik_certs_claim.metadata.0.name -} \ No newline at end of file +# +#output "traefik_certs_pvc_name" { + # value = kubernetes_persistent_volume_claim.traefik_certs_claim.metadata.0.name +#} \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml new file mode 100644 index 0000000000..edfa6f71d4 --- /dev/null +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: traefik-persistent-volume-2 + namespace: playground-prince +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 10Gi + storageClassName: standard + hostPath: + path: /tmp/traefik-persistent-volume \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml new file mode 100644 index 0000000000..0cf3b47b7c --- /dev/null +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: persistent-volume-claim + namespace: playground-prince +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: standard + volumeName: traefik-persistent-volume-1 \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index 2d63406cc1..061708b972 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -86,7 +86,7 @@ variable "access_modes" { variable "storage_type" { type = string - default = "gp3" + default = "gp2" } variable "iops" { diff --git a/src/_nebari/stages/kubernetes_ingress/template/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf index f22b30f875..77a3a351b2 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/variables.tf @@ -83,7 +83,7 @@ variable "access_modes" { variable "storage_type" { type = string - default = "gp3" + default = "gp2" } variable "iops" { From df855fe40247813f5257bd777e96b0a8060f414e Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Thu, 22 Feb 2024 20:14:36 +0000 Subject: [PATCH 03/12] terraform manifest for pvc and pv --- .../modules/kubernetes/ingress/main.tf | 78 +++++++++++-------- .../modules/kubernetes/ingress/pv.yaml | 2 +- .../modules/kubernetes/ingress/pv_claim.yaml | 4 +- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 886171a067..be0cdb0f78 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -9,7 +9,7 @@ locals { "--entrypoints.minio.http.tls.certResolver=letsencrypt", "--certificatesresolvers.letsencrypt.acme.tlschallenge", "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", - "--certificatesresolvers.letsencrypt.acme.storage=acme.json", + "--certificatesresolvers.letsencrypt.acme.storage=/tmp/acme-certificates/acme.json", "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", ] self-signed = local.default_cert @@ -216,7 +216,7 @@ resource "kubernetes_deployment" "main" { name = var.name volume_mount { name = "traefik-certs" - mount_path = "/tmp/traefik-persistent-volume" + mount_path = "/tmp/acme-certificates" } security_context { capabilities { @@ -332,7 +332,7 @@ resource "kubernetes_deployment" "main" { volume { name = "traefik-certs" persistent_volume_claim { - claim_name = "persistent-volume-claim" + claim_name = "persistent-volume-claim-2" } } } @@ -374,39 +374,49 @@ resource "kubernetes_storage_class" "traefik_certs_storage_class" { reclaim_policy = var.reclaim_policy } -resource "kubernetes_persistent_volume" "traefik_certs" { - metadata { - name = "traefik-public-certificates" - - } - - spec { - capacity = { - storage = var.storage_size - } - access_modes = var.access_modes - persistent_volume_source { - vsphere_volume { - volume_path = "/data" - } +resource "kubernetes_persistent_volume" "traefik-persistent-volume" { + manifest = +{ + "apiVersion"= "v1", + "kind"= "PersistentVolume", + "metadata"= { + "name"= "traefik-persistent-volume" + "namespace" = var.namespace + + }, + "spec"= { + "accessModes"= ["ReadWriteOnce"], + "capacity"= { + "storage"= "1Gi" + }, + "storageClassName"= "standard", + "hostPath"= { + "path"= "/tmp/acme-certificates" } - storage_class_name = kubernetes_storage_class.traefik_certs_storage_class.metadata.0.name } } + +} -# resource "kubernetes_persistent_volume_claim" "traefik_certs_claim" { - # metadata { - # name = "traefik-public-certificates-claim" - # namespace = var.namespace - # } - - # spec { - # access_modes = var.access_modes - # resources { - # requests = { - # storage = var.storage_size - # } - # } - # } -# } +resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { + manifest = +{ + "apiVersion"= "v1", + "kind"= "PersistentVolumeClaim", + "metadata"= { + "name"= "persistent-volume-claim" + "namespace" = var.namespace + }, + "spec"= { + "accessModes"= ["ReadWriteOnce"], + "resources"= { + "requests"= { + "storage"= "1Gi" + } + }, + "storageClassName"= "standard", + "volumeName"= "traefik-persistent-volume" + } +} +} diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml index edfa6f71d4..d6bb1be8aa 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml @@ -10,4 +10,4 @@ spec: storage: 10Gi storageClassName: standard hostPath: - path: /tmp/traefik-persistent-volume \ No newline at end of file + path: /tmp/acme-certificates \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml index 0cf3b47b7c..6a0f1cfda5 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml @@ -1,7 +1,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: persistent-volume-claim + name: persistent-volume-claim-2 namespace: playground-prince spec: accessModes: @@ -10,4 +10,4 @@ spec: requests: storage: 10Gi storageClassName: standard - volumeName: traefik-persistent-volume-1 \ No newline at end of file + volumeName: traefik-persistent-volume-2 \ No newline at end of file From a25067ed18edac57ed94962005869706a014b858 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Mon, 26 Feb 2024 17:50:21 +0000 Subject: [PATCH 04/12] manifest syntax issue fix --- .../template/modules/kubernetes/ingress/main.tf | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index be0cdb0f78..e24b819418 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -374,15 +374,13 @@ resource "kubernetes_storage_class" "traefik_certs_storage_class" { reclaim_policy = var.reclaim_policy } -resource "kubernetes_persistent_volume" "traefik-persistent-volume" { - manifest = -{ +resource "kubernetes_manifest" "traefik-persistent-volume" { + manifest = { "apiVersion"= "v1", "kind"= "PersistentVolume", "metadata"= { "name"= "traefik-persistent-volume" - "namespace" = var.namespace - + }, "spec"= { "accessModes"= ["ReadWriteOnce"], @@ -398,9 +396,8 @@ resource "kubernetes_persistent_volume" "traefik-persistent-volume" { } -resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { - manifest = -{ +resource "kubernetes_manifest" "persistent-volume-claim" { + manifest = { "apiVersion"= "v1", "kind"= "PersistentVolumeClaim", "metadata"= { From 61886ec7f6c19c6a1ef8a5cc237da9b34b10fc29 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Tue, 27 Feb 2024 15:43:03 +0000 Subject: [PATCH 05/12] terraform code fix --- .../template/modules/kubernetes/ingress/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index e24b819418..f0890f9095 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -332,7 +332,7 @@ resource "kubernetes_deployment" "main" { volume { name = "traefik-certs" persistent_volume_claim { - claim_name = "persistent-volume-claim-2" + claim_name = "persistent-volume-claim" } } } From 23e27eb00b1a350041a69b0a34dd68b36b8feda2 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Tue, 27 Feb 2024 15:56:27 +0000 Subject: [PATCH 06/12] removed hardcoded value --- .../template/modules/kubernetes/ingress/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index f0890f9095..1de6a8d564 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -332,7 +332,7 @@ resource "kubernetes_deployment" "main" { volume { name = "traefik-certs" persistent_volume_claim { - claim_name = "persistent-volume-claim" + claim_name = kubernetes_persistent_volume_claim.persistent-volume-claim.name } } } From cde9a6e87d1b287625408c097b1d5bbdaccd4890 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 28 Feb 2024 15:04:43 +0000 Subject: [PATCH 07/12] PV implimentation --- .../kubernetes_ingress/template/main.tf | 5 +- .../modules/kubernetes/ingress/main.tf | 104 +++++++----------- .../modules/kubernetes/ingress/pv.yaml | 13 --- .../modules/kubernetes/ingress/pv_claim.yaml | 13 --- .../modules/kubernetes/ingress/variables.tf | 19 +--- .../kubernetes_ingress/template/variables.tf | 15 --- 6 files changed, 43 insertions(+), 126 deletions(-) delete mode 100644 src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml delete mode 100644 src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml diff --git a/src/_nebari/stages/kubernetes_ingress/template/main.tf b/src/_nebari/stages/kubernetes_ingress/template/main.tf index f1df788bf7..988ba6121b 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/main.tf @@ -15,9 +15,6 @@ module "kubernetes-ingress" { load-balancer-ip = var.load-balancer-ip additional-arguments = var.additional-arguments - storage_size = var.storage_size access_modes = var.access_modes - storage_type = var.storage_type - iops = var.iops - reclaim_policy = var.reclaim_policy + } diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 1de6a8d564..c09ea8d4ab 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -171,6 +171,46 @@ resource "kubernetes_service" "traefik_internal" { } } + + + +resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { + metadata { + name = "persistent-volume-claim" + namespace = var.namespace + } + spec { + access_modes = ["ReadWriteMany"] + storage_class_name="standard" + resources { + requests = { + storage = "5Gi" + } + } + volume_name = "${kubernetes_persistent_volume.traefik-persistent-volume.metadata.0.name}" + } +} + +resource "kubernetes_persistent_volume" "traefik-persistent-volume" { + metadata { + name = "traefik-persistent-volume" + } + spec { + capacity = { + storage = "10Gi" + } + storage_class_name="standard" + access_modes = ["ReadWriteMany"] + + persistent_volume_source { + vsphere_volume { + volume_path = "/tmp/acme-certificates" + } + } + } +} + + resource "kubernetes_deployment" "main" { metadata { name = "${var.name}-traefik-ingress" @@ -214,10 +254,6 @@ resource "kubernetes_deployment" "main" { container { image = "${var.traefik-image.image}:${var.traefik-image.tag}" name = var.name - volume_mount { - name = "traefik-certs" - mount_path = "/tmp/acme-certificates" - } security_context { capabilities { drop = ["ALL"] @@ -332,7 +368,7 @@ resource "kubernetes_deployment" "main" { volume { name = "traefik-certs" persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim.persistent-volume-claim.name + claim_name = kubernetes_persistent_volume_claim.persistent-volume-claim.metadata.0.name } } } @@ -359,61 +395,3 @@ resource "kubernetes_manifest" "tlsstore_default" { } - -resource "kubernetes_storage_class" "traefik_certs_storage_class" { - metadata { - name = "traefik-certs-storage-class" - - } - - storage_provisioner = "kubernetes.io/aws-ebs" - parameters = { - type = var.storage_type - iops = var.iops - } - reclaim_policy = var.reclaim_policy -} - -resource "kubernetes_manifest" "traefik-persistent-volume" { - manifest = { - "apiVersion"= "v1", - "kind"= "PersistentVolume", - "metadata"= { - "name"= "traefik-persistent-volume" - - }, - "spec"= { - "accessModes"= ["ReadWriteOnce"], - "capacity"= { - "storage"= "1Gi" - }, - "storageClassName"= "standard", - "hostPath"= { - "path"= "/tmp/acme-certificates" - } - } -} - -} - -resource "kubernetes_manifest" "persistent-volume-claim" { - manifest = { - "apiVersion"= "v1", - "kind"= "PersistentVolumeClaim", - "metadata"= { - "name"= "persistent-volume-claim" - "namespace" = var.namespace - }, - "spec"= { - "accessModes"= ["ReadWriteOnce"], - "resources"= { - "requests"= { - "storage"= "1Gi" - } - }, - "storageClassName"= "standard", - "volumeName"= "traefik-persistent-volume" - } -} - -} diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml deleted file mode 100644 index d6bb1be8aa..0000000000 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolume -metadata: - name: traefik-persistent-volume-2 - namespace: playground-prince -spec: - accessModes: - - ReadWriteOnce - capacity: - storage: 10Gi - storageClassName: standard - hostPath: - path: /tmp/acme-certificates \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml deleted file mode 100644 index 6a0f1cfda5..0000000000 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/pv_claim.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: persistent-volume-claim-2 - namespace: playground-prince -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: standard - volumeName: traefik-persistent-volume-2 \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index 061708b972..2d73c435ac 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -74,30 +74,13 @@ variable "additional-arguments" { default = [] } -variable "storage_size" { - type = string - default = "1Gi" -} + variable "access_modes" { type = list(string) default = ["ReadWriteOnce"] } -variable "storage_type" { - type = string - default = "gp2" -} - -variable "iops" { - type = number - default = 100 -} - -variable "reclaim_policy" { - type = string - default = "Retain" -} variable "path" { type = string default ="./" diff --git a/src/_nebari/stages/kubernetes_ingress/template/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf index 77a3a351b2..6a3ce37742 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/variables.tf @@ -80,18 +80,3 @@ variable "access_modes" { type = list(string) default = ["ReadWriteOnce"] } - -variable "storage_type" { - type = string - default = "gp2" -} - -variable "iops" { - type = number - default = 100 -} - -variable "reclaim_policy" { - type = string - default = "Retain" -} \ No newline at end of file From 979ff7f4cb6388d16761c673de4999a0e0a5e715 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 28 Feb 2024 16:10:01 +0000 Subject: [PATCH 08/12] variable name fix --- .../stages/kubernetes_ingress/template/main.tf | 4 ++++ .../template/modules/kubernetes/ingress/main.tf | 6 +++--- .../modules/kubernetes/ingress/variables.tf | 10 +++++++++- .../stages/kubernetes_ingress/template/variables.tf | 13 +++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/main.tf b/src/_nebari/stages/kubernetes_ingress/template/main.tf index 988ba6121b..06851ff131 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/main.tf @@ -16,5 +16,9 @@ module "kubernetes-ingress" { additional-arguments = var.additional-arguments access_modes = var.access_modes + pvc_name = var.pvc_name + pv_name = var.pv_name + path = var.path + } diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index c09ea8d4ab..91960e7eec 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -176,7 +176,7 @@ resource "kubernetes_service" "traefik_internal" { resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { metadata { - name = "persistent-volume-claim" + name = "{var.pvc_name}-traefik-ingress" namespace = var.namespace } spec { @@ -193,7 +193,7 @@ resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { resource "kubernetes_persistent_volume" "traefik-persistent-volume" { metadata { - name = "traefik-persistent-volume" + name = "{var.pv_name}-traefik-ingress" } spec { capacity = { @@ -204,7 +204,7 @@ resource "kubernetes_persistent_volume" "traefik-persistent-volume" { persistent_volume_source { vsphere_volume { - volume_path = "/tmp/acme-certificates" + volume_path = var.path } } } diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index 2d73c435ac..1c5e814a05 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -80,8 +80,16 @@ variable "access_modes" { type = list(string) default = ["ReadWriteOnce"] } +variable "pvc_name"{ + type = string + default = "persistent_volume_claim" +} +variable "pv_name"{ + type = string + default = "traefik_persistent_volume" +} variable "path" { type = string - default ="./" + default = "/tmp/acme-certificates" } \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf index 6a3ce37742..3de1742f7a 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/variables.tf @@ -80,3 +80,16 @@ variable "access_modes" { type = list(string) default = ["ReadWriteOnce"] } +variable "pvc_name"{ + type = string + default = "persistent-volume-claim" +} + +variable "pv_name"{ + type = string + default = "traefik-persistent-volume" +} +variable "path" { + type = string + default = "/tmp/acme-certificates" +} \ No newline at end of file From 0fdd80be9239e18550b030011b5ec5bd39c6723f Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 28 Feb 2024 17:29:06 +0000 Subject: [PATCH 09/12] resource name fix --- .../template/modules/kubernetes/ingress/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index 91960e7eec..de6de8b54e 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -174,7 +174,7 @@ resource "kubernetes_service" "traefik_internal" { -resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { +resource "kubernetes_persistent_volume_claim" "persistent_volume_claim" { metadata { name = "{var.pvc_name}-traefik-ingress" namespace = var.namespace @@ -191,7 +191,7 @@ resource "kubernetes_persistent_volume_claim" "persistent-volume-claim" { } } -resource "kubernetes_persistent_volume" "traefik-persistent-volume" { +resource "kubernetes_persistent_volume" "traefik_persistent_volume" { metadata { name = "{var.pv_name}-traefik-ingress" } From d789dcfa37b4fffa6dbbfab7c30d250c2efe8859 Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 28 Feb 2024 19:49:11 +0000 Subject: [PATCH 10/12] vriable name fix --- .../template/modules/kubernetes/ingress/main.tf | 8 ++++---- .../template/modules/kubernetes/ingress/variables.tf | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index de6de8b54e..fd4462a9fa 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -176,7 +176,7 @@ resource "kubernetes_service" "traefik_internal" { resource "kubernetes_persistent_volume_claim" "persistent_volume_claim" { metadata { - name = "{var.pvc_name}-traefik-ingress" + name = "${var.pvc_name}-traefik-ingress" namespace = var.namespace } spec { @@ -187,13 +187,13 @@ resource "kubernetes_persistent_volume_claim" "persistent_volume_claim" { storage = "5Gi" } } - volume_name = "${kubernetes_persistent_volume.traefik-persistent-volume.metadata.0.name}" + volume_name = "${kubernetes_persistent_volume.traefik_persistent_volume.metadata.0.name}" } } resource "kubernetes_persistent_volume" "traefik_persistent_volume" { metadata { - name = "{var.pv_name}-traefik-ingress" + name = "${var.pv_name}-traefik-ingress" } spec { capacity = { @@ -368,7 +368,7 @@ resource "kubernetes_deployment" "main" { volume { name = "traefik-certs" persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim.persistent-volume-claim.metadata.0.name + claim_name = kubernetes_persistent_volume_claim.persistent_volume_claim.metadata.0.name } } } diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index 1c5e814a05..a90a51c5d0 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -82,12 +82,12 @@ variable "access_modes" { } variable "pvc_name"{ type = string - default = "persistent_volume_claim" + default = "persistent-volume-claim" } variable "pv_name"{ type = string - default = "traefik_persistent_volume" + default = "traefik-persistent-volume" } variable "path" { type = string From 34fe52884dc199d04658cfe23cb2a0517c393bce Mon Sep 17 00:00:00 2001 From: Prince Chima Ogbonna Date: Wed, 6 Mar 2024 14:53:26 +0000 Subject: [PATCH 11/12] eks code fix for volume source --- .../template/modules/kubernetes/ingress/main.tf | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index fd4462a9fa..a28eea9d8b 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -203,8 +203,8 @@ resource "kubernetes_persistent_volume" "traefik_persistent_volume" { access_modes = ["ReadWriteMany"] persistent_volume_source { - vsphere_volume { - volume_path = var.path + host_path { + path = var.path } } } @@ -254,6 +254,10 @@ resource "kubernetes_deployment" "main" { container { image = "${var.traefik-image.image}:${var.traefik-image.tag}" name = var.name + volume_mount { + mount_path = var.path + name = "data" + } security_context { capabilities { drop = ["ALL"] @@ -366,7 +370,7 @@ resource "kubernetes_deployment" "main" { } } volume { - name = "traefik-certs" + name = "data" persistent_volume_claim { claim_name = kubernetes_persistent_volume_claim.persistent_volume_claim.metadata.0.name } From 9140482145550687c43814bc1ad82d602eb09c19 Mon Sep 17 00:00:00 2001 From: Ken Foster Date: Wed, 6 Mar 2024 15:23:37 +0000 Subject: [PATCH 12/12] Fix DOS line endings --- .../kubernetes_ingress/template/main.tf | 48 +- .../modules/kubernetes/ingress/main.tf | 802 +++++++++--------- .../modules/kubernetes/ingress/outputs.tf | 28 +- .../modules/kubernetes/ingress/variables.tf | 188 ++-- .../kubernetes_ingress/template/outputs.tf | 10 +- .../kubernetes_ingress/template/variables.tf | 188 ++-- 6 files changed, 632 insertions(+), 632 deletions(-) diff --git a/src/_nebari/stages/kubernetes_ingress/template/main.tf b/src/_nebari/stages/kubernetes_ingress/template/main.tf index 06851ff131..2f20a31fa8 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/main.tf @@ -1,24 +1,24 @@ -module "kubernetes-ingress" { - source = "./modules/kubernetes/ingress" - - namespace = var.environment - - node-group = var.node_groups.general - - traefik-image = var.traefik-image - - certificate-service = var.certificate-service - acme-email = var.acme-email - acme-server = var.acme-server - certificate-secret-name = var.certificate-secret-name - load-balancer-annotations = var.load-balancer-annotations - load-balancer-ip = var.load-balancer-ip - additional-arguments = var.additional-arguments - - access_modes = var.access_modes - pvc_name = var.pvc_name - pv_name = var.pv_name - path = var.path - - -} +module "kubernetes-ingress" { + source = "./modules/kubernetes/ingress" + + namespace = var.environment + + node-group = var.node_groups.general + + traefik-image = var.traefik-image + + certificate-service = var.certificate-service + acme-email = var.acme-email + acme-server = var.acme-server + certificate-secret-name = var.certificate-secret-name + load-balancer-annotations = var.load-balancer-annotations + load-balancer-ip = var.load-balancer-ip + additional-arguments = var.additional-arguments + + access_modes = var.access_modes + pvc_name = var.pvc_name + pv_name = var.pv_name + path = var.path + + +} diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf index a28eea9d8b..132113e436 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/main.tf @@ -1,401 +1,401 @@ -locals { - default_cert = [ - "--entrypoints.websecure.http.tls.certResolver=default", - "--entrypoints.minio.http.tls.certResolver=default", - ] - certificate-settings = { - lets-encrypt = [ - "--entrypoints.websecure.http.tls.certResolver=letsencrypt", - "--entrypoints.minio.http.tls.certResolver=letsencrypt", - "--certificatesresolvers.letsencrypt.acme.tlschallenge", - "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", - "--certificatesresolvers.letsencrypt.acme.storage=/tmp/acme-certificates/acme.json", - "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", - ] - self-signed = local.default_cert - existing = local.default_cert - disabled = [] - } - add-certificate = local.certificate-settings[var.certificate-service] -} - - -resource "kubernetes_service_account" "main" { - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - } -} - - -resource "kubernetes_cluster_role" "main" { - metadata { - name = "${var.name}-traefik-ingress" - } - - rule { - api_groups = [""] - resources = ["services", "endpoints", "secrets"] - verbs = ["get", "list", "watch"] - } - - rule { - api_groups = ["extensions", "networking.k8s.io"] - resources = ["ingresses", "ingressclasses"] - verbs = ["get", "list", "watch"] - } - - rule { - api_groups = ["extensions"] - resources = ["ingresses/status"] - verbs = ["update"] - } - - rule { - api_groups = ["traefik.containo.us"] - resources = ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "middlewaretcps", "tlsoptions", "tlsstores", "traefikservices", "serverstransports"] - verbs = ["get", "list", "watch"] - } -} - - -resource "kubernetes_cluster_role_binding" "main" { - metadata { - name = "${var.name}-traefik-ingress" - } - - role_ref { - api_group = "rbac.authorization.k8s.io" - kind = "ClusterRole" - name = kubernetes_cluster_role.main.metadata.0.name - } - subject { - kind = "ServiceAccount" - name = kubernetes_service_account.main.metadata.0.name - namespace = var.namespace - } -} - - -resource "kubernetes_service" "main" { - wait_for_load_balancer = true - - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - annotations = var.load-balancer-annotations - } - - spec { - selector = { - "app.kubernetes.io/component" = "traefik-ingress" - } - - port { - name = "http" - protocol = "TCP" - port = 80 - target_port = 80 - } - - port { - name = "https" - protocol = "TCP" - port = 443 - target_port = 443 - } - - port { - name = "ssh" - protocol = "TCP" - port = 8022 - target_port = 8022 - } - - port { - name = "sftp" - protocol = "TCP" - port = 8023 - target_port = 8023 - } - - port { - name = "minio" - protocol = "TCP" - port = 9080 - target_port = 9080 - } - - port { - name = "tcp" - protocol = "TCP" - port = 8786 - target_port = 8786 - } - - type = "LoadBalancer" - load_balancer_ip = var.load-balancer-ip - } -} - -resource "kubernetes_service" "traefik_internal" { - wait_for_load_balancer = true - - metadata { - name = "${var.name}-traefik-internal" - namespace = var.namespace - annotations = { - "prometheus.io/scrape" = "true" - "prometheus.io/path" = "/metrics" - "prometheus.io/port" = 9000 - } - labels = { - "app.kubernetes.io/component" = "traefik-internal-service" - "app.kubernetes.io/part-of" = "traefik-ingress" - } - } - - spec { - selector = { - "app.kubernetes.io/component" = "traefik-ingress" - } - - port { - name = "http" - protocol = "TCP" - port = 9000 - target_port = 9000 - } - - type = "ClusterIP" - } -} - - - - -resource "kubernetes_persistent_volume_claim" "persistent_volume_claim" { - metadata { - name = "${var.pvc_name}-traefik-ingress" - namespace = var.namespace - } - spec { - access_modes = ["ReadWriteMany"] - storage_class_name="standard" - resources { - requests = { - storage = "5Gi" - } - } - volume_name = "${kubernetes_persistent_volume.traefik_persistent_volume.metadata.0.name}" - } -} - -resource "kubernetes_persistent_volume" "traefik_persistent_volume" { - metadata { - name = "${var.pv_name}-traefik-ingress" - } - spec { - capacity = { - storage = "10Gi" - } - storage_class_name="standard" - access_modes = ["ReadWriteMany"] - - persistent_volume_source { - host_path { - path = var.path - } - } - } -} - - -resource "kubernetes_deployment" "main" { - metadata { - name = "${var.name}-traefik-ingress" - namespace = var.namespace - } - - spec { - replicas = 1 - - selector { - match_labels = { - "app.kubernetes.io/component" = "traefik-ingress" - } - } - - template { - metadata { - labels = { - "app.kubernetes.io/component" = "traefik-ingress" - } - } - - spec { - service_account_name = kubernetes_service_account.main.metadata.0.name - termination_grace_period_seconds = 60 - - affinity { - node_affinity { - required_during_scheduling_ignored_during_execution { - node_selector_term { - match_expressions { - key = var.node-group.key - operator = "In" - values = [var.node-group.value] - } - } - } - } - } - - container { - image = "${var.traefik-image.image}:${var.traefik-image.tag}" - name = var.name - volume_mount { - mount_path = var.path - name = "data" - } - security_context { - capabilities { - drop = ["ALL"] - add = ["NET_BIND_SERVICE"] - } - } - - args = concat([ - # Do not send usage stats - "--global.checknewversion=false", - "--global.sendanonymoususage=false", - # allow access to the dashboard directly through the port - # TODO: eventually needs to be tied into traefik middle - # security possibly using jupyterhub auth this is not a - # security risk at the moment since this port is not - # externally accessible - "--api.insecure=true", - "--api.dashboard=true", - "--ping=true", - # Start the Traefik Kubernetes Ingress Controller - "--providers.kubernetesingress=true", - "--providers.kubernetesingress.namespaces=${var.namespace}", - "--providers.kubernetesingress.ingressclass=traefik", - # Start the Traefik Kubernetes CRD Controller Provider - "--providers.kubernetescrd", - "--providers.kubernetescrd.namespaces=${var.namespace}", - "--providers.kubernetescrd.throttleduration=2s", - "--providers.kubernetescrd.allowcrossnamespace=false", - # Define two entrypoint ports, and setup a redirect from HTTP to HTTPS. - "--entryPoints.web.address=:80", - "--entryPoints.websecure.address=:443", - "--entrypoints.ssh.address=:8022", - "--entrypoints.sftp.address=:8023", - "--entryPoints.tcp.address=:8786", - "--entryPoints.traefik.address=:9000", - # Define the entrypoint port for Minio - "--entryPoints.minio.address=:9080", - # Redirect http -> https - "--entrypoints.web.http.redirections.entryPoint.to=websecure", - "--entrypoints.web.http.redirections.entryPoint.scheme=https", - # Enable Prometheus Monitoring of Traefik - "--metrics.prometheus=true", - # Enable debug logging. Useful to work out why something might not be - # working. Fetch logs of the pod. - "--log.level=${var.loglevel}", - ], - local.add-certificate, - var.additional-arguments, - ) - - port { - name = "http" - container_port = 80 - } - - port { - name = "https" - container_port = 443 - } - - port { - name = "ssh" - container_port = 8022 - } - - port { - name = "sftp" - container_port = 8023 - } - - port { - name = "tcp" - container_port = 8786 - } - - port { - name = "traefik" - container_port = 9000 - } - - port { - name = "minio" - container_port = 9080 - } - - liveness_probe { - http_get { - path = "/ping" - port = "traefik" - } - - initial_delay_seconds = 10 - timeout_seconds = 2 - period_seconds = 10 - failure_threshold = 3 - success_threshold = 1 - } - - readiness_probe { - http_get { - path = "/ping" - port = "traefik" - } - - initial_delay_seconds = 10 - timeout_seconds = 2 - period_seconds = 10 - failure_threshold = 1 - success_threshold = 1 - } - } - volume { - name = "data" - persistent_volume_claim { - claim_name = kubernetes_persistent_volume_claim.persistent_volume_claim.metadata.0.name - } - } - } - } - } -} - - -resource "kubernetes_manifest" "tlsstore_default" { - count = var.certificate-secret-name != null ? 1 : 0 - manifest = { - "apiVersion" = "traefik.containo.us/v1alpha1" - "kind" = "TLSStore" - "metadata" = { - "name" = "default" - "namespace" = var.namespace - } - "spec" = { - "defaultCertificate" = { - "secretName" = var.certificate-secret-name - } - } - } -} - - +locals { + default_cert = [ + "--entrypoints.websecure.http.tls.certResolver=default", + "--entrypoints.minio.http.tls.certResolver=default", + ] + certificate-settings = { + lets-encrypt = [ + "--entrypoints.websecure.http.tls.certResolver=letsencrypt", + "--entrypoints.minio.http.tls.certResolver=letsencrypt", + "--certificatesresolvers.letsencrypt.acme.tlschallenge", + "--certificatesresolvers.letsencrypt.acme.email=${var.acme-email}", + "--certificatesresolvers.letsencrypt.acme.storage=/tmp/acme-certificates/acme.json", + "--certificatesresolvers.letsencrypt.acme.caserver=${var.acme-server}", + ] + self-signed = local.default_cert + existing = local.default_cert + disabled = [] + } + add-certificate = local.certificate-settings[var.certificate-service] +} + + +resource "kubernetes_service_account" "main" { + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + } +} + + +resource "kubernetes_cluster_role" "main" { + metadata { + name = "${var.name}-traefik-ingress" + } + + rule { + api_groups = [""] + resources = ["services", "endpoints", "secrets"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["extensions", "networking.k8s.io"] + resources = ["ingresses", "ingressclasses"] + verbs = ["get", "list", "watch"] + } + + rule { + api_groups = ["extensions"] + resources = ["ingresses/status"] + verbs = ["update"] + } + + rule { + api_groups = ["traefik.containo.us"] + resources = ["ingressroutes", "ingressroutetcps", "ingressrouteudps", "middlewares", "middlewaretcps", "tlsoptions", "tlsstores", "traefikservices", "serverstransports"] + verbs = ["get", "list", "watch"] + } +} + + +resource "kubernetes_cluster_role_binding" "main" { + metadata { + name = "${var.name}-traefik-ingress" + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "ClusterRole" + name = kubernetes_cluster_role.main.metadata.0.name + } + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.main.metadata.0.name + namespace = var.namespace + } +} + + +resource "kubernetes_service" "main" { + wait_for_load_balancer = true + + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + annotations = var.load-balancer-annotations + } + + spec { + selector = { + "app.kubernetes.io/component" = "traefik-ingress" + } + + port { + name = "http" + protocol = "TCP" + port = 80 + target_port = 80 + } + + port { + name = "https" + protocol = "TCP" + port = 443 + target_port = 443 + } + + port { + name = "ssh" + protocol = "TCP" + port = 8022 + target_port = 8022 + } + + port { + name = "sftp" + protocol = "TCP" + port = 8023 + target_port = 8023 + } + + port { + name = "minio" + protocol = "TCP" + port = 9080 + target_port = 9080 + } + + port { + name = "tcp" + protocol = "TCP" + port = 8786 + target_port = 8786 + } + + type = "LoadBalancer" + load_balancer_ip = var.load-balancer-ip + } +} + +resource "kubernetes_service" "traefik_internal" { + wait_for_load_balancer = true + + metadata { + name = "${var.name}-traefik-internal" + namespace = var.namespace + annotations = { + "prometheus.io/scrape" = "true" + "prometheus.io/path" = "/metrics" + "prometheus.io/port" = 9000 + } + labels = { + "app.kubernetes.io/component" = "traefik-internal-service" + "app.kubernetes.io/part-of" = "traefik-ingress" + } + } + + spec { + selector = { + "app.kubernetes.io/component" = "traefik-ingress" + } + + port { + name = "http" + protocol = "TCP" + port = 9000 + target_port = 9000 + } + + type = "ClusterIP" + } +} + + + + +resource "kubernetes_persistent_volume_claim" "persistent_volume_claim" { + metadata { + name = "${var.pvc_name}-traefik-ingress" + namespace = var.namespace + } + spec { + access_modes = ["ReadWriteMany"] + storage_class_name="standard" + resources { + requests = { + storage = "5Gi" + } + } + volume_name = "${kubernetes_persistent_volume.traefik_persistent_volume.metadata.0.name}" + } +} + +resource "kubernetes_persistent_volume" "traefik_persistent_volume" { + metadata { + name = "${var.pv_name}-traefik-ingress" + } + spec { + capacity = { + storage = "10Gi" + } + storage_class_name="standard" + access_modes = ["ReadWriteMany"] + + persistent_volume_source { + host_path { + path = var.path + } + } + } +} + + +resource "kubernetes_deployment" "main" { + metadata { + name = "${var.name}-traefik-ingress" + namespace = var.namespace + } + + spec { + replicas = 1 + + selector { + match_labels = { + "app.kubernetes.io/component" = "traefik-ingress" + } + } + + template { + metadata { + labels = { + "app.kubernetes.io/component" = "traefik-ingress" + } + } + + spec { + service_account_name = kubernetes_service_account.main.metadata.0.name + termination_grace_period_seconds = 60 + + affinity { + node_affinity { + required_during_scheduling_ignored_during_execution { + node_selector_term { + match_expressions { + key = var.node-group.key + operator = "In" + values = [var.node-group.value] + } + } + } + } + } + + container { + image = "${var.traefik-image.image}:${var.traefik-image.tag}" + name = var.name + volume_mount { + mount_path = var.path + name = "data" + } + security_context { + capabilities { + drop = ["ALL"] + add = ["NET_BIND_SERVICE"] + } + } + + args = concat([ + # Do not send usage stats + "--global.checknewversion=false", + "--global.sendanonymoususage=false", + # allow access to the dashboard directly through the port + # TODO: eventually needs to be tied into traefik middle + # security possibly using jupyterhub auth this is not a + # security risk at the moment since this port is not + # externally accessible + "--api.insecure=true", + "--api.dashboard=true", + "--ping=true", + # Start the Traefik Kubernetes Ingress Controller + "--providers.kubernetesingress=true", + "--providers.kubernetesingress.namespaces=${var.namespace}", + "--providers.kubernetesingress.ingressclass=traefik", + # Start the Traefik Kubernetes CRD Controller Provider + "--providers.kubernetescrd", + "--providers.kubernetescrd.namespaces=${var.namespace}", + "--providers.kubernetescrd.throttleduration=2s", + "--providers.kubernetescrd.allowcrossnamespace=false", + # Define two entrypoint ports, and setup a redirect from HTTP to HTTPS. + "--entryPoints.web.address=:80", + "--entryPoints.websecure.address=:443", + "--entrypoints.ssh.address=:8022", + "--entrypoints.sftp.address=:8023", + "--entryPoints.tcp.address=:8786", + "--entryPoints.traefik.address=:9000", + # Define the entrypoint port for Minio + "--entryPoints.minio.address=:9080", + # Redirect http -> https + "--entrypoints.web.http.redirections.entryPoint.to=websecure", + "--entrypoints.web.http.redirections.entryPoint.scheme=https", + # Enable Prometheus Monitoring of Traefik + "--metrics.prometheus=true", + # Enable debug logging. Useful to work out why something might not be + # working. Fetch logs of the pod. + "--log.level=${var.loglevel}", + ], + local.add-certificate, + var.additional-arguments, + ) + + port { + name = "http" + container_port = 80 + } + + port { + name = "https" + container_port = 443 + } + + port { + name = "ssh" + container_port = 8022 + } + + port { + name = "sftp" + container_port = 8023 + } + + port { + name = "tcp" + container_port = 8786 + } + + port { + name = "traefik" + container_port = 9000 + } + + port { + name = "minio" + container_port = 9080 + } + + liveness_probe { + http_get { + path = "/ping" + port = "traefik" + } + + initial_delay_seconds = 10 + timeout_seconds = 2 + period_seconds = 10 + failure_threshold = 3 + success_threshold = 1 + } + + readiness_probe { + http_get { + path = "/ping" + port = "traefik" + } + + initial_delay_seconds = 10 + timeout_seconds = 2 + period_seconds = 10 + failure_threshold = 1 + success_threshold = 1 + } + } + volume { + name = "data" + persistent_volume_claim { + claim_name = kubernetes_persistent_volume_claim.persistent_volume_claim.metadata.0.name + } + } + } + } + } +} + + +resource "kubernetes_manifest" "tlsstore_default" { + count = var.certificate-secret-name != null ? 1 : 0 + manifest = { + "apiVersion" = "traefik.containo.us/v1alpha1" + "kind" = "TLSStore" + "metadata" = { + "name" = "default" + "namespace" = var.namespace + } + "spec" = { + "defaultCertificate" = { + "secretName" = var.certificate-secret-name + } + } + } +} + + diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf index f7186db304..173c902a01 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/outputs.tf @@ -1,15 +1,15 @@ -locals { - ingress = kubernetes_service.main.status.0.load_balancer.0.ingress -} - -output "endpoint" { - description = "traefik load balancer endpoint" - // handles the case when ingress is empty list - value = length(local.ingress) == 0 ? null : local.ingress.0 -} - - -# -#output "traefik_certs_pvc_name" { - # value = kubernetes_persistent_volume_claim.traefik_certs_claim.metadata.0.name +locals { + ingress = kubernetes_service.main.status.0.load_balancer.0.ingress +} + +output "endpoint" { + description = "traefik load balancer endpoint" + // handles the case when ingress is empty list + value = length(local.ingress) == 0 ? null : local.ingress.0 +} + + +# +#output "traefik_certs_pvc_name" { + # value = kubernetes_persistent_volume_claim.traefik_certs_claim.metadata.0.name #} \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf index a90a51c5d0..59aa8816c9 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/modules/kubernetes/ingress/variables.tf @@ -1,95 +1,95 @@ -variable "name" { - description = "name prefix to assign to traefik" - type = string - default = "nebari" -} - -variable "namespace" { - description = "namespace to deploy traefik" - type = string -} - -variable "node-group" { - description = "Node group to associate ingress deployment" - type = object({ - key = string - value = string - }) - -} - -variable "traefik-image" { - description = "traefik image to use" - type = object({ - image = string - tag = string - }) -} - -variable "loglevel" { - description = "traefik log level" - default = "WARN" -} - -variable "acme-email" { - description = "ACME server email" - default = "costrouchov@quansight.com" -} - -variable "acme-server" { - description = "ACME server" - # for testing use the letencrypt staging server - # - staging: https://acme-staging-v02.api.letsencrypt.org/directory - # - production: https://acme-v02.api.letsencrypt.org/directory - default = "https://acme-staging-v02.api.letsencrypt.org/directory" -} - -variable "certificate-secret-name" { - description = "Kubernetes secret used for certificate" - type = string - default = null -} - -variable "load-balancer-ip" { - description = "IP Address of the load balancer" - type = string - default = null -} - -variable "load-balancer-annotations" { - description = "Annotations for the load balancer" - type = map(string) - default = null -} - -variable "certificate-service" { - description = "The certificate service to use" - type = string - default = "self-signed" -} - -variable "additional-arguments" { - description = "Additional command line arguments to supply to traefik ingress" - type = list(string) - default = [] -} - - - -variable "access_modes" { - type = list(string) - default = ["ReadWriteOnce"] -} -variable "pvc_name"{ - type = string - default = "persistent-volume-claim" -} - -variable "pv_name"{ - type = string - default = "traefik-persistent-volume" -} -variable "path" { - type = string - default = "/tmp/acme-certificates" +variable "name" { + description = "name prefix to assign to traefik" + type = string + default = "nebari" +} + +variable "namespace" { + description = "namespace to deploy traefik" + type = string +} + +variable "node-group" { + description = "Node group to associate ingress deployment" + type = object({ + key = string + value = string + }) + +} + +variable "traefik-image" { + description = "traefik image to use" + type = object({ + image = string + tag = string + }) +} + +variable "loglevel" { + description = "traefik log level" + default = "WARN" +} + +variable "acme-email" { + description = "ACME server email" + default = "costrouchov@quansight.com" +} + +variable "acme-server" { + description = "ACME server" + # for testing use the letencrypt staging server + # - staging: https://acme-staging-v02.api.letsencrypt.org/directory + # - production: https://acme-v02.api.letsencrypt.org/directory + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} + +variable "certificate-secret-name" { + description = "Kubernetes secret used for certificate" + type = string + default = null +} + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(string) + default = null +} + +variable "certificate-service" { + description = "The certificate service to use" + type = string + default = "self-signed" +} + +variable "additional-arguments" { + description = "Additional command line arguments to supply to traefik ingress" + type = list(string) + default = [] +} + + + +variable "access_modes" { + type = list(string) + default = ["ReadWriteOnce"] +} +variable "pvc_name"{ + type = string + default = "persistent-volume-claim" +} + +variable "pv_name"{ + type = string + default = "traefik-persistent-volume" +} +variable "path" { + type = string + default = "/tmp/acme-certificates" } \ No newline at end of file diff --git a/src/_nebari/stages/kubernetes_ingress/template/outputs.tf b/src/_nebari/stages/kubernetes_ingress/template/outputs.tf index 8de01ef97f..ab1b043874 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/outputs.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/outputs.tf @@ -1,5 +1,5 @@ -output "load_balancer_address" { - description = "traefik load balancer address" - value = module.kubernetes-ingress.endpoint -} - +output "load_balancer_address" { + description = "traefik load balancer address" + value = module.kubernetes-ingress.endpoint +} + diff --git a/src/_nebari/stages/kubernetes_ingress/template/variables.tf b/src/_nebari/stages/kubernetes_ingress/template/variables.tf index 3de1742f7a..9d605debec 100644 --- a/src/_nebari/stages/kubernetes_ingress/template/variables.tf +++ b/src/_nebari/stages/kubernetes_ingress/template/variables.tf @@ -1,95 +1,95 @@ -variable "name" { - description = "Prefix name to assign to ingress kubernetes resources" - type = string -} - -variable "environment" { - description = "Kubernetes namespace to deploy ingress resources" - type = string -} - -variable "node_groups" { - description = "Node group selectors for kubernetes resources" - type = map(object({ - key = string - value = string - })) -} - -variable "traefik-image" { - description = "traefik image to use" - type = object({ - image = string - tag = string - }) -} - -variable "acme-email" { - description = "ACME server email" - default = "nebari@example.com" -} - -variable "acme-server" { - description = "ACME server" - # for testing use the letencrypt staging server - # - staging: https://acme-staging-v02.api.letsencrypt.org/directory - # - production: https://acme-v02.api.letsencrypt.org/directory - default = "https://acme-staging-v02.api.letsencrypt.org/directory" -} - -variable "certificate-secret-name" { - description = "Kubernetes secret used for certificate" - default = "" -} - - -variable "load-balancer-ip" { - description = "IP Address of the load balancer" - type = string - default = null -} - - -variable "load-balancer-annotations" { - description = "Annotations for the load balancer" - type = map(string) - default = null -} - - -variable "certificate-service" { - description = "The certificate service to use" - type = string - default = "self-signed" -} - - -variable "additional-arguments" { - description = "Additional command line arguments to supply to traefik ingress" - type = list(string) - default = [] -} - - -variable "storage_size" { - type = string - default = "1Gi" -} - -variable "access_modes" { - type = list(string) - default = ["ReadWriteOnce"] -} -variable "pvc_name"{ - type = string - default = "persistent-volume-claim" -} - -variable "pv_name"{ - type = string - default = "traefik-persistent-volume" -} -variable "path" { - type = string - default = "/tmp/acme-certificates" +variable "name" { + description = "Prefix name to assign to ingress kubernetes resources" + type = string +} + +variable "environment" { + description = "Kubernetes namespace to deploy ingress resources" + type = string +} + +variable "node_groups" { + description = "Node group selectors for kubernetes resources" + type = map(object({ + key = string + value = string + })) +} + +variable "traefik-image" { + description = "traefik image to use" + type = object({ + image = string + tag = string + }) +} + +variable "acme-email" { + description = "ACME server email" + default = "nebari@example.com" +} + +variable "acme-server" { + description = "ACME server" + # for testing use the letencrypt staging server + # - staging: https://acme-staging-v02.api.letsencrypt.org/directory + # - production: https://acme-v02.api.letsencrypt.org/directory + default = "https://acme-staging-v02.api.letsencrypt.org/directory" +} + +variable "certificate-secret-name" { + description = "Kubernetes secret used for certificate" + default = "" +} + + +variable "load-balancer-ip" { + description = "IP Address of the load balancer" + type = string + default = null +} + + +variable "load-balancer-annotations" { + description = "Annotations for the load balancer" + type = map(string) + default = null +} + + +variable "certificate-service" { + description = "The certificate service to use" + type = string + default = "self-signed" +} + + +variable "additional-arguments" { + description = "Additional command line arguments to supply to traefik ingress" + type = list(string) + default = [] +} + + +variable "storage_size" { + type = string + default = "1Gi" +} + +variable "access_modes" { + type = list(string) + default = ["ReadWriteOnce"] +} +variable "pvc_name"{ + type = string + default = "persistent-volume-claim" +} + +variable "pv_name"{ + type = string + default = "traefik-persistent-volume" +} +variable "path" { + type = string + default = "/tmp/acme-certificates" } \ No newline at end of file