From 5b9643ae06381603f392339b70a2c0e8a20b9a6c Mon Sep 17 00:00:00 2001 From: Marius Sili Date: Wed, 12 Jun 2024 09:51:12 +0200 Subject: [PATCH] feat: HPA (#376) --- .pnp.loader.mjs | 72 ++++----- README.md | 22 +++ deployment/base/autoscaling.yaml | 19 +++ deployment/base/deployment.yaml | 4 + deployment/base/kustomization.yaml | 2 +- deployment/local/deployment.yaml | 3 +- deployment/local/kustomization.yaml | 1 + deployment/local/postgres.yaml | 57 ++++++++ deployment/staging/kustomization.yaml | 1 + deployment/{base => staging}/postgres.yaml | 0 .../adapter.apiservice.yaml | 14 ++ .../prometheusAdapter-apiService.yaml | 18 +++ .../prometheusAdapter-clusterRole.yaml | 21 +++ ...er-clusterRoleAggregatedMetricsReader.yaml | 22 +++ .../prometheusAdapter-clusterRoleBinding.yaml | 17 +++ ...usAdapter-clusterRoleBindingDelegator.yaml | 17 +++ ...eusAdapter-clusterRoleServerResources.yaml | 16 ++ .../prometheusAdapter-configMap.yaml | 81 +++++++++++ .../prometheusAdapter-deployment.yaml | 103 +++++++++++++ .../prometheusAdapter-networkPolicy.yaml | 23 +++ ...prometheusAdapter-podDisruptionBudget.yaml | 17 +++ ...ometheusAdapter-roleBindingAuthReader.yaml | 18 +++ .../prometheusAdapter-service.yaml | 19 +++ .../prometheusAdapter-serviceAccount.yaml | 11 ++ .../templates/prometheus/clusterRole.yaml | 33 +++++ .../templates/prometheus/config-map.yaml | 137 ++++++++++++++++++ .../prometheus/prometheus-deployment.yaml | 38 +++++ .../prometheus/prometheus-service.yaml | 27 ++++ src/server/api/prod-server.ts | 29 +++- 29 files changed, 803 insertions(+), 39 deletions(-) create mode 100644 deployment/base/autoscaling.yaml create mode 100644 deployment/local/postgres.yaml rename deployment/{base => staging}/postgres.yaml (100%) create mode 100644 deployment/templates/prometheus-adapter/adapter.apiservice.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-apiService.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-clusterRole.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBinding.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBindingDelegator.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleServerResources.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-configMap.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-deployment.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-networkPolicy.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-podDisruptionBudget.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-roleBindingAuthReader.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-service.yaml create mode 100644 deployment/templates/prometheus-adapter/prometheusAdapter-serviceAccount.yaml create mode 100644 deployment/templates/prometheus/clusterRole.yaml create mode 100644 deployment/templates/prometheus/config-map.yaml create mode 100644 deployment/templates/prometheus/prometheus-deployment.yaml create mode 100644 deployment/templates/prometheus/prometheus-service.yaml diff --git a/.pnp.loader.mjs b/.pnp.loader.mjs index cb7b75ba..4b7f9f84 100644 --- a/.pnp.loader.mjs +++ b/.pnp.loader.mjs @@ -1,9 +1,9 @@ -import { fileURLToPath, pathToFileURL, URL as URL$1 } from "url"; -import fs from "fs"; -import path from "path"; -import moduleExports, { Module } from "module"; -import { EOL } from "os"; -import assert from "assert"; +import { URL as URL$1, fileURLToPath, pathToFileURL } from 'url'; +import fs from 'fs'; +import path from 'path'; +import moduleExports, { Module } from 'module'; +import { EOL } from 'os'; +import assert from 'assert'; const SAFE_TIME = 456789e3; @@ -95,24 +95,24 @@ async function copyImpl(prelayout, postlayout, updateTime, destinationFs, destin let updated; switch (true) { case sourceStat.isDirectory(): - { - updated = await copyFolder(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); - } + { + updated = await copyFolder(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); + } break; case sourceStat.isFile(): - { - updated = await copyFile(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); - } + { + updated = await copyFile(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); + } break; case sourceStat.isSymbolicLink(): - { - updated = await copySymlink(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); - } + { + updated = await copySymlink(prelayout, postlayout, updateTime, destinationFs, destination, destinationStat, sourceFs, source, sourceStat, opts); + } break; default: - { - throw new Error(`Unsupported file type (${sourceStat.mode})`); - } + { + throw new Error(`Unsupported file type (${sourceStat.mode})`); + } } if (updated || ((_a = destinationStat == null ? void 0 : destinationStat.mtime) == null ? void 0 : _a.getTime()) !== mtime.getTime() || ((_b = destinationStat == null ? void 0 : destinationStat.atime) == null ? void 0 : _b.getTime()) !== atime.getTime()) { postlayout.push(() => updateTime(destination, atime, mtime)); @@ -1646,27 +1646,27 @@ function getPackageScopeConfig(resolved, readFileSyncFn) { } /** - @license - Copyright Node.js contributors. All rights reserved. + @license + Copyright Node.js contributors. All rights reserved. - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to - deal in the Software without restriction, including without limitation the - rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to + deal in the Software without restriction, including without limitation the + rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + sell copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - IN THE SOFTWARE. - */ + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + IN THE SOFTWARE. +*/ function throwImportNotDefined(specifier, packageJSONUrl, base) { throw new ERR_PACKAGE_IMPORT_NOT_DEFINED( specifier, diff --git a/README.md b/README.md index ebafedab..e8902d91 100644 --- a/README.md +++ b/README.md @@ -128,3 +128,25 @@ yarn aleo:deploy-leaderboard > It's normal for the script to take a while to complete. If you wish to re-deploy the leaderboard you will have to update its name in the `.env` before re-running the deploy script. Aleo uses program names as identifiers, thus 2 programs with the same name cannot co-exist on the same network. + +### Deploy HPA + +Create `monitoring` namespace: + +```sh +kubectl create namespace monitoring +``` + +Deploy Prometheus: + +```sh +kubectl apply -f deployment/templates/prometheus/ +``` + +Deploy Prometheus adapter: + +```sh +kubectl apply -f deployment/templates/prometheus-adapter/ +``` + +You may need to restart the Prometheus adapter pod. diff --git a/deployment/base/autoscaling.yaml b/deployment/base/autoscaling.yaml new file mode 100644 index 00000000..e0d61c3d --- /dev/null +++ b/deployment/base/autoscaling.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: bot-busters +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: bot-busters + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Pods + pods: + metric: + name: active_connections_total + target: + type: AverageValue + averageValue: "4000" diff --git a/deployment/base/deployment.yaml b/deployment/base/deployment.yaml index dc4928fe..61c9e03b 100644 --- a/deployment/base/deployment.yaml +++ b/deployment/base/deployment.yaml @@ -10,6 +10,10 @@ spec: metadata: labels: app: bot-busters + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "3000" spec: containers: - name: bot-busters diff --git a/deployment/base/kustomization.yaml b/deployment/base/kustomization.yaml index 22633fdf..1ddc7ecd 100644 --- a/deployment/base/kustomization.yaml +++ b/deployment/base/kustomization.yaml @@ -1,7 +1,7 @@ resources: - deployment.yaml - - postgres.yaml - service.yaml - postgres-service.yaml + - autoscaling.yaml generatorOptions: disableNameSuffixHash: true diff --git a/deployment/local/deployment.yaml b/deployment/local/deployment.yaml index fb9bb51d..11786437 100644 --- a/deployment/local/deployment.yaml +++ b/deployment/local/deployment.yaml @@ -38,9 +38,10 @@ spec: value: ws://bot-busters.localhost - name: PLAYERS_PER_MATCH value: "2" - - name: ALEO_NETWORK_URL + - name: ALEO_NETWORK_URL # TODO: update to either minikube.internal or obscura URL value: http://localhost:3030 - name: ALEO_PRIVATE_KEY value: APrivateKey1zkp8CZNn3yeCseEtxuVPbDCwSyhGW6yZKUYKfgXmcpoGPWH # not a real account - name: AWS_INFERENCE_URL value: https://nez5g29b86.execute-api.eu-central-1.amazonaws.com/staging/generate +# missing secrets: AWS_ACCESS_KEY, AWS_SECRET, LAMBDA_TOKEN, OBSCURA_API_KEY (not mandatory for now) diff --git a/deployment/local/kustomization.yaml b/deployment/local/kustomization.yaml index 8839617f..fdfa9711 100644 --- a/deployment/local/kustomization.yaml +++ b/deployment/local/kustomization.yaml @@ -4,6 +4,7 @@ resources: - namespace.yaml - ingress.yaml - secrets.yaml + - postgres.yaml patchesStrategicMerge: - deployment.yaml namespace: bot-busters-local diff --git a/deployment/local/postgres.yaml b/deployment/local/postgres.yaml new file mode 100644 index 00000000..6842bd3a --- /dev/null +++ b/deployment/local/postgres.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: postgres + labels: + app: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + serviceName: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: ankane/pgvector:latest + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: backend-secrets + key: db_password + - name: PGDATA + value: /data/pgdata + volumeMounts: + - name: postgres-data + mountPath: /data + ports: + - containerPort: 5432 + name: http + protocol: TCP + + - name: adminer + image: adminer + ports: + - containerPort: 8080 + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeClaimTemplates: + - metadata: + name: postgres-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi diff --git a/deployment/staging/kustomization.yaml b/deployment/staging/kustomization.yaml index 8223533c..f23e7a06 100644 --- a/deployment/staging/kustomization.yaml +++ b/deployment/staging/kustomization.yaml @@ -4,6 +4,7 @@ resources: - namespace.yaml - ingress.yaml - secrets.yaml + - postgres.yaml patchesStrategicMerge: - deployment.yaml namespace: bot-busters-staging diff --git a/deployment/base/postgres.yaml b/deployment/staging/postgres.yaml similarity index 100% rename from deployment/base/postgres.yaml rename to deployment/staging/postgres.yaml diff --git a/deployment/templates/prometheus-adapter/adapter.apiservice.yaml b/deployment/templates/prometheus-adapter/adapter.apiservice.yaml new file mode 100644 index 00000000..e6530644 --- /dev/null +++ b/deployment/templates/prometheus-adapter/adapter.apiservice.yaml @@ -0,0 +1,14 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: prometheus-adapter + namespace: monitoring + port: 443 + version: v1beta1 + versionPriority: 100 diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-apiService.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-apiService.yaml new file mode 100644 index 00000000..4618e9c1 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-apiService.yaml @@ -0,0 +1,18 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: v1beta1.metrics.k8s.io +spec: + group: metrics.k8s.io + groupPriorityMinimum: 100 + insecureSkipTLSVerify: true + service: + name: prometheus-adapter + namespace: monitoring + version: v1beta1 + versionPriority: 100 diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRole.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRole.yaml new file mode 100644 index 00000000..d1af3163 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRole.yaml @@ -0,0 +1,21 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter +rules: +- apiGroups: + - "" + resources: + - nodes + - namespaces + - pods + - services + verbs: + - get + - list + - watch diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml new file mode 100644 index 00000000..672c8c43 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleAggregatedMetricsReader.yaml @@ -0,0 +1,22 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + rbac.authorization.k8s.io/aggregate-to-admin: "true" + rbac.authorization.k8s.io/aggregate-to-edit: "true" + rbac.authorization.k8s.io/aggregate-to-view: "true" + name: system:aggregated-metrics-reader +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBinding.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBinding.yaml new file mode 100644 index 00000000..1c6cb300 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-adapter +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBindingDelegator.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBindingDelegator.yaml new file mode 100644 index 00000000..63f7ccc7 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleBindingDelegator.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: resource-metrics:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleServerResources.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleServerResources.yaml new file mode 100644 index 00000000..22eba1d0 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-clusterRoleServerResources.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: resource-metrics-server-resources +rules: +- apiGroups: + - metrics.k8s.io + resources: + - '*' + verbs: + - '*' diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-configMap.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-configMap.yaml new file mode 100644 index 00000000..8979d8dc --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-configMap.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +data: + config.yaml: |- + rules: + - seriesQuery: 'active_connections_total' + resources: + overrides: + kubernetes_namespace: + resource: namespace + kubernetes_pod_name: + resource: pod + name: + matches: "^(.*)_total" + as: "${1}_total" + metricsQuery: <<.Series>> + "resourceRules": + "cpu": + "containerLabel": "container" + "containerQuery": | + sum by (<<.GroupBy>>) ( + irate ( + container_cpu_usage_seconds_total{<<.LabelMatchers>>,container!="",pod!=""}[120s] + ) + ) + "nodeQuery": | + sum by (<<.GroupBy>>) ( + 1 - irate( + node_cpu_seconds_total{mode="idle"}[60s] + ) + * on(namespace, pod) group_left(node) ( + node_namespace_pod:kube_pod_info:{<<.LabelMatchers>>} + ) + ) + or sum by (<<.GroupBy>>) ( + 1 - irate( + windows_cpu_time_total{mode="idle", job="windows-exporter",<<.LabelMatchers>>}[4m] + ) + ) + "resources": + "overrides": + "namespace": + "resource": "namespace" + "node": + "resource": "node" + "pod": + "resource": "pod" + "memory": + "containerLabel": "container" + "containerQuery": | + sum by (<<.GroupBy>>) ( + container_memory_working_set_bytes{<<.LabelMatchers>>,container!="",pod!=""} + ) + "nodeQuery": | + sum by (<<.GroupBy>>) ( + node_memory_MemTotal_bytes{job="node-exporter",<<.LabelMatchers>>} + - + node_memory_MemAvailable_bytes{job="node-exporter",<<.LabelMatchers>>} + ) + or sum by (<<.GroupBy>>) ( + windows_cs_physical_memory_bytes{job="windows-exporter",<<.LabelMatchers>>} + - + windows_memory_available_bytes{job="windows-exporter",<<.LabelMatchers>>} + ) + "resources": + "overrides": + "instance": + "resource": "node" + "namespace": + "resource": "namespace" + "pod": + "resource": "pod" + "window": "5m" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: adapter-config + namespace: monitoring diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-deployment.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-deployment.yaml new file mode 100644 index 00000000..8b3b97dd --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-deployment.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + template: + metadata: + annotations: + checksum.config/md5: 3b1ebf7df0232d1675896f67b66373db + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + spec: + automountServiceAccountToken: true + containers: + - args: + - --cert-dir=/var/run/serving-cert + - --config=/etc/adapter/config.yaml + - --metrics-relist-interval=1m + - --prometheus-url=http://prometheus.monitoring.svc.cluster.local:9090/ + - --secure-port=6443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA + image: registry.k8s.io/prometheus-adapter/prometheus-adapter:v0.12.0 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 5 + name: prometheus-adapter + ports: + - containerPort: 6443 + name: https + readinessProbe: + failureThreshold: 5 + httpGet: + path: /readyz + port: https + scheme: HTTPS + periodSeconds: 5 + resources: + limits: + cpu: 250m + memory: 180Mi + requests: + cpu: 102m + memory: 180Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 18 + httpGet: + path: /livez + port: https + scheme: HTTPS + periodSeconds: 10 + volumeMounts: + - mountPath: /tmp + name: tmpfs + readOnly: false + - mountPath: /var/run/serving-cert + name: volume-serving-cert + readOnly: false + - mountPath: /etc/adapter + name: config + readOnly: false + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: prometheus-adapter + volumes: + - emptyDir: {} + name: tmpfs + - emptyDir: {} + name: volume-serving-cert + - configMap: + name: adapter-config + name: config diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-networkPolicy.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-networkPolicy.yaml new file mode 100644 index 00000000..df8c6644 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-networkPolicy.yaml @@ -0,0 +1,23 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter + namespace: monitoring +spec: + egress: + - {} + ingress: + - {} + podSelector: + matchLabels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + policyTypes: + - Egress + - Ingress diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-podDisruptionBudget.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-podDisruptionBudget.yaml new file mode 100644 index 00000000..09348f7e --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-podDisruptionBudget.yaml @@ -0,0 +1,17 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter + namespace: monitoring +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-roleBindingAuthReader.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-roleBindingAuthReader.yaml new file mode 100644 index 00000000..c21491a5 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-roleBindingAuthReader.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: resource-metrics-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: prometheus-adapter + namespace: monitoring diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-service.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-service.yaml new file mode 100644 index 00000000..033f93eb --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter + namespace: monitoring +spec: + ports: + - name: https + port: 443 + targetPort: 6443 + selector: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus diff --git a/deployment/templates/prometheus-adapter/prometheusAdapter-serviceAccount.yaml b/deployment/templates/prometheus-adapter/prometheusAdapter-serviceAccount.yaml new file mode 100644 index 00000000..04ab1787 --- /dev/null +++ b/deployment/templates/prometheus-adapter/prometheusAdapter-serviceAccount.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: metrics-adapter + app.kubernetes.io/name: prometheus-adapter + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 0.12.0 + name: prometheus-adapter + namespace: monitoring diff --git a/deployment/templates/prometheus/clusterRole.yaml b/deployment/templates/prometheus/clusterRole.yaml new file mode 100644 index 00000000..8d106ea4 --- /dev/null +++ b/deployment/templates/prometheus/clusterRole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: + - apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] + - apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] + - nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: + - kind: ServiceAccount + name: default + namespace: monitoring diff --git a/deployment/templates/prometheus/config-map.yaml b/deployment/templates/prometheus/config-map.yaml new file mode 100644 index 00000000..a7210fe0 --- /dev/null +++ b/deployment/templates/prometheus/config-map.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus + labels: + name: prometheus + namespace: monitoring +data: + prometheus.rules: |- + groups: + - name: devopscube demo alert + rules: + - alert: High Pod Memory + expr: sum(container_memory_usage_bytes) > 1 + for: 1m + labels: + severity: slack + annotations: + summary: High Memory Usage + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" + scrape_configs: + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_endpoints_name] + regex: 'node-exporter' + action: keep + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name diff --git a/deployment/templates/prometheus/prometheus-deployment.yaml b/deployment/templates/prometheus/prometheus-deployment.yaml new file mode 100644 index 00000000..2a49247a --- /dev/null +++ b/deployment/templates/prometheus/prometheus-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus + namespace: monitoring + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus + + - name: prometheus-storage-volume + emptyDir: {} diff --git a/deployment/templates/prometheus/prometheus-service.yaml b/deployment/templates/prometheus/prometheus-service.yaml new file mode 100644 index 00000000..1dbef1e2 --- /dev/null +++ b/deployment/templates/prometheus/prometheus-service.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: prometheus + app.kubernetes.io/instance: k8s + app.kubernetes.io/name: prometheus + app.kubernetes.io/part-of: kube-prometheus + app.kubernetes.io/version: 2.52.0 + name: prometheus + namespace: monitoring + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" +spec: + type: ClusterIP + selector: + app: prometheus + ports: + - port: 9090 + targetPort: 9090 + protocol: TCP + name: web + - port: 8080 + targetPort: 8080 + protocol: TCP + name: reloader-web diff --git a/src/server/api/prod-server.ts b/src/server/api/prod-server.ts index d3058b2e..7be4a09e 100644 --- a/src/server/api/prod-server.ts +++ b/src/server/api/prod-server.ts @@ -13,10 +13,35 @@ const port = 3000; const app = next({ dev: env.NODE_ENV !== "production" }); const handle = app.getRequestHandler(); +const metrics = { + activeConnections: 0, + + incrActiveConnections() { + this.activeConnections++; + }, + + decrActiveConnections() { + if (this.activeConnections > 0) { + this.activeConnections--; + } else { + this.activeConnections = 0; + } + }, +}; + void app.prepare().then(() => { const server = http.createServer((req, res) => { + if (req.url === "/metrics") { + res.end( + "# HELP active_connections_total The amount of total active ws connections\n# TYPE active_connections_total counter\nactive_connections_total " + + metrics.activeConnections + + "\n", + ); + return; + } + const parsedUrl = parse(req.url!, true); - void handle(req, res, parsedUrl).catch((error) => { + handle(req, res, parsedUrl).catch((error) => { console.error("Request handling error:", error); }); }); @@ -30,6 +55,7 @@ void app.prepare().then(() => { wss.on("connection", (ws) => { console.log(`++ Connection (${wss.clients.size})`); + metrics.incrActiveConnections(); ws.on("error", (error) => { console.error("ws error: ", error); @@ -37,6 +63,7 @@ void app.prepare().then(() => { ws.once("close", () => { console.log(`-- Connection (${wss.clients.size})`); + metrics.decrActiveConnections(); }); });