From c9a426c666ea23d6a18b2727a573b5a0d65d8d8f Mon Sep 17 00:00:00 2001 From: Peter Vlugter <59895+pvlugter@users.noreply.github.com> Date: Tue, 10 Oct 2023 18:20:08 +1300 Subject: [PATCH] sample: multidimensional autoscaling for local-drone-control-java --- .../autoscaling/README.md | 135 ++++++++++++++++++ .../autoscaling/kubernetes/deployment.yaml | 107 ++++++++++++++ .../autoscaling/kubernetes/hpa.yaml | 19 +++ .../autoscaling/kubernetes/pdb.yaml | 9 ++ .../autoscaling/kubernetes/rbac.yaml | 20 +++ .../autoscaling/kubernetes/service.yaml | 13 ++ .../kubernetes/serviceaccount.yaml | 4 + .../kubernetes/servicemonitor.yaml | 16 +++ .../autoscaling/kubernetes/vpa.yaml | 26 ++++ .../autoscaling/local/autoscaler/.gitignore | 2 + .../autoscaling/local/autoscaler/Chart.yaml | 8 ++ .../autoscaling/local/autoscaler/values.yaml | 13 ++ .../autoscaling/local/down.sh | 42 ++++++ .../autoscaling/local/ingress/route.yaml | 14 ++ .../autoscaling/local/monitoring/.gitignore | 2 + .../autoscaling/local/monitoring/Chart.yaml | 11 ++ .../autoscaling/local/monitoring/values.yaml | 20 +++ .../autoscaling/local/persistence/.gitignore | 2 + .../autoscaling/local/persistence/Chart.yaml | 8 ++ .../postgresql-initdb-configmap.yaml | 89 ++++++++++++ .../autoscaling/local/persistence/values.yaml | 6 + .../autoscaling/local/up.sh | 123 ++++++++++++++++ .../autoscaling/simulator/pom.xml | 134 +++++++++++++++++ .../test/java/local/drones/Coordinates.java | 115 +++++++++++++++ .../src/test/java/local/drones/Load.java | 82 +++++++++++ .../src/test/proto/common/coordinates.proto | 15 ++ .../local/drones/deliveries_queue_api.proto | 31 ++++ .../test/proto/local/drones/drone_api.proto | 40 ++++++ .../src/test/resources/application.conf | 8 ++ .../simulator/src/test/resources/logback.xml | 15 ++ samples/grpc/local-drone-control-java/pom.xml | 13 ++ .../main/java/local/drones/ClusteredMain.java | 3 + .../src/main/java/local/drones/Drone.java | 19 ++- .../src/main/java/local/drones/Telemetry.java | 51 +++++++ .../main/resources/application-cluster.conf | 3 + .../src/main/resources/cluster.conf | 5 +- .../src/main/resources/local1.conf | 1 + .../src/main/resources/local2.conf | 4 +- .../src/main/resources/local3.conf | 2 + 39 files changed, 1226 insertions(+), 4 deletions(-) create mode 100644 samples/grpc/local-drone-control-java/autoscaling/README.md create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/deployment.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/hpa.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/pdb.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/rbac.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/service.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/serviceaccount.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/servicemonitor.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/kubernetes/vpa.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/.gitignore create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/Chart.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/values.yaml create mode 100755 samples/grpc/local-drone-control-java/autoscaling/local/down.sh create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/ingress/route.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/monitoring/.gitignore create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/monitoring/Chart.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/monitoring/values.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/persistence/.gitignore create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/persistence/Chart.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/persistence/templates/postgresql-initdb-configmap.yaml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/local/persistence/values.yaml create mode 100755 samples/grpc/local-drone-control-java/autoscaling/local/up.sh create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/pom.xml create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Coordinates.java create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Load.java create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/common/coordinates.proto create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/deliveries_queue_api.proto create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/drone_api.proto create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/application.conf create mode 100644 samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/logback.xml create mode 100644 samples/grpc/local-drone-control-java/src/main/java/local/drones/Telemetry.java diff --git a/samples/grpc/local-drone-control-java/autoscaling/README.md b/samples/grpc/local-drone-control-java/autoscaling/README.md new file mode 100644 index 000000000..506782a4c --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/README.md @@ -0,0 +1,135 @@ +# Autoscaling example + +This example demonstrates multidimensional autoscaling, to scale the Local Drone Control service to +and from "near zero" — scaling down to a state of minimal resource usage when idle, scaling up and +out when load is increased. + +The example uses GraalVM Native Image builds for low resource usage, combines the Kubernetes +vertical and horizontal pod autoscalers, and runs in a k3s cluster (lightweight Kubernetes). + + +## Requirements + +The following tools are required to run this example locally: + +- [docker](https://www.docker.com) - Docker engine for building and running containers +- [kubectl](https://kubernetes.io/docs/reference/kubectl) - Kubernetes command line tool +- [k3d](https://k3d.io) - k3s (lightweight Kubernetes) in Docker +- [helm](https://helm.sh) - package manager for Kubernetes + + +## Build local-drone-control Docker image + +First build a Docker image for the Local Drone Control service, as a native image and configured to +run as a multi-node Akka Cluster with PostgreSQL. From the `local-drone-control-java` directory: + +``` +docker build -f native-image/Dockerfile --build-arg profile=clustered -t local-drone-control . +``` + +See the native-image build for more information. + + +## Run the Central Drone Control service + +Run the Central Drone Control service. By default, the example assumes this is running locally, but +it can also be deployed. + +To run locally, from the `restaurant-drone-deliveries-service-java` directory: + +``` +docker compose up --wait + +docker exec -i postgres_db psql -U postgres -t < ddl-scripts/create_tables.sql + +mvn compile exec:exec -DAPP_CONFIG=local1.conf +``` + +Or see the documentation for deploying to Kubernetes in a cloud environment. + + +## Start the Local Drone Control service in k3s + +A convenience script starts a k3d cluster (k3s cluster in Docker), installs the infrastructure +dependencies for persistence, monitoring, and autoscaling, and then installs the Local Drone +Control service configured for multidimensional autoscaling. + +To start the Local Drone Control service in a local k3s cluster, run the `up.sh` script: + +``` +autoscaling/local/up.sh +``` + +If the Central Drone Control service has been deployed somewhere other than locally on +`localhost:8101`, the connection details can be specified using arguments to the script: + +``` +autoscaling/local/up.sh --central-host deployed.app --central-port 443 --central-tls true +``` + + +## Autoscaling infrastructure + +This example uses multidimensional autoscaling, combining the Kubernetes vertical and horizontal +pod autoscalers, so that when the service is idle it is both _scaled down_ with minimal resource +requests, and _scaled in_ to a minimal number of pods. The same metrics should not be used for both +the vertical and horizontal autoscalers, so the horizontal pod autoscaler is configured to use a +custom metric — the number of active drones. When activity for the service increases, the vertical +pod autoscaler (VPA) will increase the resource requests, and when the number of active drones +increases, the horizontal pod autoscaler (HPA) will increase the number of pods in the deployment. + +The default vertical pod autoscaler recommends new resource requests and limits over long time +frames. In this example, a custom VPA recommender has been configured for short cycles and metric +history, to scale up quickly. The horizontal scaling has been configured for minimum 2 replicas, to +ensure availability of the service (when pods are recreated on vertical scaling), and a pod +disruption budget has been configured to ensure that no more than one pod is unavailable at a time. + +You can see the current state and recommendations for the autoscalers by running: + +``` +kubectl get hpa,vpa +``` + + +## Simulate drone activity + +A simple load simulator is available, to demonstrate autoscaling behavior given increasing load. + +This simulator moves drones on random delivery paths, frequently reporting updated locations. + +In the `autoscaling/simulator` directory, run the Gatling load test: + +``` +mvn gatling:test +``` + +You can see the current resource usage for pods by running: + +``` +kubectl top pods +``` + +And the current state of the autoscalers and deployed pods with: + +``` +kubectl get hpa,vpa,deployments,pods +``` + +The vertical pod autoscaler will increase the resource requests for pods as needed. The current CPU +requests for pods can be seen by running: + +``` +kubectl get pods -o custom-columns='NAME:metadata.name,CPU:spec.containers[].resources.requests.cpu' +``` + +When the simulated load has finished, and idle entities have been passivated, the autoscalers will +eventually scale the service back down. + + +## Stop the Local Drone Control service + +To stop and delete the Local Drone Control service and k3s cluster, run the `down.sh` script: + +``` +autoscaling/local/down.sh +``` diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/deployment.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/deployment.yaml new file mode 100644 index 000000000..af060b440 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/deployment.yaml @@ -0,0 +1,107 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-drone-control + labels: + app: local-drone-control +spec: + replicas: 2 + selector: + matchLabels: + app: local-drone-control + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + app: local-drone-control + spec: + serviceAccountName: local-drone-control + containers: + - name: local-drone-control + image: local-drone-control:latest + imagePullPolicy: Never + resources: + requests: + cpu: 100m + memory: 256Mi + livenessProbe: + httpGet: + path: /alive + port: management + readinessProbe: + httpGet: + path: /ready + port: management + args: + - "-Dconfig.resource=application-cluster.conf" + env: + - name: LOCATION_ID + # one of the location ids supported by the restaurant-drone-deliveries service + value: "sweden/stockholm/kungsholmen" + - name: GRPC_PORT + value: "8080" + - name: REMOTE_PORT + value: "2552" + - name: HTTP_MGMT_PORT + value: "8558" + - name: PROMETHEUS_PORT + value: "9090" + - name: REQUIRED_CONTACT_POINT_NR + value: "1" + - name: CENTRAL_DRONE_CONTROL_HOST + valueFrom: + secretKeyRef: + name: central-drone-control + key: host + - name: CENTRAL_DRONE_CONTROL_PORT + valueFrom: + secretKeyRef: + name: central-drone-control + key: port + - name: CENTRAL_DRONE_CONTROL_TLS + valueFrom: + secretKeyRef: + name: central-drone-control + key: tls + - name: DB_HOST + valueFrom: + secretKeyRef: + name: database-credentials + key: host + - name: DB_PORT + valueFrom: + secretKeyRef: + name: database-credentials + key: port + - name: DB_DATABASE + valueFrom: + secretKeyRef: + name: database-credentials + key: database + - name: DB_USER + valueFrom: + secretKeyRef: + name: database-credentials + key: user + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: database-credentials + key: password + ports: + - name: grpc + containerPort: 8080 + protocol: TCP + - name: remote + containerPort: 2552 + protocol: TCP + - name: management + containerPort: 8558 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/hpa.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/hpa.yaml new file mode 100644 index 000000000..39d75d465 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/hpa.yaml @@ -0,0 +1,19 @@ +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: local-drone-control +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: local-drone-control + minReplicas: 2 + maxReplicas: 5 + metrics: + - type: Pods + pods: + metric: + name: local_drone_control_active_entities + target: + type: Value + averageValue: 100 diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/pdb.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/pdb.yaml new file mode 100644 index 000000000..7a5d22f90 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/pdb.yaml @@ -0,0 +1,9 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: local-drone-control +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: local-drone-control diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/rbac.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/rbac.yaml new file mode 100644 index 000000000..2f319ab36 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/rbac.yaml @@ -0,0 +1,20 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: pod-reader +rules: +- apiGroups: [""] # "" indicates the core API group + resources: ["pods"] + verbs: ["get", "watch", "list"] +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: read-pods +subjects: +- kind: ServiceAccount + name: local-drone-control +roleRef: + kind: Role + name: pod-reader + apiGroup: rbac.authorization.k8s.io diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/service.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/service.yaml new file mode 100644 index 000000000..e246b47ce --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: local-drone-control + labels: + app: local-drone-control +spec: + type: ClusterIP + ports: + - port: 8080 + targetPort: 8080 + selector: + app: local-drone-control diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/serviceaccount.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/serviceaccount.yaml new file mode 100644 index 000000000..5a1a6bd9e --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-drone-control diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/servicemonitor.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/servicemonitor.yaml new file mode 100644 index 000000000..2c3b50742 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/servicemonitor.yaml @@ -0,0 +1,16 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: local-drone-control + labels: + release: local +spec: + endpoints: + - interval: 10s + targetPort: metrics + namespaceSelector: + matchNames: + - default + selector: + matchLabels: + app: local-drone-control diff --git a/samples/grpc/local-drone-control-java/autoscaling/kubernetes/vpa.yaml b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/vpa.yaml new file mode 100644 index 000000000..c2550c268 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/kubernetes/vpa.yaml @@ -0,0 +1,26 @@ +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: local-drone-control +spec: + recommenders: + - name: custom + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: local-drone-control + updatePolicy: + updateMode: "Auto" + minReplicas: 2 + resourcePolicy: + containerPolicies: + - containerName: local-drone-control + mode: "Auto" + minAllowed: + cpu: 100m + memory: 256Mi + maxAllowed: + cpu: 1000m + memory: 1024Mi + controlledResources: ["cpu", "memory"] + controlledValues: RequestsAndLimits diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/.gitignore b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/.gitignore new file mode 100644 index 000000000..9169e44a3 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/.gitignore @@ -0,0 +1,2 @@ +charts/*.tgz +Chart.lock diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/Chart.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/Chart.yaml new file mode 100644 index 000000000..26e57a569 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: autoscaler +description: Vertical pod autoscaler for drones in local k3s +version: 0.1.0 +dependencies: +- name: vertical-pod-autoscaler + version: "~9.3.0" + repository: "https://cowboysysop.github.io/charts" diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/values.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/values.yaml new file mode 100644 index 000000000..d8fc00d0f --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/autoscaler/values.yaml @@ -0,0 +1,13 @@ +vertical-pod-autoscaler: + recommender: + extraArgs: + recommender-name: custom + recommender-interval: 10s + cpu-histogram-decay-half-life: 30s + storage: prometheus + prometheus-address: "http://local-monitoring-prometheus.monitoring:9090" + v: 4 + updater: + extraArgs: + updater-interval: 10s + v: 4 diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/down.sh b/samples/grpc/local-drone-control-java/autoscaling/local/down.sh new file mode 100755 index 000000000..c93fa9d15 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/down.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# logs and failures + +function red { + echo -en "\033[0;31m$@\033[0m" +} + +function blue { + echo -en "\033[0;34m$@\033[0m" +} + +function info { + echo + echo $(blue "$@") + echo +} + +function error { + echo $(red "$@") 1>&2 +} + +function fail { + error "$@" + exit 1 +} + +# requirements + +function command_exists { + type -P "$1" > /dev/null 2>&1 +} + +command_exists "k3d" || fail "k3d is required (https://k3d.io)" + +# destroy k3s cluster + +info "Deleting k3s cluster ..." + +k3d cluster delete edge diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/ingress/route.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/ingress/route.yaml new file mode 100644 index 000000000..21a30ed3c --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/ingress/route.yaml @@ -0,0 +1,14 @@ +apiVersion: traefik.containo.us/v1alpha1 +kind: IngressRoute +metadata: + name: local-drone-control +spec: + entryPoints: + - web + routes: + - match: PathPrefix(`/`) + kind: Rule + services: + - name: local-drone-control + port: 8080 + scheme: h2c diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/.gitignore b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/.gitignore new file mode 100644 index 000000000..9169e44a3 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/.gitignore @@ -0,0 +1,2 @@ +charts/*.tgz +Chart.lock diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/Chart.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/Chart.yaml new file mode 100644 index 000000000..c865eb82e --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: monitoring +description: Prometheus monitoring for drones in local k3s +version: 0.1.0 +dependencies: +- name: kube-prometheus-stack + version: "~51.2.0" + repository: "https://prometheus-community.github.io/helm-charts" +- name: prometheus-adapter + version: "~4.5.0" + repository: "https://prometheus-community.github.io/helm-charts" diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/values.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/values.yaml new file mode 100644 index 000000000..33d35320d --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/monitoring/values.yaml @@ -0,0 +1,20 @@ +kube-prometheus-stack: + nameOverride: 'monitoring' + alertmanager: + enabled: false + grafana: + enabled: false + +prometheus-adapter: + prometheus: + url: http://{{ .Release.Name }}-monitoring-prometheus.{{ .Release.Namespace }}.svc + rules: + default: false + custom: + - seriesQuery: '{__name__=~"^local_drone_control_.*"}' + resources: + overrides: + pod: { resource: "pod" } + namespace: { resource: "namespace" } + metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)' + diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/persistence/.gitignore b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/.gitignore new file mode 100644 index 000000000..9169e44a3 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/.gitignore @@ -0,0 +1,2 @@ +charts/*.tgz +Chart.lock diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/persistence/Chart.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/Chart.yaml new file mode 100644 index 000000000..32ed19f4a --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/Chart.yaml @@ -0,0 +1,8 @@ +apiVersion: v2 +name: persistence +description: Postgres persistence for drones in local k3s +version: 0.1.0 +dependencies: +- name: postgresql + version: "~12.12.10" + repository: "oci://registry-1.docker.io/bitnamicharts" diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/persistence/templates/postgresql-initdb-configmap.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/templates/postgresql-initdb-configmap.yaml new file mode 100644 index 000000000..cb7a84456 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/templates/postgresql-initdb-configmap.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: postgresql-initdb +data: + create_tables.sql: | + CREATE TABLE IF NOT EXISTS event_journal( + slice INT NOT NULL, + entity_type VARCHAR(255) NOT NULL, + persistence_id VARCHAR(255) NOT NULL, + seq_nr BIGINT NOT NULL, + db_timestamp timestamp with time zone NOT NULL, + event_ser_id INTEGER NOT NULL, + event_ser_manifest VARCHAR(255) NOT NULL, + event_payload BYTEA NOT NULL, + deleted BOOLEAN DEFAULT FALSE NOT NULL, + writer VARCHAR(255) NOT NULL, + adapter_manifest VARCHAR(255), + tags TEXT ARRAY, + meta_ser_id INTEGER, + meta_ser_manifest VARCHAR(255), + meta_payload BYTEA, + PRIMARY KEY(persistence_id, seq_nr) + ); + + CREATE INDEX IF NOT EXISTS event_journal_slice_idx ON event_journal(slice, entity_type, db_timestamp, seq_nr); + + CREATE TABLE IF NOT EXISTS snapshot( + slice INT NOT NULL, + entity_type VARCHAR(255) NOT NULL, + persistence_id VARCHAR(255) NOT NULL, + seq_nr BIGINT NOT NULL, + db_timestamp timestamp with time zone, + write_timestamp BIGINT NOT NULL, + ser_id INTEGER NOT NULL, + ser_manifest VARCHAR(255) NOT NULL, + snapshot BYTEA NOT NULL, + tags TEXT ARRAY, + meta_ser_id INTEGER, + meta_ser_manifest VARCHAR(255), + meta_payload BYTEA, + PRIMARY KEY(persistence_id) + ); + + CREATE INDEX IF NOT EXISTS snapshot_slice_idx ON snapshot(slice, entity_type, db_timestamp); + + CREATE TABLE IF NOT EXISTS durable_state ( + slice INT NOT NULL, + entity_type VARCHAR(255) NOT NULL, + persistence_id VARCHAR(255) NOT NULL, + revision BIGINT NOT NULL, + db_timestamp timestamp with time zone NOT NULL, + state_ser_id INTEGER NOT NULL, + state_ser_manifest VARCHAR(255), + state_payload BYTEA NOT NULL, + tags TEXT ARRAY, + PRIMARY KEY(persistence_id, revision) + ); + + CREATE INDEX IF NOT EXISTS durable_state_slice_idx ON durable_state(slice, entity_type, db_timestamp, revision); + + CREATE TABLE IF NOT EXISTS akka_projection_offset_store ( + projection_name VARCHAR(255) NOT NULL, + projection_key VARCHAR(255) NOT NULL, + current_offset VARCHAR(255) NOT NULL, + manifest VARCHAR(32) NOT NULL, + mergeable BOOLEAN NOT NULL, + last_updated BIGINT NOT NULL, + PRIMARY KEY(projection_name, projection_key) + ); + + CREATE TABLE IF NOT EXISTS akka_projection_timestamp_offset_store ( + projection_name VARCHAR(255) NOT NULL, + projection_key VARCHAR(255) NOT NULL, + slice INT NOT NULL, + persistence_id VARCHAR(255) NOT NULL, + seq_nr BIGINT NOT NULL, + timestamp_offset timestamp with time zone NOT NULL, + timestamp_consumed timestamp with time zone NOT NULL, + PRIMARY KEY(slice, projection_name, timestamp_offset, persistence_id, seq_nr) + ); + + CREATE TABLE IF NOT EXISTS akka_projection_management ( + projection_name VARCHAR(255) NOT NULL, + projection_key VARCHAR(255) NOT NULL, + paused BOOLEAN NOT NULL, + last_updated BIGINT NOT NULL, + PRIMARY KEY(projection_name, projection_key) + ); diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/persistence/values.yaml b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/values.yaml new file mode 100644 index 000000000..19017b081 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/persistence/values.yaml @@ -0,0 +1,6 @@ +postgresql: + auth: + postgresPassword: "postgres" + primary: + initdb: + scriptsConfigMap: "postgresql-initdb" diff --git a/samples/grpc/local-drone-control-java/autoscaling/local/up.sh b/samples/grpc/local-drone-control-java/autoscaling/local/up.sh new file mode 100755 index 000000000..9de57d6c8 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/local/up.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# logs and failures + +function red { + echo -en "\033[0;31m$@\033[0m" +} + +function green { + echo -en "\033[0;32m$@\033[0m" +} + +function blue { + echo -en "\033[0;34m$@\033[0m" +} + +function info { + echo + echo $(blue "$@") + echo +} + +function success { + echo + echo $(green "$@") + echo +} + +function error { + echo $(red "$@") 1>&2 +} + +function fail { + error "$@" + exit 1 +} + +# requirements + +function command_exists { + type -P "$1" > /dev/null 2>&1 +} + +command_exists "docker" || fail "docker is required (https://www.docker.com)" +command_exists "kubectl" || fail "kubectl is required (https://kubernetes.io/docs/reference/kubectl)" +command_exists "k3d" || fail "k3d is required (https://k3d.io)" +command_exists "helm" || fail "helm is required (https://helm.sh)" + +# options + +declare local_image="local-drone-control:latest" +declare central_host="host.k3d.internal" +declare central_port="8101" +declare central_tls="false" + +while [[ $# -gt 0 ]] ; do + case "$1" in + --local-image ) local_image="$2" ; shift 2 ;; + --central-host ) central_host="$2" ; shift 2 ;; + --central-port ) central_port="$2" ; shift 2 ;; + --central-tls ) central_tls="$2" ; shift 2 ;; + * ) error "unknown option: $1" ; shift ;; + esac +done + +# image exists check + +[ -n "$(docker images -q "$local_image")" ] || fail "Docker image [$local_image] not found. Build locally before running." + +# directories + +readonly local=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +readonly autoscaling="$(cd "$local/.." && pwd)" + +# deploy to local k3s cluster + +info "Creating k3s cluster ..." + +# with port mapping for traefik ingress +k3d cluster create edge --port 8080:80@loadbalancer + +info "Installing postgresql persistence ..." + +helm dependency update "$local/persistence" +helm install local "$local/persistence" --create-namespace --namespace persistence --wait + +info "Installing prometheus monitoring stack ..." + +helm dependency update "$local/monitoring" +helm install local "$local/monitoring" --create-namespace --namespace monitoring --wait + +info "Installing vertical pod autoscaler ..." + +helm dependency update "$local/autoscaler" +helm install local "$local/autoscaler" --create-namespace --namespace kube-system --wait + +info "Deploying local-drone-control service ..." + +k3d image import --cluster edge "$local_image" + +kubectl create secret generic central-drone-control \ + --from-literal=host=$central_host \ + --from-literal=port=$central_port \ + --from-literal=tls=$central_tls + +kubectl create secret generic database-credentials \ + --from-literal=host=local-postgresql.persistence.svc \ + --from-literal=port=5432 \ + --from-literal=database=postgres \ + --from-literal=user=postgres \ + --from-literal=password=postgres + +kubectl apply -f "$autoscaling/kubernetes" +kubectl wait pods -l app=local-drone-control --for condition=Ready --timeout=120s +kubectl get pods + +info "Setting up ingress ..." + +kubectl apply -f "$local/ingress" + +success "Local Drone Control service running in k3s and available at localhost:8080" diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/pom.xml b/samples/grpc/local-drone-control-java/autoscaling/simulator/pom.xml new file mode 100644 index 000000000..a4d7a7339 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/pom.xml @@ -0,0 +1,134 @@ + + + + 4.0.0 + drone-simulator + com.lightbend.akka.samples + 1.0 + + + + Public Domain (CC0) + http://creativecommons.org/publicdomain/zero/1.0/ + + + + + UTF-8 + 3.9.5 + 0.16.0 + 0.15.1 + 4.6.0 + 3.22.2 + 0.6.1 + 1.58.0 + 1.7.1 + + + + + io.gatling.highcharts + gatling-charts-highcharts + ${gatling.version} + test + + + com.github.phisgr + gatling-grpc + ${gatling-grpc.version} + test + + + com.github.phisgr + gatling-grpc-kt + ${gatling-grpc-kt.version} + test + + + com.google.protobuf + protobuf-java + ${protobuf.version} + test + + + javax.annotation + javax.annotation-api + 1.3.2 + test + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.1 + + 11 + + + + + io.gatling + gatling-maven-plugin + ${gatling-maven-plugin.version} + + local.drones.Load + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java + io.grpc:protoc-gen-grpc-java:${grpc-java.version}:exe:${os.detected.classifier} + + + + + test-compile + test-compile-custom + + + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.35.0 + + + + 1.17.0 + + + + + + + format + process-test-sources + + apply + + + + + + + diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Coordinates.java b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Coordinates.java new file mode 100644 index 000000000..08a0f3d60 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Coordinates.java @@ -0,0 +1,115 @@ +package local.drones; + +import java.util.Iterator; +import java.util.concurrent.ThreadLocalRandom; + +public final class Coordinates { + + private static final int EARTH_RADIUS_METRES = 6371000; + + public final double latitude; + public final double longitude; + + public Coordinates(double latitude, double longitude) { + this.latitude = latitude; + this.longitude = longitude; + } + + public common.proto.Coordinates toProto() { + return common.proto.Coordinates.newBuilder() + .setLatitude(latitude) + .setLongitude(longitude) + .build(); + } + + // calculate distance between coordinates in metres + public static double distance(Coordinates start, Coordinates destination) { + return unitDistance(start, destination) * EARTH_RADIUS_METRES; + } + + // calculate unit distance between coordinates + public static double unitDistance(Coordinates start, Coordinates destination) { + double φ1 = Math.toRadians(start.latitude); + double λ1 = Math.toRadians(start.longitude); + double φ2 = Math.toRadians(destination.latitude); + double λ2 = Math.toRadians(destination.longitude); + + double Δφ = φ2 - φ1; + double Δλ = λ2 - λ1; + + double a = + Math.sin(Δφ / 2) * Math.sin(Δφ / 2) + + Math.cos(φ1) * Math.cos(φ2) * Math.sin(Δλ / 2) * Math.sin(Δλ / 2); + return 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + } + + // calculate destination coordinates given start coordinates, initial bearing, and distance + public static Coordinates destination( + Coordinates start, double initialBearing, double distanceMetres) { + double φ1 = Math.toRadians(start.latitude); + double λ1 = Math.toRadians(start.longitude); + double θ = Math.toRadians(initialBearing); + double δ = distanceMetres / EARTH_RADIUS_METRES; + + double φ2 = Math.asin(Math.sin(φ1) * Math.cos(δ) + Math.cos(φ1) * Math.sin(δ) * Math.cos(θ)); + double λ2 = + λ1 + + Math.atan2( + Math.sin(θ) * Math.sin(δ) * Math.cos(φ1), + Math.cos(δ) - Math.sin(φ1) * Math.sin(φ2)); + + return new Coordinates(Math.toDegrees(φ2), Math.toDegrees(λ2)); + } + + // calculate the intermediate coordinates on the path to a destination + // given the fraction of the distance travelled (fraction between 0 and 1) + public static Coordinates intermediate( + Coordinates start, Coordinates destination, double fraction) { + double φ1 = Math.toRadians(start.latitude); + double λ1 = Math.toRadians(start.longitude); + double φ2 = Math.toRadians(destination.latitude); + double λ2 = Math.toRadians(destination.longitude); + + double δ = unitDistance(start, destination); + + double A = Math.sin((1 - fraction) * δ) / Math.sin(δ); + double B = Math.sin(fraction * δ) / Math.sin(δ); + + double x = A * Math.cos(φ1) * Math.cos(λ1) + B * Math.cos(φ2) * Math.cos(λ2); + double y = A * Math.cos(φ1) * Math.sin(λ1) + B * Math.cos(φ2) * Math.sin(λ2); + double z = A * Math.sin(φ1) + B * Math.sin(φ2); + + double φ3 = Math.atan2(z, Math.sqrt(x * x + y * y)); + double λ3 = Math.atan2(y, x); + + return new Coordinates(Math.toDegrees(φ3), Math.toDegrees(λ3)); + } + + // iterate a path of intermediate coordinates between start and destination, every so many metres + public static Iterator path( + Coordinates start, Coordinates destination, double everyMetres) { + return new Iterator<>() { + private final double distance = Coordinates.distance(start, destination); + private final double step = everyMetres / distance; + private double fraction = 0.0; + + @Override + public boolean hasNext() { + return fraction < 1.0; + } + + @Override + public Coordinates next() { + fraction = fraction + step; + return (fraction >= 1.0) ? destination : intermediate(start, destination, fraction); + } + }; + } + + // select random coordinates within a circle defined by a centre and radius + public static Coordinates random(Coordinates centre, int radiusMetres) { + double bearing = ThreadLocalRandom.current().nextDouble() * 360; + double distance = ThreadLocalRandom.current().nextDouble() * radiusMetres; + return destination(centre, bearing, distance); + } +} diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Load.java b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Load.java new file mode 100644 index 000000000..20de9d343 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/java/local/drones/Load.java @@ -0,0 +1,82 @@ +package local.drones; + +import static com.github.phisgr.gatling.kt.grpc.GrpcDsl.*; +import static io.gatling.javaapi.core.CoreDsl.*; + +import com.github.phisgr.gatling.kt.grpc.StaticGrpcProtocol; +import com.github.phisgr.gatling.kt.grpc.action.GrpcCallActionBuilder; +import com.google.protobuf.Empty; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; +import io.gatling.javaapi.core.ScenarioBuilder; +import io.gatling.javaapi.core.Simulation; +import io.grpc.ManagedChannelBuilder; +import java.time.Duration; +import java.util.Iterator; +import java.util.Map; +import java.util.stream.Stream; +import local.drones.proto.DroneServiceGrpc; +import local.drones.proto.ReportLocationRequest; + +public class Load extends Simulation { + + static final Duration RAMP_TIME = Duration.ofMinutes(5); + static final int MAX_NUMBER_OF_DRONES = 500; + static final int NUMBER_OF_DELIVERIES_PER_DRONE = 2; + + static final Coordinates LOCATION = new Coordinates(59.33258, 18.0649); + static final int START_RADIUS = 1000; // metres + static final int DESTINATION_RADIUS = 5000; // metres + + // 2m / 100ms = 72 km/hour + static final Duration REPORT_EVERY = Duration.ofMillis(100); + static final int TRAVEL_DISTANCE = 2; // metres + + final Config config = ConfigFactory.load().getConfig("local-drone-control"); + final String host = config.getString("host"); + final int port = config.getInt("port"); + final boolean tls = config.getBoolean("tls"); + + final ManagedChannelBuilder builder = ManagedChannelBuilder.forAddress(host, port); + final ManagedChannelBuilder channelBuilder = tls ? builder : builder.usePlaintext(); + final StaticGrpcProtocol grpcProtocol = grpc(channelBuilder); + + Iterator> droneIds() { + return Stream.iterate(1, n -> n + 1) + .map(id -> Map.of("droneId", (Object) String.format("drone-%05d", id))) + .iterator(); + } + + Iterator randomDeliveryPath() { + Coordinates start = Coordinates.random(LOCATION, START_RADIUS); + Coordinates destination = Coordinates.random(LOCATION, DESTINATION_RADIUS); + return Coordinates.path(start, destination, TRAVEL_DISTANCE); + } + + GrpcCallActionBuilder reportLocation = + grpc("report location") + .rpc(DroneServiceGrpc.getReportLocationMethod()) + .payload( + session -> { + Coordinates coordinates = session.>get("path").next(); + return ReportLocationRequest.newBuilder() + .setDroneId(session.getString("droneId")) + .setCoordinates(coordinates.toProto()) + .setAltitude(10) + .build(); + }); + + ScenarioBuilder updateDrones = + scenario("update drones") + .feed(droneIds()) + .repeat(NUMBER_OF_DELIVERIES_PER_DRONE) + .on( + exec(session -> session.set("path", randomDeliveryPath())) + .doWhile(session -> session.>get("path").hasNext()) + .on(pace(REPORT_EVERY).exec(reportLocation))); + + { + setUp(updateDrones.injectOpen(rampUsers(MAX_NUMBER_OF_DRONES).during(RAMP_TIME))) + .protocols(grpcProtocol); + } +} diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/common/coordinates.proto b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/common/coordinates.proto new file mode 100644 index 000000000..99f385d72 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/common/coordinates.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "common.proto"; + +package common; + +// generic messages, shared between local-drone-control and restaurant-drone-deliveries + +message Coordinates { + // latitude (north-south) in decimal degree coordinates + double latitude = 1; + // longitude (east west) in decimal degree coordinates + double longitude = 2; +} diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/deliveries_queue_api.proto b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/deliveries_queue_api.proto new file mode 100644 index 000000000..19bff6e72 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/deliveries_queue_api.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "local.drones.proto"; + +import "google/protobuf/empty.proto"; +import "common/coordinates.proto"; + +package local.drones; + +// gRPC definition for DroneService, for drones to interact with + +service DeliveriesQueueService { + rpc GetCurrentQueue (google.protobuf.Empty) returns (GetCurrentQueueResponse) {} +} + +message GetCurrentQueueResponse { + repeated WaitingDelivery waitingDeliveries = 1; + repeated DeliveryInProgress deliveriesInProgress = 2; +} + +message WaitingDelivery { + string delivery_id = 1; + common.Coordinates from = 2; + common.Coordinates to = 3; +} + +message DeliveryInProgress { + string delivery_id = 1; + string drone_id = 2; +} \ No newline at end of file diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/drone_api.proto b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/drone_api.proto new file mode 100644 index 000000000..853967fc0 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/proto/local/drones/drone_api.proto @@ -0,0 +1,40 @@ +syntax = "proto3"; + +option java_multiple_files = true; +option java_package = "local.drones.proto"; + +import "google/protobuf/empty.proto"; +import "common/coordinates.proto"; + +package local.drones; + +// gRPC definition for DroneService, for drones to interact with + +service DroneService { + rpc ReportLocation (ReportLocationRequest) returns (google.protobuf.Empty) {} + + // deliveries + rpc RequestNextDelivery (RequestNextDeliveryRequest) returns (RequestNextDeliveryResponse) {} + rpc CompleteDelivery (CompleteDeliveryRequest) returns (google.protobuf.Empty) {} +} + +message ReportLocationRequest { + string drone_id = 1; + common.Coordinates coordinates = 2; + // altitude in meters + double altitude = 4; +} + +message RequestNextDeliveryRequest { + string drone_id = 1; +} + +message RequestNextDeliveryResponse { + string delivery_id = 1; + common.Coordinates from = 2; + common.Coordinates to = 3; +} + +message CompleteDeliveryRequest { + string delivery_id = 1; +} diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/application.conf b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/application.conf new file mode 100644 index 000000000..bc117755d --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/application.conf @@ -0,0 +1,8 @@ +local-drone-control { + host = "localhost" + host = ${?LOCAL_DRONE_CONTROL_HOST} + port = 8080 + port = ${?LOCAL_DRONE_CONTROL_PORT} + tls = false + tls = ${?LOCAL_DRONE_CONTROL_TLS} +} diff --git a/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/logback.xml b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/logback.xml new file mode 100644 index 000000000..5f4c8a008 --- /dev/null +++ b/samples/grpc/local-drone-control-java/autoscaling/simulator/src/test/resources/logback.xml @@ -0,0 +1,15 @@ + + + + %date{ISO8601} %-5level %logger - %msg%n + + + + + + + + + + + diff --git a/samples/grpc/local-drone-control-java/pom.xml b/samples/grpc/local-drone-control-java/pom.xml index fa2866815..2547f8e3a 100644 --- a/samples/grpc/local-drone-control-java/pom.xml +++ b/samples/grpc/local-drone-control-java/pom.xml @@ -26,6 +26,7 @@ 2.3.4 2.3.4 1.3.11 + 0.16.0 4.13.1 2.13 0.9.26 @@ -150,6 +151,18 @@ ${logback.version} + + + io.prometheus + simpleclient + ${prometheus.client.version} + + + io.prometheus + simpleclient_httpserver + ${prometheus.client.version} + + com.typesafe.akka akka-actor-testkit-typed_${scala.binary.version} diff --git a/samples/grpc/local-drone-control-java/src/main/java/local/drones/ClusteredMain.java b/samples/grpc/local-drone-control-java/src/main/java/local/drones/ClusteredMain.java index 48c5f73f8..e3432fb34 100644 --- a/samples/grpc/local-drone-control-java/src/main/java/local/drones/ClusteredMain.java +++ b/samples/grpc/local-drone-control-java/src/main/java/local/drones/ClusteredMain.java @@ -38,6 +38,9 @@ private static Behavior rootBehavior() { Drone.init(context.getSystem()); DroneEvents.initEventToCloudDaemonProcess(context.getSystem(), settings); + // start prometheus for custom metrics + Telemetry.Id.get(context.getSystem()).start(); + // consume delivery events from the cloud service, single queue in cluster singleton var deliveriesQueue = ClusterSingleton.get(context.getSystem()) diff --git a/samples/grpc/local-drone-control-java/src/main/java/local/drones/Drone.java b/samples/grpc/local-drone-control-java/src/main/java/local/drones/Drone.java index dac370083..57a95df0f 100644 --- a/samples/grpc/local-drone-control-java/src/main/java/local/drones/Drone.java +++ b/samples/grpc/local-drone-control-java/src/main/java/local/drones/Drone.java @@ -6,7 +6,9 @@ import akka.actor.typed.ActorRef; import akka.actor.typed.ActorSystem; import akka.actor.typed.Behavior; +import akka.actor.typed.PostStop; import akka.actor.typed.SupervisorStrategy; +import akka.actor.typed.javadsl.Behaviors; import akka.cluster.sharding.typed.javadsl.ClusterSharding; import akka.cluster.sharding.typed.javadsl.Entity; import akka.cluster.sharding.typed.javadsl.EntityTypeKey; @@ -16,6 +18,7 @@ import akka.persistence.typed.javadsl.Effect; import akka.persistence.typed.javadsl.EventHandler; import akka.persistence.typed.javadsl.EventSourcedBehavior; +import akka.persistence.typed.javadsl.SignalHandler; import akka.serialization.jackson.CborSerializable; import com.fasterxml.jackson.annotation.JsonCreator; import java.time.Duration; @@ -111,11 +114,23 @@ Optional coarseGrainedCoordinates() { public static void init(ActorSystem system) { ClusterSharding.get(system) - .init(Entity.of(ENTITY_KEY, entityContext -> new Drone(entityContext.getEntityId()))); + .init(Entity.of(ENTITY_KEY, entityContext -> Drone.create(entityContext.getEntityId()))); } public static Behavior create(String entityId) { - return new Drone(entityId); + return Behaviors.setup( + context -> { + Telemetry telemetry = Telemetry.Id.get(context.getSystem()); + telemetry.droneEntityActivated(); + return new Drone(entityId) { + @Override + public SignalHandler signalHandler() { + return newSignalHandlerBuilder() + .onSignal(PostStop.instance(), state -> telemetry.droneEntityPassivated()) + .build(); + } + }; + }); } private Drone(String entityId) { diff --git a/samples/grpc/local-drone-control-java/src/main/java/local/drones/Telemetry.java b/samples/grpc/local-drone-control-java/src/main/java/local/drones/Telemetry.java new file mode 100644 index 000000000..0966ebfa7 --- /dev/null +++ b/samples/grpc/local-drone-control-java/src/main/java/local/drones/Telemetry.java @@ -0,0 +1,51 @@ +package local.drones; + +import akka.actor.typed.ActorSystem; +import akka.actor.typed.Extension; +import akka.actor.typed.ExtensionId; +import io.prometheus.client.Gauge; +import io.prometheus.client.exporter.HTTPServer; +import java.io.IOException; + +public class Telemetry implements Extension { + + public static class Id extends ExtensionId { + + private static final Id instance = new Id(); + + private Id() {} + + @Override + public Telemetry createExtension(ActorSystem system) { + return new Telemetry(system); + } + + public static Telemetry get(ActorSystem system) { + return instance.apply(system); + } + } + + private final int prometheusPort; + + private Telemetry(ActorSystem system) { + this.prometheusPort = system.settings().config().getInt("prometheus.port"); + } + + public void start() throws IOException { + new HTTPServer.Builder().withPort(prometheusPort).build(); + } + + private final Gauge activeDroneEntities = + Gauge.build() + .name("local_drone_control_active_entities") + .help("Number of currently active drone entities.") + .register(); + + public void droneEntityActivated() { + activeDroneEntities.inc(); + } + + public void droneEntityPassivated() { + activeDroneEntities.dec(); + } +} diff --git a/samples/grpc/local-drone-control-java/src/main/resources/application-cluster.conf b/samples/grpc/local-drone-control-java/src/main/resources/application-cluster.conf index db6bf40f6..1f28a6ad5 100644 --- a/samples/grpc/local-drone-control-java/src/main/resources/application-cluster.conf +++ b/samples/grpc/local-drone-control-java/src/main/resources/application-cluster.conf @@ -17,3 +17,6 @@ local-drone-control { akka { loglevel = DEBUG } + +prometheus.port = 9090 +prometheus.port = ${?PROMETHEUS_PORT} diff --git a/samples/grpc/local-drone-control-java/src/main/resources/cluster.conf b/samples/grpc/local-drone-control-java/src/main/resources/cluster.conf index 9e1d57973..1b4140745 100644 --- a/samples/grpc/local-drone-control-java/src/main/resources/cluster.conf +++ b/samples/grpc/local-drone-control-java/src/main/resources/cluster.conf @@ -16,7 +16,10 @@ akka.cluster { least-shard-allocation-strategy.rebalance-absolute-limit = 20 passivation { strategy = default-strategy - active-entity-limit = 1000 + default-strategy { + active-entity-limit = 1000 + idle-entity.timeout = 3 minutes + } } } } diff --git a/samples/grpc/local-drone-control-java/src/main/resources/local1.conf b/samples/grpc/local-drone-control-java/src/main/resources/local1.conf index d7a7043d6..e85fda025 100644 --- a/samples/grpc/local-drone-control-java/src/main/resources/local1.conf +++ b/samples/grpc/local-drone-control-java/src/main/resources/local1.conf @@ -8,3 +8,4 @@ local-drone-control.grpc.port = 8080 akka.remote.artery.canonical.port = 2651 akka.management.http.port = 9201 +prometheus.port = 9091 diff --git a/samples/grpc/local-drone-control-java/src/main/resources/local2.conf b/samples/grpc/local-drone-control-java/src/main/resources/local2.conf index 7ab596b7f..5c3115e2d 100644 --- a/samples/grpc/local-drone-control-java/src/main/resources/local2.conf +++ b/samples/grpc/local-drone-control-java/src/main/resources/local2.conf @@ -5,5 +5,7 @@ include "local-shared" local-drone-control.grpc.port = 8081 -akka.management.http.port = 9202 akka.remote.artery.canonical.port = 2652 +akka.management.http.port = 9202 + +prometheus.port = 9092 diff --git a/samples/grpc/local-drone-control-java/src/main/resources/local3.conf b/samples/grpc/local-drone-control-java/src/main/resources/local3.conf index bbedac07c..f434374e0 100644 --- a/samples/grpc/local-drone-control-java/src/main/resources/local3.conf +++ b/samples/grpc/local-drone-control-java/src/main/resources/local3.conf @@ -7,3 +7,5 @@ local-drone-control.grpc.port = 8082 akka.remote.artery.canonical.port = 2653 akka.management.http.port = 9203 + +prometheus.port = 9093