From cba0c41afddf09f916e51bf0b6ec2e6f6a8032b5 Mon Sep 17 00:00:00 2001 From: Leonardo Graboski Veiga Date: Fri, 12 Jul 2024 22:46:09 -0300 Subject: [PATCH 1/3] data: fill DB on demo mode Use a number greater than the InfluxDB data retention period, so that the DB is filled and the retention discard is tested. Signed-off-by: Leonardo Graboski Veiga --- dataAcquisition/get-build-status.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dataAcquisition/get-build-status.sh b/dataAcquisition/get-build-status.sh index e4dc769..4f99354 100755 --- a/dataAcquisition/get-build-status.sh +++ b/dataAcquisition/get-build-status.sh @@ -57,7 +57,7 @@ logger "Polling interval set to $POLL_INTERVAL_SEC seconds" "INFO" if [[ -n "$DEMO" ]]; then logger "Demo mode enabled" "INFO" logger "Generating fake data at every poll interval" "INFO" - FAKEDATECOUNT=30 + FAKEDATECOUNT=190 else logger "Demo mode disabled" "INFO" fi From 9ab6763a0f1fe486886424431a52b01f2d04164f Mon Sep 17 00:00:00 2001 From: Leonardo Graboski Veiga Date: Sat, 13 Jul 2024 10:22:03 -0300 Subject: [PATCH 2/3] influxdb: limit container RAM allocation When the DB starts to get too much data, InfluxDB gets really memory-hungry. This leads to unexpected reboots as the OOM seems to be killing the Docker daemon, which is watched by SystemD watchdog. This commit does not fix the underlying issue with InfluxDB memory consumption, but it prevents the unexpected reboots. Signed-off-by: Leonardo Graboski Veiga --- dataAcquisition/docker-compose.prod.yml | 4 ++++ dataAcquisition/docker-compose.yml | 4 ++++ docker-compose.prod.yml | 4 ++++ docker-compose.rc.yml | 4 ++++ docker-compose.yml | 4 ++++ 5 files changed, 20 insertions(+) diff --git a/dataAcquisition/docker-compose.prod.yml b/dataAcquisition/docker-compose.prod.yml index d5ba4d0..c46c1b9 100644 --- a/dataAcquisition/docker-compose.prod.yml +++ b/dataAcquisition/docker-compose.prod.yml @@ -27,6 +27,10 @@ services: - DOCKER_INFLUXDB_INIT_RETENTION=60m - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=T0rizonTh3Ea&IndustrialPlatform - INFLUXD_LOG_LEVEL=error + deploy: + resources: + limits: + memory: 1G grafana: platform: linux/arm64 ports: diff --git a/dataAcquisition/docker-compose.yml b/dataAcquisition/docker-compose.yml index 2a96df3..ba4a205 100644 --- a/dataAcquisition/docker-compose.yml +++ b/dataAcquisition/docker-compose.yml @@ -35,6 +35,10 @@ services: - DOCKER_INFLUXDB_INIT_RETENTION=60m - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=T0rizonTh3Ea&IndustrialPlatform - INFLUXD_LOG_LEVEL=error + deploy: + resources: + limits: + memory: 1G grafana: platform: linux/arm64 diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b5ef82d..b32bfb8 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -32,6 +32,10 @@ services: volumes: - "influxdb-storage:/var/lib/influxdb2" restart: on-failure + deploy: + resources: + limits: + memory: 1G grafana: image: leograba/torizon-os-build-status-grafana:0.4.0 diff --git a/docker-compose.rc.yml b/docker-compose.rc.yml index c28b71f..af50e7f 100644 --- a/docker-compose.rc.yml +++ b/docker-compose.rc.yml @@ -32,6 +32,10 @@ services: volumes: - "influxdb-storage:/var/lib/influxdb2" restart: on-failure + deploy: + resources: + limits: + memory: 1G grafana: image: leograba/torizon-os-build-status-grafana:rc diff --git a/docker-compose.yml b/docker-compose.yml index 62980b4..065645c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,10 @@ services: volumes: - "influxdb-volume:/var/lib/influxdb2" restart: on-failure + deploy: + resources: + limits: + memory: 1G grafana: image: ${DOCKER_LOGIN}/torizon-os-build-status-grafana:${TAG} From 6128cbc9606287348b16fa1accbcc7a207786d49 Mon Sep 17 00:00:00 2001 From: Leonardo Graboski Veiga Date: Sat, 13 Jul 2024 11:31:28 -0300 Subject: [PATCH 3/3] Release 0.4.1 * Enable uptime Fluent Bit metrics * Fill the entire DB on demo mode * Limit InfluxDB container RAM allocation Signed-off-by: Leonardo Graboski Veiga --- .vscode/settings.json | 2 +- browser/.vscode/settings.json | 2 +- browser/docker-compose.prod.yml | 2 +- dataAcquisition/.vscode/settings.json | 2 +- dataAcquisition/docker-compose.prod.yml | 56 ++++++++++++------------- dataAcquisition/docker-compose.yml | 4 +- docker-compose.prod.yml | 10 ++--- grafana/.vscode/settings.json | 2 +- grafana/docker-compose.prod.yml | 4 +- influxdb/.vscode/settings.json | 2 +- influxdb/docker-compose.prod.yml | 4 +- tcb/docker-compose.prod.yml | 10 ++--- weston/.vscode/settings.json | 2 +- weston/docker-compose.prod.yml | 4 +- 14 files changed, 53 insertions(+), 53 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 6eb70f6..068925f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "__change__", "tcb.version": "3.8.1", - "project_version": "0.4.0" + "project_version": "0.4.1" } diff --git a/browser/.vscode/settings.json b/browser/.vscode/settings.json index 760c332..b438b21 100644 --- a/browser/.vscode/settings.json +++ b/browser/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "TorizonOSBuildStatusBrowser", "tcb.version": "3.8.1", - "docker_tag": "0.4.0" + "docker_tag": "0.4.1" } \ No newline at end of file diff --git a/browser/docker-compose.prod.yml b/browser/docker-compose.prod.yml index dd0d007..f0ea600 100644 --- a/browser/docker-compose.prod.yml +++ b/browser/docker-compose.prod.yml @@ -1,4 +1,4 @@ version: "3.9" services: torizon-os-build-status-browser: - image: leograba/torizon-os-build-status-browser:0.4.0 + image: leograba/torizon-os-build-status-browser:0.4.1 diff --git a/dataAcquisition/.vscode/settings.json b/dataAcquisition/.vscode/settings.json index f1efcf3..3719f95 100644 --- a/dataAcquisition/.vscode/settings.json +++ b/dataAcquisition/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "TorizonOSBuildStatusDataAcquisition", "tcb.version": "3.8.1", - "docker_tag": "0.4.0" + "docker_tag": "0.4.1" } \ No newline at end of file diff --git a/dataAcquisition/docker-compose.prod.yml b/dataAcquisition/docker-compose.prod.yml index c46c1b9..b6a6b89 100644 --- a/dataAcquisition/docker-compose.prod.yml +++ b/dataAcquisition/docker-compose.prod.yml @@ -1,9 +1,7 @@ +version: "3.9" services: torizon-os-build-status-data-acquisition: platform: linux/arm64 - environment: - - DEBUG=DEBUG - - DEMO=1 networks: - influxdbnet - frontend @@ -12,12 +10,35 @@ services: - "8086" depends_on: - influxdb - image: leograba/torizon-os-build-status-data-acquisition:rc + image: leograba/torizon-os-build-status-data-acquisition:0.4.1 + environment: + - DEBUG=DEBUG + - DEMO=1 + grafana: + platform: linux/arm64 + volumes: + - $PWD/debug/grafana-config/grafana.ini:/etc/grafana/grafana.ini + - $PWD/debug/grafana-config/provisioning/datasources/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - $PWD/debug/grafana-config/provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml + - $PWD/debug/grafana-config/provisioning/dashboards/dashboard.json:/var/lib/grafana/dashboards/dashboard.json + ports: + - 3000:3000 + networks: + - influxdbnet + - frontend + depends_on: + - influxdb + - torizon-os-build-status-data-acquisition + image: leograba/torizon-os-build-status-grafana:0.4.1 influxdb: - image: leograba/torizon-os-build-status-influxdb:0.4.0 platform: linux/arm64 networks: - influxdbnet + deploy: + resources: + limits: + memory: 1G + image: leograba/torizon-os-build-status-influxdb:0.4.1 environment: - DOCKER_INFLUXDB_INIT_MODE=setup - DOCKER_INFLUXDB_INIT_USERNAME=torizon @@ -27,29 +48,8 @@ services: - DOCKER_INFLUXDB_INIT_RETENTION=60m - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=T0rizonTh3Ea&IndustrialPlatform - INFLUXD_LOG_LEVEL=error - deploy: - resources: - limits: - memory: 1G - grafana: - platform: linux/arm64 - ports: - - 3000:3000 - networks: - - influxdbnet - - frontend - image: leograba/torizon-os-build-status-grafana:0.4.0 - depends_on: - - influxdb - - torizon-os-build-status-data-acquisition - volumes: - - $PWD/debug/grafana-config/grafana.ini:/etc/grafana/grafana.ini - - $PWD/debug/grafana-config/provisioning/datasources/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml - - $PWD/debug/grafana-config/provisioning/dashboards/dashboard.yaml:/etc/grafana/provisioning/dashboards/dashboard.yaml - - $PWD/debug/grafana-config/provisioning/dashboards/dashboard.json:/var/lib/grafana/dashboards/dashboard.json networks: - frontend: - internal: false influxdbnet: internal: true -version: "3.9" + frontend: + internal: false diff --git a/dataAcquisition/docker-compose.yml b/dataAcquisition/docker-compose.yml index ba4a205..a093e39 100644 --- a/dataAcquisition/docker-compose.yml +++ b/dataAcquisition/docker-compose.yml @@ -23,7 +23,7 @@ services: influxdb: platform: linux/arm64 - image: leograba/torizon-os-build-status-influxdb:0.4.0 + image: leograba/torizon-os-build-status-influxdb:0.4.1 networks: - influxdbnet environment: @@ -42,7 +42,7 @@ services: grafana: platform: linux/arm64 - image: leograba/torizon-os-build-status-grafana:0.4.0 + image: leograba/torizon-os-build-status-grafana:0.4.1 networks: - influxdbnet - frontend diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b32bfb8..1889b09 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -2,7 +2,7 @@ version: "3.9" services: data-acquisition: - image: leograba/torizon-os-build-status-data-acquisition:0.4.0 + image: leograba/torizon-os-build-status-data-acquisition:0.4.1 platform: linux/arm64 networks: - influxdbnet @@ -16,7 +16,7 @@ services: restart: on-failure influxdb: - image: leograba/torizon-os-build-status-influxdb:0.4.0 + image: leograba/torizon-os-build-status-influxdb:0.4.1 platform: linux/arm64 networks: - influxdbnet @@ -38,7 +38,7 @@ services: memory: 1G grafana: - image: leograba/torizon-os-build-status-grafana:0.4.0 + image: leograba/torizon-os-build-status-grafana:0.4.1 platform: linux/arm64 networks: - influxdbnet @@ -53,7 +53,7 @@ services: restart: on-failure weston: - image: leograba/torizon-os-build-status-weston:0.4.0 + image: leograba/torizon-os-build-status-weston:0.4.1 platform: linux/arm # Required to get udev events from host udevd via netlink network_mode: host @@ -80,7 +80,7 @@ services: restart: on-failure browser: - image: leograba/torizon-os-build-status-browser:0.4.0 + image: leograba/torizon-os-build-status-browser:0.4.1 platform: linux/arm environment: - MACHINE diff --git a/grafana/.vscode/settings.json b/grafana/.vscode/settings.json index 2b6ef5f..38e1bcd 100644 --- a/grafana/.vscode/settings.json +++ b/grafana/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "TorizonOSBuildStatusGrafana", "tcb.version": "3.8.1", - "docker_tag": "0.4.0" + "docker_tag": "0.4.1" } \ No newline at end of file diff --git a/grafana/docker-compose.prod.yml b/grafana/docker-compose.prod.yml index e74cbed..54d9aff 100644 --- a/grafana/docker-compose.prod.yml +++ b/grafana/docker-compose.prod.yml @@ -1,4 +1,4 @@ +version: "3.9" services: torizon-os-build-status-grafana: - image: leograba/torizon-os-build-status-grafana:0.4.0 -version: "3.9" + image: leograba/torizon-os-build-status-grafana:0.4.1 diff --git a/influxdb/.vscode/settings.json b/influxdb/.vscode/settings.json index 7b2f303..8594491 100644 --- a/influxdb/.vscode/settings.json +++ b/influxdb/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "TorizonOSBuildStatusInfluxDB", "tcb.version": "3.8.1", - "docker_tag": "0.4.0" + "docker_tag": "0.4.1" } \ No newline at end of file diff --git a/influxdb/docker-compose.prod.yml b/influxdb/docker-compose.prod.yml index dd38529..410a002 100644 --- a/influxdb/docker-compose.prod.yml +++ b/influxdb/docker-compose.prod.yml @@ -1,4 +1,4 @@ -version: "3.9" services: torizon-os-build-status-influxdb: - image: leograba/torizon-os-build-status-influxdb:0.4.0 + image: leograba/torizon-os-build-status-influxdb:0.4.1 +version: "3.9" diff --git a/tcb/docker-compose.prod.yml b/tcb/docker-compose.prod.yml index 75ee597..19c12fd 100644 --- a/tcb/docker-compose.prod.yml +++ b/tcb/docker-compose.prod.yml @@ -2,7 +2,7 @@ version: "3.9" services: data-acquisition: - image: leograba/torizon-os-build-status-data-acquisition:0.4.0 + image: leograba/torizon-os-build-status-data-acquisition:0.4.1 networks: - influxdbnet - frontend @@ -12,7 +12,7 @@ services: restart: on-failure influxdb: - image: leograba/torizon-os-build-status-influxdb:0.4.0 + image: leograba/torizon-os-build-status-influxdb:0.4.1 networks: - influxdbnet environment: @@ -28,7 +28,7 @@ services: restart: on-failure grafana: - image: leograba/torizon-os-build-status-grafana:0.4.0 + image: leograba/torizon-os-build-status-grafana:0.4.1 networks: - influxdbnet - frontend @@ -42,7 +42,7 @@ services: restart: on-failure weston: - image: leograba/torizon-os-build-status-weston:0.4.0 + image: leograba/torizon-os-build-status-weston:0.4.1 # Required to get udev events from host udevd via netlink network_mode: host volumes: @@ -68,7 +68,7 @@ services: restart: on-failure browser: - image: leograba/torizon-os-build-status-browser:0.4.0 + image: leograba/torizon-os-build-status-browser:0.4.1 environment: - MACHINE security_opt: diff --git a/weston/.vscode/settings.json b/weston/.vscode/settings.json index 2dfaa46..ca5be50 100644 --- a/weston/.vscode/settings.json +++ b/weston/.vscode/settings.json @@ -15,5 +15,5 @@ "torizon_app_root": "/home/torizon", "tcb.packageName": "TorizonOSBuildStatusWeston", "tcb.version": "3.8.1", - "docker_tag": "0.4.0" + "docker_tag": "0.4.1" } \ No newline at end of file diff --git a/weston/docker-compose.prod.yml b/weston/docker-compose.prod.yml index 1e1e6dd..08d0232 100644 --- a/weston/docker-compose.prod.yml +++ b/weston/docker-compose.prod.yml @@ -1,4 +1,4 @@ -version: "3.9" services: torizon-os-build-status-weston: - image: leograba/torizon-os-build-status-weston:0.4.0 + image: leograba/torizon-os-build-status-weston:0.4.1 +version: "3.9"