Collect data from the log that can be processed into a notification a…

…nd passed to a monitoring system.

.gitlab-ci.yml

@@ -23,6 +23,11 @@ run-backups-ph3-local:
     SSH_PORT: 10022
     SSH_PRIVKEY: $PH3LOCAL_SSH_PRIVKEY
     SSH_KNOWN_HOSTS: $PH3LOCAL_SSH_KNOWN_HOSTS
+    SOURCE_DATASET: SSD1/VMs/machines/portainer1.ghanima.net
+    DESTINATION_DATASET: SLAB/Backups/Syncoid/ph3.local/portainer1.ghanima.net
+  artifacts:
+    paths:
+      - backup_result.json
   rules:
     - if: $RUNBACKUPS
   script: |
@@ -31,6 +36,9 @@ run-backups-ph3-local:
     SSH_PORT="${SSH_PORT:-10022}"
     SSH_PRIVKEY="${SSH_PRIVKEY}"
     SSH_KNOWN_HOSTS="${SSH_KNOWN_HOSTS}"
+    SOURCE_DATASET="${SOURCE_DATASET}"
+    DESTINATION_DATASET="${DESTINATION_DATASET}"
+    start_time=$(date +%s)
 
     # Don't assume that these normal environment variables are set.
     if [ "${USER}" == "" ]; then
@@ -82,8 +90,26 @@ run-backups-ph3-local:
     chmod 600 ${HOME}/.ssh/known_hosts
     cat ${HOME}/.ssh/known_hosts
 
-    syncoid --debug --dumpsnaps --create-bookmark --no-sync-snap --no-privilege-elevation --sendoptions="w" "${SSH_USERNAME}@${SSH_REMOTE_HOST}:SSD1/VMs/machines/portainer1.ghanima.net" "SLAB/Backups/Syncoid/ph3.local/portainer1.ghanima.net"
-    #syncoid --debug --dumpsnaps --create-bookmark --no-sync-snap --no-privilege-elevation --sendoptions="w" "${SSH_USERNAME}@${SSH_REMOTE_HOST}:RaidZ7T/Backups/Syncoid/ph3.local/portainer1.ghanima.net" "SLAB/Backups/Syncoid/ph3.local/portainer1.ghanima.net"
+    syncoid_log="/tmp/${SSH_REMOTE_HOST}.log"
+    syncoid --debug --dumpsnaps --create-bookmark --no-sync-snap --no-privilege-elevation --sendoptions="w" "${SSH_USERNAME}@${SSH_REMOTE_HOST}:${SOURCE_DATASET}" "${DESTINATION_DATASET}" | tee "${syncoid_log}"
+
+    send_size=$(cat "${syncoid_log}" | grep '^DEBUG: sendsize = ' | awk 'BEGIN{FS=" = ";total=0}{total=total+$2}END{print total}')
+    result_json=$(echo "{}" | jq ".sendsize=${send_size}")
+    resumed=$(cat "${syncoid_log}" | grep -c "^Resuming interrupted zfs send/receive")
+    result_json=$(printf '%s' "${result_json}" | jq ".resumed=${resumed}")
+    duration=$(( $(date +%s) - $start_time ))
+    result_json=$(printf '%s' "${result_json}" | jq ".resumed=${duration}")
+    error_msg=$(cat "${syncoid_log}" | grep -B 1 "CRITICAL ERROR:")
+    result_json=$(printf '%s' "${result_json}" | jq ".resumed=${error_msg}")
+    error_count=$(cat "${syncoid_log}" | grep -c "CRITICAL ERROR:")
+    result_json=$(printf '%s' "${result_json}" | jq ".resumed=${error_count}")
+    error_out_of_space=$(cat "${syncoid_log}" | grep -c "out of space")
+    result_json=$(printf '%s' "${result_json}" | jq ".commonIssues={destinationOutOfSpace:${error_out_of_space}}")
+    error_connection_refused=$((cat "${syncoid_log}" | grep -c 'ssh: connect to host chisel port.*Connection refused')
+    result_json=$(printf '%s' "${result_json}" | jq ".commonIssues={connectionToSourceRefused:${error_connection_refused}}")
+
+    printf '%s' "${result_json}" | jq -C
+    printf '%s' "${result_json}" > backup_result.json
 
 docker-chisel-image-get-version:
   stage: docker-get-version

pull-server/Dockerfile.syncoid-pull-server

@@ -1,20 +1,13 @@
 # TODO: Make this dynamic
 ARG BASE_IMAGE=gitlab/gitlab-runner:v16.0.2
 
-#I don't think we need chisel in here.
-#FROM cr.ghanima.net/applications/sanoid/chisel:latest AS chisel
-
 FROM ${BASE_IMAGE}
 ARG TARGETPLATFORM
 
-#ADD ./pull-server/entrypoint.sh /entrypoint.sh
-
 RUN        --mount=type=cache,target=/var/cache/apt,sharing=locked,id=var-cache-apt-$TARGETPLATFORM \
            --mount=type=cache,target=/var/lib/apt,sharing=locked,id=var-lib-apt-$TARGETPLATFORM \
            --mount=type=tmpfs,target=/var/cache/apk \
            --mount=type=tmpfs,target=/tmp \
            apt update; apt install -y zfsutils-linux mbuffer lzop sanoid
 
-#RUN chmod +x /entrypoint.sh
 
-#ENTRYPOINT /entrypoint.sh

pull-server/notifications/googlechat/backupFailure.json

@@ -0,0 +1,27 @@
+{
+    "cardsV2": [
+      {
+        "cardId": "test1",
+        "card": {
+          "name": "cardname",
+          "header": {
+            "title": "Backup Failed",
+            "imageUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/8/8a/Breezeicons-status-16-dropboxstatus-x.svg/1024px-Breezeicons-status-16-dropboxstatus-x.svg.png"
+          },
+          "sections": [
+            {
+              "widgets": [
+                {
+                  "textParagraph": {
+                    "text": "paragraph"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+    ]
+  }
+
+

pull-server/notifications/googlechat/backupSuccess.json

@@ -0,0 +1,27 @@
+{
+    "cardsV2": [
+      {
+        "cardId": "test1",
+        "card": {
+          "name": "cardname",
+          "header": {
+            "title": "Backup Success",
+            "imageUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/0/00/Breezeicons-status-16-dropboxstatus-idle.svg/1024px-Breezeicons-status-16-dropboxstatus-idle.svg.png"
+          },
+          "sections": [
+            {
+              "widgets": [
+                {
+                  "textParagraph": {
+                    "text": "paragraph"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+    ]
+  }
+
+

pull-server/notifications/googlechat/backupWarning.json

@@ -0,0 +1,27 @@
+{
+    "cardsV2": [
+      {
+        "cardId": "test1",
+        "card": {
+          "name": "cardname",
+          "header": {
+            "title": "Backup Warning",
+            "imageUrl": "https://upload.wikimedia.org/wikipedia/commons/thumb/7/75/Breezeicons-status-16-state-warning.svg/1024px-Breezeicons-status-16-state-warning.svg.png"
+          },
+          "sections": [
+            {
+              "widgets": [
+                {
+                  "textParagraph": {
+                    "text": "paragraph"
+                  }
+                }
+              ]
+            }
+          ]
+        }
+      }
+    ]
+  }
+
+