Implement automation for archival snapshots

.github/workflows/deploy-archival-snapshots.yml

@@ -0,0 +1,57 @@
+name: Export archival snapshots
+
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  pull_request:
+    paths:
+      - 'ansible/archival-snapshots/**'
+  push:
+    paths:
+      - 'ansible/archival-snapshots/**'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Ansible
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ansible
+
+      - name: Download and install Cloudflared
+        run: |
+          wget https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb
+          sudo dpkg -i cloudflared-linux-amd64.deb
+          cloudflared --version
+
+      - name: Configure ssh-agent
+        uses: webfactory/[email protected]
+        with:
+          ssh-private-key: ${{ secrets.ARCHIE_PRIVATE_KEY }}
+
+      - name: Store SSH key
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.ARCHIE_PRIVATE_KEY }}
+        run: |
+          cat "$GITHUB_WORKSPACE/ansible/archival-snapshots/resources/ssh_config" >> ~/.ssh/config
+          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa_archie
+          chmod 600 ~/.ssh/id_rsa_archie
+
+      - name: Run Ansible playbook
+        env:
+          ANSIBLE_HOST_KEY_CHECKING: "False"
+          ARCHIVAL_SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+          ENDPOINT: https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/
+        run: |
+          ansible-playbook -i ansible/archival-snapshots/inventory.ini ansible/archival-snapshots/playbook.yml

ansible/archival-snapshots/inventory.ini

@@ -0,0 +1,2 @@
+[remote_server]
+archie.chainsafe.dev ansible_user=archie ansible_ssh_private_key_file=~/.ssh/id_rsa_archie

ansible/archival-snapshots/playbook.yml

@@ -0,0 +1,66 @@
+---
+- name: Automate packaging, transferring, and executing a script
+  hosts: remote_server
+  vars:
+    local_resources_path: "resources"
+    remote_resources_path: "/mnt/md0/exported/archival"
+    zip_file_name: "resources.zip"
+    forest_version: "v0.17.2"
+    forest_release_url: "https://github.com/ChainSafe/forest/releases/download/{{ forest_version }}/forest-{{ forest_version }}-linux-amd64.zip"
+  tasks:
+    - name: Check if AWS CLI is installed
+      ansible.builtin.command:
+        cmd: "which aws"
+      register: aws_installed
+      changed_when: false
+      ignore_errors: true
+
+    - name: Install AWS CLI if not installed
+      ansible.builtin.command:
+        cmd: "sudo apt-get update && sudo apt-get install -y awscli"
+      when: aws_installed.rc != 0
+
+    - name: Check if Ruby is installed
+      ansible.builtin.command:
+        cmd: "which ruby"
+      register: ruby_installed
+      changed_when: false
+      ignore_errors: true
+
+    - name: Install Ruby if not installed
+      ansible.builtin.command:
+        cmd: "sudo apt-get update && sudo apt-get install -y ruby-full"
+      when: ruby_installed.rc != 0
+
+    - name: Zip the resources folder
+      ansible.builtin.command:
+        cmd: "zip -r {{ zip_file_name }} ."
+        chdir: "{{ local_resources_path }}"
+      delegate_to: localhost
+
+    - name: Transfer the zip file to the remote server
+      ansible.builtin.copy:
+        src: "{{ local_resources_path }}/{{ zip_file_name }}"
+        dest: "{{ remote_resources_path }}/{{ zip_file_name }}"
+
+    - name: Unzip the resources folder on the remote server
+      ansible.builtin.command:
+        cmd: "unzip -o {{ zip_file_name }}"
+        chdir: "{{ remote_resources_path }}"
+
+    - name: Download Forest release package
+      ansible.builtin.get_url:
+        url: "{{ forest_release_url }}"
+        dest: "{{ remote_resources_path }}/forest.zip"
+
+    - name: Unzip the Forest release package on the remote server
+      ansible.builtin.command:
+        cmd: "unzip -jo {{ remote_resources_path }}/forest.zip -d {{ remote_resources_path }}/forest/"
+
+    - name: Execute the init.sh script
+      ansible.builtin.shell:
+        cmd: "nohup ./init.sh > init.log 2>&1 &"
+        chdir: "{{ remote_resources_path }}"
+      environment:
+        ARCHIVAL_SLACK_TOKEN: "{{ lookup('env', 'ARCHIVAL_SLACK_TOKEN') }}"
+        ENDPOINT: "{{ lookup('env', 'ENDPOINT') }}"

ansible/archival-snapshots/resources/config.toml

@@ -0,0 +1,3 @@
+[client]
+data_dir         = "/mnt/md0/forest-archival-data"
+encrypt_keystore = false

ansible/archival-snapshots/resources/diff_script.sh

@@ -0,0 +1,41 @@
+#!/bin/env bash
+
+set -euxo pipefail
+
+FOREST=/mnt/md0/exported/archival/forest/forest-tool
+UPLOADED_DIFFS=/mnt/md0/exported/archival/uploaded-diff-snaps.txt
+UPLOAD_QUEUE="/mnt/md0/exported/archival/upload_files.txt"
+
+EPOCH_START="$1"
+shift
+DIFF_STEP=3000
+DIFF_COUNT=10
+GENESIS_TIMESTAMP=1598306400
+SECONDS_PER_EPOCH=30
+
+# Clear Upload List
+if [ -f "$UPLOAD_QUEUE" ]; then
+    # Clear the contents of the file
+    true > "$UPLOAD_QUEUE"
+fi
+
+
+aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/diff/" > "$UPLOADED_DIFFS"
+
+for i in $(seq 1 $DIFF_COUNT); do
+    EPOCH=$((EPOCH_START+DIFF_STEP*i))
+    EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + EPOCH*SECONDS_PER_EPOCH))
+    DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I)
+    FILE_NAME="forest_diff_mainnet_${DATE}_height_$((EPOCH-DIFF_STEP))+$DIFF_STEP.forest.car.zst"
+    FILE="/mnt/md0/exported/archival/diff_snapshots/$FILE_NAME"
+    if ! grep -q "$FILE_NAME" "$UPLOADED_DIFFS"; then
+        if ! test -f "$FILE"; then
+            # Export diff snapshot
+            "$FOREST" archive export --depth "$DIFF_STEP" --epoch "$EPOCH" --diff $((EPOCH-DIFF_STEP)) --diff-depth 900 --output-path "$FILE" "$@"
+        fi
+        # Add exported diff snapshot to upload queue
+        echo "$FILE" >> "$UPLOAD_QUEUE"
+    else
+        echo "Skipping $FILE"
+    fi
+done

ansible/archival-snapshots/resources/init.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+## Enable strict error handling, command tracing, and pipefail
+set -eux
+
+# Initialize snapshots directory
+LITE_SNAPSHOT_DIR="/mnt/md0/exported/archival/lite_snapshots"
+DIFF_SNAPSHOT_DIR="/mnt/md0/exported/archival/diff_snapshots"
+FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots
+
+if [ ! -d "$LITE_SNAPSHOT_DIR" ]; then
+    mkdir -p "$LITE_SNAPSHOT_DIR"
+    echo "Created $LITE_SNAPSHOT_DIR"
+else
+    echo "$LITE_SNAPSHOT_DIR exists"
+fi
+
+if [ ! -d "$DIFF_SNAPSHOT_DIR" ]; then
+    mkdir -p "$DIFF_SNAPSHOT_DIR"
+    echo "Created $DIFF_SNAPSHOT_DIR"
+else
+    echo "$DIFF_SNAPSHOT_DIR exists"
+fi
+
+if [ ! -d "$FULL_SNAPSHOTS_DIR" ]; then
+    mkdir -p "$FULL_SNAPSHOTS_DIR"
+    echo "Created $FULL_SNAPSHOTS_DIR"
+else
+    echo "$FULL_SNAPSHOTS_DIR exists"
+fi
+
+# Trigger main script
+./main.sh
+
+EXIT_STATUS=$?
+
+# Notify on slack channel
+if [ "$EXIT_STATUS" -eq 0 ]; then
+    echo "Script executed successfully"
+    ruby notify.rb success
+else
+    echo "Script execution failed"
+    ruby notify.rb failure
+fi
+
+exit "$EXIT_STATUS"

ansible/archival-snapshots/resources/main.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+
+## Enable strict error handling, command tracing, and pipefail
+set -euxo pipefail
+
+## Set constants
+GENESIS_TIMESTAMP=1598306400
+SECONDS_PER_EPOCH=30
+
+## Set forest artifacts path
+FOREST="/mnt/md0/exported/archival/forest/forest"
+FOREST_CLI="/mnt/md0/exported/archival/forest/forest-cli"
+FOREST_TOOL="/mnt/md0/exported/archival/forest/forest-tool"
+
+LITE_SNAPSHOT_DIR=/mnt/md0/exported/archival/lite_snapshots
+FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots
+
+## Fetch last snapshot details
+LAST_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}')
+LAST_EPOCH=$(echo "$LAST_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}')
+LAST_FULL_SNAPSHOT_PATH="$FULL_SNAPSHOTS_DIR/$LAST_SNAPSHOT"
+
+if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then
+    echo "Downloading last snapshot: $LAST_FULL_SNAPSHOT_PATH"
+    aws --profile prod --endpoint "$ENDPOINT" s3 cp "s3://forest-archive/mainnet/lite/$LAST_SNAPSHOT" "$LAST_FULL_SNAPSHOT_PATH"
+    echo "Last snapshot download: $LAST_FULL_SNAPSHOT_PATH"
+else
+    echo "$LAST_FULL_SNAPSHOT_PATH snapshot exists."
+fi
+
+# Clean forest db
+$FOREST_TOOL db destroy --force
+
+echo "Starting forest daemon"
+nohup $FOREST --no-gc --config ./config.toml --save-token ./admin_token --rpc-address 127.0.0.1:3456 --metrics-address 127.0.0.1:5000 --import-snapshot "$LAST_FULL_SNAPSHOT_PATH" > forest.log 2>&1 &
+FOREST_NODE_PID=$!
+
+sleep 30
+echo "Forest process started with PID: $FOREST_NODE_PID"
+
+# Function to kill Forest daemon
+function kill_forest_daemon {
+    echo "Killing Forest daemon with PID: $FOREST_NODE_PID"
+    kill -KILL $FOREST_NODE_PID
+}
+
+# Set trap to kill Forest daemon on script exit or error
+trap kill_forest_daemon EXIT
+
+# Set required env variables
+function set_fullnode_api_info {
+    ADMIN_TOKEN=$(cat admin_token)
+    export FULLNODE_API_INFO="$ADMIN_TOKEN:/ip4/127.0.0.1/tcp/3456/http"
+    echo "Using: $FULLNODE_API_INFO"
+}
+set_fullnode_api_info
+
+# Wait for network to sync
+echo "Waiting for forest to sync to latest network head"
+$FOREST_CLI sync wait
+
+# Get latest epoch using sync status
+echo "Current Height: $LAST_EPOCH"
+LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}')
+echo "Latest Height: $LATEST_EPOCH"
+
+while ((LATEST_EPOCH - LAST_EPOCH > 30000)); do
+   set_fullnode_api_info
+   NEW_EPOCH=$((LAST_EPOCH + 30000))
+   echo "Next Height: $NEW_EPOCH"
+
+   # Export full snapshot to generate lite and diff snapshots
+   EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + NEW_EPOCH*SECONDS_PER_EPOCH))
+   DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I)
+   NEW_SNAPSHOT="forest_snapshot_mainnet_${DATE}_height_${NEW_EPOCH}.forest.car.zst"
+   if [ ! -f "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" ]; then
+        echo "Exporting snapshot: $FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
+        echo "USING FULLNODE API: $FULLNODE_API_INFO"
+        $FOREST_CLI snapshot export --tipset "$NEW_EPOCH" --depth 30000 -o "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" > export.txt
+        echo "Snapshot exported: $FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
+    else
+        echo "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT already exists."
+    fi
+
+   # Generate and upload lite snapshot
+   if [ ! -f "$LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" ]; then
+        echo "Generating Lite snapshot: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT"
+        $FOREST_TOOL archive export --epoch "$NEW_EPOCH" --output-path "$LITE_SNAPSHOT_DIR" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
+        echo "Lite snapshot generated: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT"
+    else
+        echo "$NEW_SNAPSHOT lite snapshot already exists."
+    fi
+   echo "Uploading Lite snapshot: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT"
+   aws --profile prod --endpoint "$ENDPOINT" s3 cp "$LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" "s3://forest-archive/mainnet/lite/"
+   echo "Lite snapshot uploaded: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT"
+
+   # Generate and upload diff snapshots
+   if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then
+       echo "File does not exist. Exporting..."
+       $FOREST_CLI snapshot export --tipset "$LAST_EPOCH" --depth 30000 -o "$LAST_FULL_SNAPSHOT_PATH"
+   else
+       echo "$LAST_FULL_SNAPSHOT_PATH file exists."
+   fi
+   echo "Generating Diff snapshots: $LAST_EPOCH - $NEW_EPOCH"
+   ./diff_script.sh "$LAST_EPOCH" "$LAST_FULL_SNAPSHOT_PATH" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT"
+   echo "Diff snapshots generated successfully"
+   echo "Uploading Diff snapshots"
+   ./upload_diff.sh "$ENDPOINT"
+   echo "Diff snapshots uploaded successfully"
+
+   LAST_EPOCH=$NEW_EPOCH
+done

ansible/archival-snapshots/resources/notify.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'slack-ruby-client'
+
+CHANNEL = '#forest-dump'
+SLACK_TOKEN = ENV.fetch('ARCHIVAL_SLACK_TOKEN')
+STATUS = ARGV[0]
+
+client = Slack::Web::Client.new(token: SLACK_TOKEN)
+
+message = if STATUS == 'success'
+            '✅ Lite and Diff snapshots updated. 🌲🌳🌲🌳🌲'
+          else
+            '❌ Failed to update Lite and Diff snapshots. 🔥🌲🔥'
+          end
+
+client.chat_postMessage(channel: CHANNEL, text: message, as_user: true)

ansible/archival-snapshots/resources/ssh_config

@@ -0,0 +1,3 @@
+Host archie.chainsafe.dev
+  ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h
+  User archie

ansible/archival-snapshots/resources/upload_diff.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+## Enable strict error handling, command tracing, and pipefail
+set -euxo pipefail
+
+ENDPOINT="$1"
+
+while read -r file; do
+    # Upload the file to the S3 bucket
+    aws --profile prod --endpoint "$ENDPOINT" s3 cp "$file" "s3://forest-archive/mainnet/diff/"
+done < /mnt/md0/exported/archival/upload_files.txt