diff --git a/docs/README.md b/docs/README.md
new file mode 100644
index 00000000..e5171776
--- /dev/null
+++ b/docs/README.md
@@ -0,0 +1,109 @@
+# Recreate the full CAR archive
+
+If you're ready for months of toiling away, going through 100s of terabytes of data over and over, spending some hunnids per month on hardware and bandwidth and going through all the error and trial required for this task please continue reading below.
+
+
+## Hardware & other requirements
+
+1. Servers
+2. Object storage
+3. Software
+
+### Servers
+
+You'll need _big_ machines:
+- 20Tb working storage for juggling multiple snapshots + working data should be comfortable
+- 10Gbit uplink can shave hours of downloading/uploading stuff for each epoch
+- Good CPU power speeds up car generation
+
+We've used this Ryzen 9 from Hetzner with great success
+
+- AX102 with 10Gbit uplink + 4x7.68 NVMe (disks are pooled using LVM)
+
+https://www.hetzner.com/dedicated-rootserver/ax102/configurator/#/
+
+Order multiple servers to parallelize the work and speed up the process.
+
+### Object storage:
+
+We've uploaded all CAR files + indexes to Backblaze B2 due to storage cost + bandwidth alliance + egress fees.
+
+If we had to start again we could've probably self-hosted as upload/download performance and reliability was lacking at times.
+
+### Software:
+
+- Ubuntu 20
+- python3
+  - gsutil
+  - b2 (if you use B2 to store CAR files)
+  - `python3 -m pip install --upgrade gsutil b2`
+- s5cmd
+- zstd
+- /usr/local/bin/faithful-cli [faithful-cli](https://github.com/rpcpool/yellowstone-faithful)
+- /usr/local/bin/filecoin-data-prep [go-fil-dataprep](https://github.com/rpcpool/go-fil-dataprep)
+- Node Exporter
+
+
+### Preparation
+
+Find and replace the variables below with your own values (or use it as an Ansible template):
+
+- {{ warehouse_processor_snapshot_folder }} - Where to store snapshots, i.e. `/snapshots`
+- {{ warehouse_processor_car_folder }} - Where to save CAR files, i.e. `/cars`
+- {{ warehouse_slack_token }} - Webhook token to send messages to your Slack Channel
+- {{ inventory_hostname }} - Hostname where the script is installed, i.e. `$(hostname)`
+
+## Running
+
+We've used Rundeck as a controller node to initiate and run jobs on the multiple processing nodes.
+
+On each, run the script and specify a range of _epochs_ to go through where the script will attempt to download all relevant snapshots from Solana Foundation GCP buckets in order to gather all relevant slots.
+
+
+`create-cars.sh 100-200`
+
+`create-cars.sh 300-500`
+
+`create-cars.sh 600-620`
+
+There's some functions that send notifications to Slack and create metric files, you can comment it out if not relevant for you.
+## Manual CAR generation
+
+create-cars.sh only tries to find snapshots in the EU region GCP bucket.
+
+When script fails to find the required snapshots, you may try to do it manually following the steps below:
+
+```
+export EPOCH=500
+
+# try to find snapshots in other regions
+download_ledgers_gcp.py $EPOCH us
+# or download_ledgers_gcp.py $EPOCH ap
+
+# check the given snapshots contain all required slots
+/usr/local/bin/radiance car create2 --db snapshot1/rocksdb $EPOCH --db snapshots2/rocksdb --out="$CARDIR/epoch-$EPOCH.car" --check
+
+# generate car
+/usr/local/bin/radiance car create2 --db snapshot1/rocksdb $EPOCH --db snapshots2/rocksdb --out="$CARDIR/epoch-$EPOCH.car"
+
+```
+
+Then you can create a copy of `create-cars.sh` that skips the downloading and cargen steps and proceeds with the remaining tasks.
+
+## Costs
+
+Using B2 object storage service and the Hetzner node recommended above your costs:
+
+- servers: €500/month per processor node
+- object storage: 230TB will cost min 1300$/month - data includes CAR + split file + index storage
+
+## Dashboards
+
+create-cars.sh is creating .prom files in `/var/lib/node_exporter/`, so by having node exporter running and being scraped (Prometheus) you can monitor the car generation progress on Grafana .
+
+Import dashboard.yml in this directory into Grafana, update the variable with your node names (or use a query instead).
+
+## Gotchas
+
+1. First 8 epochs are missing bounds.txt file so you may need to do it manually.
+2. Around epoch 32 there was a change in `shred revision`, depending on what snapshot you use you may need to adjust `SHRED_REVISION_CHANGE_EPOCH` (currently set at 32, but could be 24)
diff --git a/docs/create-cars.sh b/docs/create-cars.sh
new file mode 100644
index 00000000..42b79e4a
--- /dev/null
+++ b/docs/create-cars.sh
@@ -0,0 +1,221 @@
+#!/bin/bash
+
+set -e
+shopt -s nullglob
+
+SNAPSHOT_FOLDER="{{ warehouse_processor_snapshot_folder }}"
+
+# find out the right number
+SHRED_REVISION_CHANGE_EPOCH="32"
+
+CARDIR="{{ warehouse_processor_car_folder }}"
+
+# add defaults?
+RANGE_START=$1
+
+RANGE_END=$2
+
+get_archive_file() {
+    find "$1" -type f \( -name "rocksdb.tar.zst" -o -name "rocksdb.tar.bz2" \)
+}
+
+cleanup_artifacts() {
+
+    cardirepoch="$1"
+    workdir="$2"
+    set -x
+    echo "cleaning up generated files"
+    rm -rf $cardirepoch/*index
+    rm -rf $cardirepoch/*gsfa.index.tar.bz2
+    rm -rf $cardirepoch/*.car
+    rm -rf $workdir/*
+    set +x
+}
+
+# remove rocksdb data
+cleanup_workfiles() {
+    workdir="$1"
+    prev_workdir="$2"
+
+    if [[ "${prev_workdir}" != "" ]]; then
+        rm -rf $prev_workdir/rocksdb*
+    fi
+    # only remove the archive, we might need prev snapshot on the next snapshot cargen
+    rm -rf $workdir/rocksdb.tar*
+}
+
+metric() {
+    EPOCH="$1"
+    STATUS="$2"
+    echo "cargen_status{epoch=\"$EPOCH\",status=\"$STATUS\"} 1" | sponge /var/lib/node_exporter/cargen_$EPOCH.prom
+}
+
+
+slack() {
+    msg="$1"
+    set -x
+    curl -XPOST "https://hooks.slack.com/services/{{ warehouse_slack_token }}" -H "Content-type: application/json" -d "{\"text\": \"$msg\"}"
+    set +x
+}
+
+find_ledgers_file_snapshot() {
+    epoch="$1"
+    ledgers_file="$SNAPSHOT_FOLDER/epoch-$epoch.ledgers"
+    set -x
+    if [ ! -f "$ledgers_file" ]; then
+        # echo "Ledgers file not found for epoch $epoch: $ledgers_file"
+        echo ""
+        return 0
+    fi
+
+    cat "$ledgers_file"
+    set +x
+}
+
+
+download_ledger() {
+    epoch="$1"
+    echo "pulling snapshot for $epoch..."
+    metric $EPOCH "downloading"
+    python3 download_ledgers_gcp.py $epoch eu
+}
+
+for EPOCH in $(seq $RANGE_START $RANGE_END); do
+
+    download_ledger $EPOCH
+    WORKDIR=`find_ledgers_file_snapshot $EPOCH | tail -n1`
+    SNAPSHOT=$(basename "$WORKDIR")
+
+    SHRED_REVISION_FLAG=""
+
+
+    # if test ! -f "$WORKDIR/epoch"; then
+    #     echo "no epoch file"
+    #     # find way to calculate epoch instead of skipping
+    #     # or manually create bounds file (only first 8 epochs)
+    #     # or manually run the cargen
+    #     continue
+    # fi
+    #
+    # EPOCH=`cat $WORKDIR/epoch | tr -d '\n'`
+
+    # epochs under SHRED_REVISION_CHANGE_EPOCH need different handling
+    if [[ "${EPOCH}" -lt "${SHRED_REVISION_CHANGE_EPOCH}" ]]; then
+        echo "setting shred-revision=1 for epoch under ${SHRED_REVISION_CHANGE_EPOCH}"
+        SHRED_REVISION_FLAG="--shred-revision=1"
+    fi
+
+    # SHRED_REVISION_CHANGE_EPOCH epoch needs special handling at a specific slot
+    if [[ "${EPOCH}" -eq "${SHRED_REVISION_CHANGE_EPOCH}" ]]; then
+        echo "need special handling for epoch ${SHRED_REVISION_CHANGE_EPOCH}"
+        SHRED_REVISION_FLAG="--shred-revision=1 --next-shred-revision-activation-slot=10367766"
+    fi
+
+    echo "Working on snapshot: $SNAPSHOT for epoch: $EPOCH"
+
+    # remove incomplete work
+    mkdir "$CARDIR/" || true
+    rm "$CARDIR/epoch-$EPOCH.car" || true
+
+    DBS=""
+
+    # The <epoch>.ledgers file is created by download_ledgers.py
+    # this will catch when previous snapshots are required from reading bounds.txt
+    echo "reading ledgers file for $EPOCH..."
+    while read -r line; do
+        DBS+=" --db=${line}/rocksdb";
+    done <<< "$(find_ledgers_file_snapshot $EPOCH)"
+
+    # use check mode to check if we need to add next snapshot
+    set -x
+    if ! /usr/local/bin/radiance car create2 "$EPOCH" $DBS --out="$CARDIR/epoch-$EPOCH.car" --check; then
+        # set +x
+
+        echo "pulling next snapshot..."
+        NEXT_SNAP=$((EPOCH + 1))
+        download_ledger "$NEXT_SNAP"
+
+        echo "reading ledgers file for $NEXT_SNAP..."
+        while read -r line; do
+            DBS+=" --db=${line}/rocksdb";
+        done <<< "$(find_ledgers_file_snapshot $NEXT_SNAP)"
+
+        # DBS="--db=$WORKDIR/rocksdb"
+    fi
+    # set +x
+
+    echo "Using databases: ${DBS}"
+
+    metric $EPOCH "cargen"
+
+    # create car file
+    set -x
+    /usr/local/bin/radiance car create2 "$EPOCH" $DBS --out="$CARDIR/epoch-$EPOCH.car" $SHRED_REVISION_FLAG
+    # set +x
+
+    # cleanup work files
+    echo "cleaning up work files"
+    set -x
+
+    # some epochs may need 2 snapshots
+    # so we need to keep at least 2 snapshots going back
+    PREV_WORKDIR_LEDGERS=$(find_ledgers_file_snapshot $((EPOCH-2)) | tail -n1)
+    if [ -n "$PREV_WORKDIR_LEDGERS" ]; then
+        cleanup_workfiles $WORKDIR $PREV_WORKDIR_CONTENT
+    else
+        cleanup_workfiles $WORKDIR
+    fi
+    # set +x
+
+    # check file exists and non empty
+    if ! test -s "$CARDIR/epoch-$EPOCH.car" ; then
+        echo "car file is empty/non existant, skipping next steps"
+        continue;
+    fi
+
+    if ! test -f "$CARDIR/$EPOCH.cid"; then
+        echo "no root cid, generation probably failed"
+        continue;
+    fi
+
+    # Create the cardir epoch dir
+    mkdir -p "$CARDIR/$EPOCH" || true
+    # move cid/recap/slots files
+    mv $CARDIR/$EPOCH.cid $CARDIR/$EPOCH/epoch-$EPOCH.cid # IMP RENAME THE CID
+    mv $CARDIR/$EPOCH.* $CARDIR/$EPOCH/
+    /usr/local/bin/radiance version >$CARDIR/$EPOCH/radiance.version.txt
+
+    # split into 30gb files
+    echo "splitting car file..."
+    metric $EPOCH "splitting"
+    /usr/local/bin/split-epoch.sh $EPOCH $CARDIR/epoch-$EPOCH.car $CARDIR
+
+    # creates the indexes and the appropriate car files
+    # since split-epoch moves the epoch file, we need to also use a different path here for the epoch file
+    echo "creating indexes..."
+    metric $EPOCH "indexing"
+    /usr/local/bin/index-epoch.sh all $EPOCH $CARDIR/$EPOCH/epoch-$EPOCH.car $CARDIR
+
+    # upload
+    echo "uploading car file..."
+    metric $EPOCH "uploading"
+    B2_ACCOUNT_INFO=/etc/b2/filecoin_account_info /usr/local/bin/upload-epoch.sh $EPOCH $CARDIR
+
+    # clean up
+    echo "cleaning up artifacts..."
+    cleanup_artifacts "$CARDIR/$EPOCH" "$WORKDIR"
+
+    slack "{{ inventory_hostname }} finished generating epoch $EPOCH car files"
+
+    metric $EPOCH "complete"
+
+    # done
+    touch "$WORKDIR/.cargen"
+
+    # reached end of range
+    if [[ "${EPOCH}" -eq "${RANGE_END}" ]]; then
+        echo "reached end of the given range, please restart job with new parameters to resume cargen"
+        exit 0
+    fi
+
+done
diff --git a/docs/dashboard.json b/docs/dashboard.json
new file mode 100644
index 00000000..ebd061f1
--- /dev/null
+++ b/docs/dashboard.json
@@ -0,0 +1,1194 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1652,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum by (nodename)(irate(\n    node_cpu_seconds_total{mode='user'}\n     * on(instance) group_left(nodename) (node_uname_info{nodename=~\"warehouse-processor.*\"})\n[5m])) * 100 > 50",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "CPU time (>50)",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "cargen": {
+                  "color": "blue",
+                  "index": 1
+                },
+                "downloading": {
+                  "color": "dark-blue",
+                  "index": 0
+                },
+                "indexing": {
+                  "color": "super-light-blue",
+                  "index": 3
+                },
+                "splitting": {
+                  "color": "light-blue",
+                  "index": 2
+                },
+                "uploading": {
+                  "color": "light-green",
+                  "index": 4
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 14,
+        "w": 12,
+        "x": 12,
+        "y": 0
+      },
+      "id": 8,
+      "maxPerRow": 3,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "auto",
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.3",
+      "repeat": "processor_nodes",
+      "repeatDirection": "v",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "cargen_status{status!~\"complete|started|failed\"}\n* on(instance) group_left(nodename) (node_uname_info{nodename=~\"$processor_nodes\"})",
+          "format": "table",
+          "instant": false,
+          "interval": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "cargen process status $processor_nodes",
+      "transformations": [
+        {
+          "id": "groupingToMatrix",
+          "options": {
+            "columnField": "epoch",
+            "emptyValue": "null",
+            "rowField": "Time",
+            "valueField": "status"
+          }
+        },
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "destinationType": "time",
+                "targetField": "Time\\epoch"
+              }
+            ],
+            "fields": {}
+          }
+        }
+      ],
+      "type": "state-timeline"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "Basic network info per interface",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 40,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "normal"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "bps"
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_bytes_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_drop_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6ED0E0",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_errs_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Recv_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#CCA300",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_bytes_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_drop_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#6ED0E0",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#E0F9D7",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_errs_eth2"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "Trans_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#CCA300",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_drop_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#99440A",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#967302",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_errs_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "recv_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_bytes_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#7EB26D",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_bytes_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#0A50A1",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_drop_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#99440A",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_drop_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#967302",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_errs_eth0"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#BF1B00",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "trans_errs_lo"
+            },
+            "properties": [
+              {
+                "id": "color",
+                "value": {
+                  "fixedColor": "#890F02",
+                  "mode": "fixed"
+                }
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "/.*trans.*/"
+            },
+            "properties": [
+              {
+                "id": "custom.transform",
+                "value": "negative-Y"
+              }
+            ]
+          },
+            "properties": [
+              {
+                "id": "custom.hideFrom",
+                "value": {
+                  "legend": false,
+                  "tooltip": false,
+                  "viz": true
+                }
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "id": 2,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Name",
+          "sortDesc": false
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(\n    irate(\n        node_network_receive_bytes_total\n        * on(instance) group_left(nodename) (node_uname_info{nodename=~\"warehouse-processor.*\"}\n    )[1m])\n) by (nodename) * 8\n-\nsum(\n    irate(\n        node_network_transmit_bytes_total\n        * on(instance) group_left(nodename) (node_uname_info{nodename=~\"warehouse-processor.*\"}\n    )[1m])\n) by (nodename) * 8",
+          "format": "time_series",
+          "interval": "",
+          "intervalFactor": 10,
+          "legendFormat": "{{  nodename }}",
+          "range": true,
+          "refId": "A",
+          "step": 240
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum(\n    irate(\n        node_network_transmit_bytes_total\n        * on(instance) group_left(nodename) (node_uname_info{nodename=~\"warehouse-processor.*\"}\n    )[1m])\n) by (nodename) \n* 8 > 1000 * 1000 * 10",
+          "format": "time_series",
+          "hide": true,
+          "intervalFactor": 10,
+          "legendFormat": "trans {{nodename }}",
+          "range": true,
+          "refId": "B",
+          "step": 240
+        }
+      ],
+      "title": "Network Traffic Basic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "description": "Disk space used of all filesystems mounted",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 17,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "links": [],
+          "mappings": [],
+          "max": 100,
+          "min": 30,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "percent"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 12,
+        "w": 12,
+        "x": 0,
+        "y": 24
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "right",
+          "showLegend": true,
+          "sortBy": "Last *",
+          "sortDesc": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "pluginVersion": "10.0.2",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "100 - (\n    (node_filesystem_avail_bytes{mountpoint=~\"/workdir.*|/storage.*|/\"} * 100) / node_filesystem_size_bytes{mountpoint=~\"/workdir.*|/storage.*|/\"}\n)\n* on(instance) group_left(nodename) (node_uname_info{nodename=~\"warehouse.*\"})\n> 30",
+          "format": "time_series",
+          "intervalFactor": 2,
+          "legendFormat": "{{mountpoint}} {{ nodename }}",
+          "range": true,
+          "refId": "A",
+          "step": 240
+        }
+      ],
+      "title": "Disk Space Used Basic",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "cargen": {
+                  "color": "blue",
+                  "index": 1
+                },
+                "downloading": {
+                  "color": "dark-blue",
+                  "index": 0
+                },
+                "indexing": {
+                  "color": "super-light-blue",
+                  "index": 3
+                },
+                "splitting": {
+                  "color": "light-blue",
+                  "index": 2
+                },
+                "uploading": {
+                  "color": "light-green",
+                  "index": 4
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 13,
+        "w": 12,
+        "x": 12,
+        "y": 70
+      },
+      "id": 11,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": false
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "auto",
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "10.0.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "cargen_status{status!~\"complete|started|failed\"} \n * on(instance) group_left(nodename) (node_uname_info)\n ",
+          "format": "table",
+          "instant": false,
+          "interval": "",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "instance process status",
+      "transformations": [
+        {
+          "id": "groupingToMatrix",
+          "options": {
+            "columnField": "nodename",
+            "emptyValue": "null",
+            "rowField": "Time",
+            "valueField": "status"
+          }
+        },
+        {
+          "id": "convertFieldType",
+          "options": {
+            "conversions": [
+              {
+                "destinationType": "time",
+                "targetField": "Time\\nodename"
+              }
+            ],
+            "fields": {}
+          }
+        }
+      ],
+      "type": "state-timeline"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 24,
+        "w": 3,
+        "x": 12,
+        "y": 83
+      },
+      "id": 10,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "10.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "sum(cargen_status{status=\"failed\"}) by (epoch)",
+          "format": "table",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "failed",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "instance": true,
+              "job": true
+            },
+            "indexByName": {},
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "align": "auto",
+            "cellOptions": {
+              "type": "auto"
+            },
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 24,
+        "w": 5,
+        "x": 15,
+        "y": 83
+      },
+      "id": 9,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "10.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "cargen_status{status=\"started\"}",
+          "format": "table",
+          "instant": true,
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "started",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "instance": true,
+              "job": true,
+              "status": true
+            },
+            "indexByName": {},
+            "renameByName": {}
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": {
+          "type": "prometheus",
+          "uid": "PBFA97CFB590B2093"
+        },
+        "definition": "label_values(node_uname_info{nodename=~\"warehouse-processor.*\"},nodename)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "processor_nodes",
+        "options": [],
+        "query": {
+          "query": "label_values(node_uname_info{nodename=~\"warehouse-processor.*\"},nodename)",
+          "refId": "PrometheusVariableQueryEditor-VariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Car generation",
+  "uid": "e81e06b7-649a-469a-bec3-b265f11eb73d",
+  "version": 43,
+  "weekStart": ""
+}
diff --git a/docs/download_ledgers_gcp.py b/docs/download_ledgers_gcp.py
new file mode 100644
index 00000000..224608ed
--- /dev/null
+++ b/docs/download_ledgers_gcp.py
@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import re
+import natsort
+import bisect
+import subprocess
+import queue
+import threading
+import concurrent.futures
+import contextvars
+import time
+import logging
+import shutil
+
+from pathlib import Path
+
+def sync_bounds(bucket, bounds_dir):
+    """
+    Syncs bounds for all epochs on gcp
+    """
+    exclude_pattern = '.*hourly.*|.*tar.*|.*rocksdb.*|.*sst.*|.*gz|.*csv|.*zst'
+    command = f"gsutil -m rsync -r -x '{exclude_pattern}'  gs://{bucket}/ {bounds_dir}"
+    logging.debug(command)
+    subprocess.run(command, shell=True, check=True)
+
+def load_epoch_ranges(dir):
+    """
+    Loads the epoch ranges from a directory containing epoch bounds
+
+    Returns a tuple of an array of start slots for the epochs contained in the dir and the end slots
+    """
+    snapshot_count = 0
+
+    # Starts ccontains the starting slot of each ledger
+    starts = []
+    # Ranges contains the epoch details of eacch snapshot, indexed by its starting slot
+    ranges = {}
+    for subdir, dirs, files in natsort.natsorted(os.walk(dir)):
+        for file in files:
+            if file != "bounds.txt":
+                continue
+
+            snapshot_count = snapshot_count + 1
+            bounds_file = os.path.join(subdir, file)
+
+            f = open(bounds_file).readlines()[0]
+            # earlier Solana releases have a shorter description without number of slots
+            matches = re.search(r"Ledger has data for (?:[0-9]+ )?slots ([0-9]+) to ([0-9]+)", f)
+            if matches == None:
+                matches = re.search(r"Ledger contains data from slots ([0-9]+) to ([0-9]+)", f)
+                if matches == None:
+                    logging.error("Could not parse bounds file: " + bounds_file)
+                    continue
+
+            start = int(matches.group(1))
+            end = int(matches.group(2))
+
+            ranges[start] = {
+                    "dir": os.path.basename(os.path.normpath(subdir)),
+                    "subdir": subdir,
+                    "start": start,
+                    "end": end
+            }
+
+            starts.append(start)
+    return (starts, ranges)
+
+def find_snapshots_for_epochs(first_epoch, last_epoch, starts, ranges):
+    """
+    Finds complete set of snapshots for a given epoch.
+
+    Returns a tuple cconsisting of a list of epochs for which complete snapshots could be found and a list of epochs for which snapshots are missing
+    """
+    epochs = []
+    missing_epochs = []
+    for i in range(first_epoch, last_epoch+1):
+        epoch_start = i * 432000
+        epoch_end = (i+1)*432000-1
+
+        logging.debug(str(epoch_start) + " " + str(epoch_end))
+
+        if i != 0:
+            left = bisect.bisect_left(starts, epoch_start)
+            if left == 0:
+                missing_epochs.append(i)
+                logging.debug("Couldn't find the right one ", left)
+                continue
+            else:
+                left = left-1
+        else:
+            left = 0
+
+        start = starts[left]
+        snap = ranges[start]
+        end = ranges[start]["end"]
+
+        snapshots = []
+
+        if start > epoch_start:
+            logging.debug(f"Couldn't find the correct epoch start epoch={i} end={end} start={epoch_start}")
+            missing_epochs.append(i)
+            continue
+        if end < epoch_start:
+            logging.debug(f"Couldn't find the correct snapshot, end is less than start epoch={i} end={end} start={epoch_start}")
+            missing_epochs.append(i)
+            continue
+
+        snapshots.append(snap)
+        while end < epoch_end:
+            if left >= len(starts):
+                logging.debug("Couldn't find the correct ending snapshot, need more data")
+                continue
+
+            start = starts[left]
+            snap = ranges[start]
+            end = ranges[start]["end"]
+            snapshots.append(snap)
+            left += 1
+
+        epochs.append({
+            "epoch": i,
+            "start": epoch_start,
+            "end": epoch_end,
+            "snapshots": snapshots
+        })
+    return (epochs, missing_epochs)
+
+def find_snapshots_for_epoch(epoch, start, ranges):
+    return find_snapshots_for_epochs(epoch, epoch, start, ranges)
+
+def sync_snapshots_from_b2(epochs, bucket, downloads_dir):
+    """
+    Syncs snapshots from gcp for a specific set of epochs
+    """
+    for epoch in epochs:
+        logging.info(f"Getting snapshot for {epoch}")
+        for s in range(len(epoch['snapshots'])):
+            snapshot = epoch['snapshots'][s]
+            try:
+                command = ['s5cmd', '--no-sign-request', '--endpoint-url', 'https://storage.googleapis.com', 'sync', '--exclude', '*snapshot-*', f"s3://{bucket}/{snapshot['dir']}/*", f"{downloads_dir}/{snapshot['dir']}/"]
+                logging.debug(command)
+                subprocess.run(command, check=True)
+            except Exception as e:
+                logging.error("error:", e, snapshot)
+
+
+# Locks to allow parallel execution of snapshot extraction
+# TODO: remove threading stuff altogether
+processing_snapshots = {}
+snapshot_lock = threading.Lock()
+
+
+def move_rocksdb_snapshot(dir):
+    """ makes sure rocksdb is moved from any subdirectory to the base snapshot dir"""
+    for root, _, _ in os.walk(dir):
+
+        rocksdb_dir = os.path.join(root, 'rocksdb')
+        if os.path.isdir(rocksdb_dir):
+            new_location = os.path.join(dir, 'rocksdb')
+            logging.info("Moving rocksdb from %s to %s" % (rocksdb_dir, new_location))
+            subprocess.run(['mv', rocksdb_dir, new_location])
+    logging.info("RocksDB directories moved successfully.")
+    #
+    # doesn't seem to work
+    # does it cause files like CURRENT to be missing? silent fail?
+    #
+    # for root, _, files in os.walk(dir):
+    #     if 'rocksdb' in files and 'rocksdb' not in root.split(os.path.sep):
+    #         rocksdb_path = os.path.join(root, 'rocksdb')
+    #         target_path = os.path.join(directory, 'rocksdb')
+    #
+    #         if not os.path.exists(target_path):
+    #             shutil.move(rocksdb_path, target_path)
+    #             print(f"Moved 'rocksdb' to {target_path}")
+    #             return
+    #
+    #         return
+
+def rocksdb_is_extracted(snapshot_path):
+    """
+    determine if snapshot extracted by checking flag file and one of the decompressed files
+    """
+    return os.path.isfile(snapshot_path + '/.extracted') and os.path.isfile(snapshot_path + '/rocksdb/CURRENT')
+
+def pigz_archived(snapshot_path):
+    """
+    if pigz was used to compress rocksdb, there'll be a 1000 .gz files
+    instead of one tar archive - check CURRENT.gz exists to determine it
+    """
+    return os.path.isfile(snapshot_path + '/rocksdb/CURRENT.gz')
+
+
+def extract_snapshot(snapshot, downloads_dir):
+    """
+    Extracts a snapshot rocksdb file. Will only extract the snapshot ones.
+    Designed to be able to run in parallel with multiple calls for the same snapshot dir not creating a new exraction.
+    """
+    logging.debug("Called extract snapshot with snapshot: " + snapshot['dir'])
+    snapshot_lock.acquire()
+    if snapshot['dir'] not in processing_snapshots:
+        # check if rocksdb has already been extracted
+        # if not os.path.isdir('download/'+snapshot['dir']+'/rocksdb'):
+        snapshot_path = downloads_dir+'/'+snapshot['dir']
+
+        if rocksdb_is_extracted(snapshot_path):
+            processing_snapshots[snapshot['dir']] = 'completed'
+            Path(snapshot_path + '/.extracted').touch()
+            Path(snapshot_path + '/.rocksdb').touch()
+            snapshot_lock.release()
+            logging.info("snapshot already extracted: " + snapshot['dir'])
+            return 0
+
+        extract_filename = ''
+        decompress_tool = False
+        if os.path.isfile(snapshot_path + '/rocksdb.tar.bz2'):
+            extract_filename = 'rocksdb.tar.bz2'
+            decompress_tool = "lbzip2"
+        elif os.path.isfile(snapshot_path + '/rocksdb.tar.gz'):
+            extract_filename = 'rocksdb.tar.gz'
+        elif os.path.isfile(snapshot_path + '/rocksdb.tar.zst'):
+            extract_filename = 'rocksdb.tar.zst'
+            decompress_tool = "pzstd"
+        elif pigz_archived(snapshot_path):
+            # there's no single archive file, just bypass check below
+            extract_filename = "n/a"
+
+        if extract_filename != '':
+            processing_snapshots[snapshot['dir']] = 'processing'
+            snapshot_lock.release()
+
+            # disabled - does not seem to be required
+            # check CURRENT file exists in the tar file before extraction, it can be in a path
+            # command = f"tar -tf {snapshot_path}/{extract_filename} | grep -qE 'CURRENT|MANIFEST'"
+            # logging.debug(command)
+            # subprocess.run(command, shell=True, check=True)
+
+            command = ['tar', '-xf', snapshot_path+'/'+extract_filename, '-C', snapshot_path]
+
+            if decompress_tool:
+                command = ['tar', '-I', decompress_tool, '-xf', snapshot_path+'/'+extract_filename, '-C', snapshot_path]
+
+            if pigz_archived(snapshot_path):
+                command = f"pigz -d {snapshot_path}/rocksdb/*.gz"
+                logging.debug(command)
+                subprocess.run(command, shell=True, check=True)
+            else:
+                logging.debug(command)
+                subprocess.run(command, check=True)
+
+            move_rocksdb_snapshot(snapshot_path)
+
+            Path(snapshot_path + '/.extracted').touch()
+            Path(snapshot_path + '/.rocksdb').touch()
+
+            snapshot_lock.acquire()
+            processing_snapshots[snapshot['dir']] = 'completed'
+            snapshot_lock.release()
+        else:
+            snapshot_lock.release()
+            logging.error('could not find rocksdb folder nor file')
+            return 1
+    else:
+        snapshot_lock.release()
+
+        # wait for snapshot to finish
+        while True:
+            time.sleep(5)
+            logging.debug('waiting for other thread to complete extraction')
+            if processing_snapshots[snapshot['dir']] == 'completed':
+                break
+
+
+    logging.info('finished snapshot extraction')
+    return 0
+
+def process_epoch(epoch, downloads_dir):
+    """
+    Processes a single epoch, running extraction on each of the epoch's snapshots
+    In the future we could add cargen here too I guess?
+    """
+    download_paths = []
+
+    logging.debug("Processing snapshots for epoch:" + str(epoch['epoch']))
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        future_to_snapshot = {executor.submit(extract_snapshot, snapshot, downloads_dir): snapshot for snapshot in epoch['snapshots']}
+        for future in concurrent.futures.as_completed(future_to_snapshot):
+            snapshot = future_to_snapshot[future]
+            try:
+                result = future.result()
+                if result != 0:
+                    logging.error('snapshot execution error: '+ str(result))
+                    return result
+            except Exception as exc:
+                logging.exception('snapshot exception' + str(exc))
+                return 1
+            else:
+                download_paths.append(downloads_dir + '/' + snapshot['dir'])
+                logging.info('snapshot extracted for epoch ' + str(epoch['epoch']) + ", snapshot " + snapshot['dir'])
+
+    with open(downloads_dir+'/epoch-'+ str(epoch['epoch']) + '.ledgers', 'w') as f:
+        # @TODO fix duplicate entry in the set, not sure why it's there
+        unique_paths = []
+        for path in download_paths:
+            if path not in unique_paths:
+                unique_paths.append(path)
+
+        for path in unique_paths:
+            f.write(path + '\n')
+
+    logging.info('finished processing processing snapshots for epoch: ' + str(epoch['epoch']) + ' found: ' + str(len(download_paths)) + ' paths. Writing to file: ' + downloads_dir+'/epoch-'+ str(epoch['epoch'])+ '.ledgers')
+    return 0
+
+logging.basicConfig(level=logging.DEBUG)
+
+def main():
+    downloads_dir = '/storage/gcp-sg-workdir'
+    bucket = 'mainnet-beta-ledger-asia-sg1'
+
+    if len(sys.argv) < 2:
+        print('missing epoch argument, run with ', sys.argv[0], ' <start_epoch>[-<end_epoch>]')
+        return 1
+
+    epoch_arg = sys.argv[1].split("-", 1)
+    start_epoch = int(epoch_arg[0])
+    region = sys.argv[2]
+    if len(epoch_arg) > 1:
+        end_epoch = int(epoch_arg[1])
+    else:
+        end_epoch = start_epoch
+
+    if region == "ap":
+        bucket = 'mainnet-beta-ledger-asia-sg1'
+        downloads_dir = '/storage/gcp-sg-workdir'
+    if region == "eu":
+        bucket = 'mainnet-beta-ledger-europe-fr2'
+        downloads_dir = '/storage/gcp-fra-workdir'
+    if region == "us":
+        bucket = 'mainnet-beta-ledger-us-ny5'
+        downloads_dir = '/storage/gcp-nyc-workdir'
+
+    sync_bounds(bucket, downloads_dir)
+    starts, ranges  = load_epoch_ranges(downloads_dir)
+
+    logging.debug("Loaded " + str(len(starts)) + " epochs")
+
+    epochs, missing_epochs = find_snapshots_for_epochs(start_epoch, end_epoch, starts, ranges)
+
+    logging.debug("Found " + str(len(epochs)) + " epochs" + " missing " + str(len(missing_epochs)))
+
+    sync_snapshots_from_b2(epochs, bucket, downloads_dir)
+
+    failed_epochs = []
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        future_to_epoch = {executor.submit(process_epoch, epoch, downloads_dir): epoch for epoch in epochs}
+        for future in concurrent.futures.as_completed(future_to_epoch):
+            epoch = future_to_epoch[future]
+            try:
+                result = future.result()
+                if result != 0:
+                    failed_epochs.append(str(epoch['epoch']))
+            except Exception as exc:
+                logging.exception("exception occurred: "+str(exc))
+                failed_epochs.append(str(epoch['epoch']))
+            else:
+                logging.info("epoch succeeded: " + str(epoch['epoch']))
+
+    if len(missing_epochs) > 0:
+        logging.error('missing epochs: ' + ','.join(missing_epochs))
+
+    if len(failed_epochs) > 0:
+        logging.error('failed epochs: ' + ','.join(failed_epochs))
+
+
+    return 0
+
+if __name__ == "__main__":
+    ret = main()
+    sys.exit(ret)
diff --git a/docs/index-epoch.sh b/docs/index-epoch.sh
new file mode 100644
index 00000000..d4bf8b2a
--- /dev/null
+++ b/docs/index-epoch.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+
+if [ "$#" -ne 4 ]; then
+    echo "Run $0 <index type> <epoch> <carfile> <output_folder>"
+    exit 4
+fi
+
+mkdir -p "${4}/tmp"
+
+/usr/local/bin/faithful-cli version >"${4}/${2}/faithful.version.txt"
+
+if [[ "${1}" != "gsfa" ]]; then
+    # Generate the indexes
+    /usr/local/bin/faithful-cli index "${1}" \
+        --network=mainnet \
+        --flush-every=1000000 \
+        --epoch=${EPOCH} \
+        --tmp-dir "${4}/tmp" \
+        "${3}" \
+        "${4}/${2}/"
+fi
+
+if [[ "${1}" == "gsfa" ]] || [[ "${1}" == "all" ]]; then
+    # Create gsfa index
+    /usr/local/bin/faithful-cli index gsfa "${3}" "${4}/${2}/"
+fi
+
+# create a tar file from the gsfa index file
+cd "${4}/${2}" && \
+ tar -cf "${4}/${2}/epoch-${2}-gsfa.index.tar.zstd" -I "zstd -T8" ${4}/${2}/epoch-${2}*gsfa.indexdir && \
+ cd - # dont put this string in quotes as we need the shell globbing
+
+# Split CAR into 30gb split files, that's the max supported by Filecoin
+/usr/local/bin/filecoin-data-prep fil-data-prep \
+    --size 30000000000 \
+    --output "${4}/${2}/sp-index-epoch-${2}" \
+    --metadata "${4}/${2}/index.csv" \
+    ${4}/${2}/*.index ${4}/${2}/*.index.tar.zstd # need to include shell globbing
diff --git a/docs/split-epoch.sh b/docs/split-epoch.sh
new file mode 100644
index 00000000..46765aba
--- /dev/null
+++ b/docs/split-epoch.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ "$#" -ne 3 ]; then
+    echo "Run $0 <epoch> <carfile> <output_folder>"
+    exit 1
+fi
+
+# Create the epoch folder if it doesn't exist
+mkdir -p "${3}/${1}"
+
+# Create 30 gib pieces
+/usr/local/bin/filecoin-data-prep -v >"${3}/${1}/dataprep.version.txt"
+/usr/local/bin/filecoin-data-prep split-and-commp --size {{ warehouse_processor_car_size }} --output "${3}/${1}/sp-epoch-${1}" --metadata "${3}/${1}/metadata.csv" "${2}"
+
+# @TODO use stream-commp to validate the hashes in the metadata.csv
+
+mv "${2}" "${3}/${1}/"
diff --git a/docs/upload-epoch.sh b/docs/upload-epoch.sh
new file mode 100644
index 00000000..09cf163f
--- /dev/null
+++ b/docs/upload-epoch.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Script to upload an epoch folder to  b2
+if [ "$#" -ne 2 ]; then
+    echo "Run $0 <epoch> <folder>"
+    exit 1
+fi
+
+if [[ ! -d "${2}/${1}" ]]; then
+  echo "Folder ${2}/${1} doesn't exist"
+  exit 1
+fi
+
+# unless doing full cargen or at least splitting, this file won't be generated
+# if [[ ! -f "${2}/${1}/metadata.csv" ]]; then
+#   echo "metadata.yaml must exist in the target folder ${2}/${1}/metadata.csv"
+#   exit 1
+# fi
+
+# fix a bug in the CID naming
+mv "${2}/${1}/${1}.cid" "${2}/${1}/epoch-${1}.cid" || true
+
+echo "uploading to b2"
+set -x
+b2 sync \
+    --includeRegex 'epoch-.*\.index' \
+    --includeRegex 'epoch-.*.index.tar.zstd' \
+    --includeRegex 'metadata.yaml' \
+    --includeRegex 'index.csv' \
+    --includeRegex 'metadata.csv' \
+    --includeRegex 'sp-*.car' \
+    --includeRegex 'epoch-.*\.car' \
+    --includeRegex 'epoch-.*\.cid' \
+    --includeRegex '.*\.slots\.txt' \
+    --includeRegex '.*\.recap\.txt' \
+    --includeRegex '.*\.recap\.yaml' \
+    --includeRegex '.*\.version\.txt' \
+    --excludeRegex '.*' "${2}/${1}/" \
+    "b2://{{ warehouse_processor_bucket }}/${1}/"
+
+b2 ls "{{ warehouse_processor_bucket }}" ${1}