ray_cluster_config.yaml

## Configuration to run a DiBB cluster


### USAGE: ###
# - Download this file to your local machine (make sure it can reach the cluster)
# - Replace all placeholders below (they look `%LIKE_THIS%`)
# - Run `ray up ray_cluster_config.yaml` from your local machine
# - The cluster is now set up and ready to use
# - Shut down the cluster by calling `ray down`, again from your local machine
##############

# For further customization:
# https://github.com/ray-project/ray/blob/master/python/ray/autoscaler/local/example-full.yaml
# Full reference: https://docs.ray.io/en/latest/cluster/config.html#examples

# Commands to start the Ray cluster on the head node
head_start_ray_commands:
    # just `ray`, or maybe `cd exp_dir && python3 -m pipenv run ray`
    - %RAY_COMMAND% stop
    - >-
        ulimit -c unlimited &&
        %RAY_COMMAND% start -v --head
        --port=6379
        --resources='{"DedicatedMachine":1}'
        --autoscaling-config=~/ray_bootstrap_config.yaml

# Commands to start ray on the worker nodes
worker_start_ray_commands:
    - %RAY_COMMAND% stop
    - >-
        %RAY_COMMAND% start -v
        --address=%IP_OF_HEAD_NODE%:6379
        --resources='{"DedicatedMachine":1}'

cluster_name: dibb_cluster
upscaling_speed: 1.0
idle_timeout_minutes: 5
provider:
    type: local
    head_ip: %IP_OF_HEAD_NODE%
    worker_ips:
        - %IP_OF_WORKER_NODE_1%
        - %IP_OF_WORKER_NODE_2%
        - %IP_OF_WORKER_NODE_...%
        - %IP_OF_WORKER_NODE_N%
auth:
    ssh_user: %USER_NAME_FOR_ALL_MACHINES%
    ssh_private_key: %PATH_TO_LOCAL_KEY_TO_CONNECT_TO_ALL_MACHINES%

file_mounts: {} # {"path_on_server" : "local_path",}
cluster_synced_files: []
file_mounts_sync_continuously: False
rsync_exclude:
    - "**/.git"
    - "**/.git/**"
rsync_filter:
    - ".gitignore"

### HOW TO PREPARE THE CLUSTER MACHINES ###
# Ray's "setup commands" support automatic workers setup
# Reference: https://docs.ray.io/en/latest/cluster/config.html#setup-commands
# Check out the following: [initialization_commands, setup_commands, head_setup_commands, worker_setup_commands]
#
# Here is what we used, for a MAAS installation using Ubuntu + pipenv
# (user `ubuntu` is pre-set with password-less `sudo` rights)
# (we only ran this once, then commented out these lines for faster cluster initialization)
#
# setup_commands:
#     - locale-gen en_US.UTF-8
#     - dpkg-reconfigure --frontend noninteractive locales
#     - apt update && apt install -y python3-pip build-essential python-dev python-setuptools
#     - python3 -m pip install pipenv
#     - cd <project_folder> && python3 -m pipenv sync
###########################################