diff --git a/benchmarks/llm/recipes/full_finetune_distributed.py b/benchmarks/llm/recipes/full_finetune_distributed.py index 0be37a9e..73ac9ca4 100755 --- a/benchmarks/llm/recipes/full_finetune_distributed.py +++ b/benchmarks/llm/recipes/full_finetune_distributed.py @@ -1,3 +1,15 @@ +#!/usr/bin/env python3 + +# As of November 2024, the development of torchrun is very rapid. +# This is the recipe based on torchrun recipe git commit e137afe (post release 0.3.1) +# https://github.com/pytorch/torchtune/blob/7bfb3336446f0d874ab5d4595249839b735b7076/recipes/lora_finetune_distributed.py + +# Torchtune 0.2.1 recipe with device instrumenation (c) Mila +# https://github.com/mila-iqia/milabench/blob/a60a3aae21e87e46bcce403620a3f56c12878554/benchmarks/llm/recipes/full_finetune_distributed.py + +# The instrumentation edits (c) AMD + + # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # diff --git a/config/base.yaml b/config/base.yaml index 9e470fa3..f61397b3 100644 --- a/config/base.yaml +++ b/config/base.yaml @@ -652,7 +652,7 @@ llm-full-mp-gpus: argv: #"{milabench_code}/recipes/full_finetune_distributed.py": true - tuneworkaroundrecipes.full_finetune_distributed: true + tuneworkaroundrecipes/full_finetune_distributed.py: true --config: "{milabench_code}/configs/llama3_70B_full.yaml" epochs=1: true output_dir={milabench_extra}/output: true