diff --git a/.github/workflows/fast_tests.yml b/.github/workflows/fast_tests.yml index 7dde80875b..31de2b8408 100644 --- a/.github/workflows/fast_tests.yml +++ b/.github/workflows/fast_tests.yml @@ -24,10 +24,10 @@ jobs: start-runner: name: Start self-hosted EC2 runner needs: authorize - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_REGION: us-east-1 - EC2_AMI_ID: ami-0a82d7d7ad5d25f56 + EC2_AMI_ID: ami-0a2179742e502fdfe EC2_INSTANCE_TYPE: dl1.24xlarge EC2_SUBNET_ID: subnet-b7533b96 EC2_SECURITY_GROUP: sg-08af7938042271373 @@ -77,7 +77,7 @@ jobs: ref: ${{ github.event.pull_request.merge_commit_sha }} - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -89,7 +89,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/fast_tests.sh diffusers: name: Run tests for optimum.habana.diffusers @@ -113,7 +113,7 @@ jobs: ref: ${{ github.event.pull_request.merge_commit_sha }} - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -125,7 +125,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/fast_tests_diffusers.sh stop-runner: name: Stop self-hosted EC2 runner @@ -134,7 +134,7 @@ jobs: - start-runner # required to get output from the start-runner job - transformers # required to wait for the tests to be finished - diffusers # required to wait for the tests to be finished - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_REGION: us-east-1 if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs diff --git a/.github/workflows/slow_tests.yml b/.github/workflows/slow_tests.yml index 62ea2743cd..5e18f2460e 100644 --- a/.github/workflows/slow_tests.yml +++ b/.github/workflows/slow_tests.yml @@ -12,10 +12,10 @@ concurrency: jobs: start-runner: name: Start self-hosted EC2 runner - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_REGION: us-west-2 - EC2_AMI_ID: ami-01b277257cd28a061 + EC2_AMI_ID: ami-0961e95b539f72c46 EC2_INSTANCE_TYPE: dl1.24xlarge EC2_SUBNET_ID: subnet-452c913d EC2_SECURITY_GROUP: sg-0894f4f70dd6bd778 @@ -55,7 +55,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -67,7 +67,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/example_diff_tests.sh stable-diffusion: name: Test Stable Diffusion @@ -83,7 +83,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -95,7 +95,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_diffusers.sh deepspeed: name: Test DeepSpeed models @@ -112,7 +112,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -124,7 +124,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_deepspeed.sh multi-card: name: Test multi-card models @@ -141,7 +141,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -153,7 +153,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_8x.sh single-card: name: Test single-card models @@ -171,7 +171,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -183,7 +183,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_1x.sh albert-xxl-single-card: name: Test single-card ALBERT XXL @@ -204,7 +204,7 @@ jobs: - name: Pull image if: github.event.schedule == '0 21 * * 6' run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run test if: github.event.schedule == '0 21 * * 6' run: | @@ -217,7 +217,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/albert_xxl_1x.sh - name: Warning if: github.event.schedule != '0 21 * * 6' @@ -240,7 +240,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -252,7 +252,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ make slow_tests_text_generation_example TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} stop-runner: name: Stop self-hosted EC2 runner @@ -264,7 +264,7 @@ jobs: - single-card - albert-xxl-single-card - text-generation - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: AWS_REGION: us-west-2 if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs diff --git a/.github/workflows/slow_tests_gaudi2.yml b/.github/workflows/slow_tests_gaudi2.yml index d1f6b15b89..4d1a5a2631 100644 --- a/.github/workflows/slow_tests_gaudi2.yml +++ b/.github/workflows/slow_tests_gaudi2.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -30,7 +30,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_diffusers.sh deepspeed: name: Test DeepSpeed models @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -56,7 +56,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_deepspeed.sh multi-card: name: Test multi-card models @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -82,7 +82,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_8x.sh single-card: name: Test single-card models @@ -96,7 +96,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -110,7 +110,7 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ /bin/bash tests/ci/slow_tests_1x.sh text-generation: name: Test text-generation example @@ -125,7 +125,7 @@ jobs: uses: actions/checkout@v2 - name: Pull image run: | - docker pull vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest + docker pull vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest - name: Run tests run: | docker run \ @@ -138,5 +138,5 @@ jobs: --cap-add=sys_nice \ --net=host \ --ipc=host \ - vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest \ + vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest \ make slow_tests_text_generation_example TOKEN=${{ secrets.TEXT_GENERATION_CI_HUB_TOKEN }} diff --git a/Makefile b/Makefile index 2a45b8bfd4..ba40ca4b93 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ slow_tests_8x: test_installs # Run DeepSpeed non-regression tests slow_tests_deepspeed: test_installs - python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 + python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 python -m pytest tests/test_examples.py -v -s -k "deepspeed" slow_tests_diffusers: test_installs @@ -58,7 +58,7 @@ slow_tests_diffusers: test_installs # Run text-generation non-regression tests slow_tests_text_generation_example: test_installs - python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 + python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 python -m pytest tests/test_text_generation_example.py tests/test_encoder_decoder_text_summarization.py -v -s --token $(TOKEN) # Check if examples are up to date with the Transformers library diff --git a/README.md b/README.md index 626674eab0..90587ec304 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The `--upgrade-strategy eager` option is needed to ensure `optimum-habana` is up > To use DeepSpeed on HPUs, you also need to run the following command: >```bash ->pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +>pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 >``` Optimum Habana is a fast-moving project, and you may want to install it from source: @@ -211,7 +211,7 @@ Please refer to Habana Gaudi's official [installation guide](https://docs.habana > Tests should be run in a Docker container based on Habana Docker images. > -> The current version has been validated for SynapseAI 1.13. +> The current version has been validated for SynapseAI 1.14. ## Development diff --git a/docs/Dockerfile b/docs/Dockerfile index 312ebcedd0..0c60a33b86 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest +FROM vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest ARG commit_sha ARG clone_url @@ -7,13 +7,7 @@ ARG clone_url RUN apt-get update && apt-get install -y \ software-properties-common \ npm - -# Need node to build doc HTML. Taken from https://stackoverflow.com/a/67491580 -RUN apt-get update && apt-get install -y \ - software-properties-common \ - npm -RUN npm install npm@9.8.1 -g && \ - npm install n -g && \ +RUN npm install n -g && \ n latest RUN git clone $clone_url optimum-habana && cd optimum-habana && git checkout $commit_sha diff --git a/docs/source/installation.mdx b/docs/source/installation.mdx index 5ad7cc4f6e..eaad58cfd1 100644 --- a/docs/source/installation.mdx +++ b/docs/source/installation.mdx @@ -23,6 +23,6 @@ python -m pip install --upgrade-strategy eager optimum[habana] To use DeepSpeed on HPUs, you also need to run the following command: ```bash -python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +python -m pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 ``` diff --git a/docs/source/usage_guides/deepspeed.mdx b/docs/source/usage_guides/deepspeed.mdx index 28280f5684..b115554c83 100644 --- a/docs/source/usage_guides/deepspeed.mdx +++ b/docs/source/usage_guides/deepspeed.mdx @@ -31,7 +31,7 @@ You can find more information about DeepSpeed Gaudi integration [here](https://d To use DeepSpeed on Gaudi, you need to install Optimum Habana and [Habana's DeepSpeed fork](https://github.com/HabanaAI/DeepSpeed) with: ```bash pip install optimum[habana] -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 ``` @@ -78,7 +78,7 @@ It is strongly advised to read [this section](https://huggingface.co/docs/transf -Other examples of configurations for HPUs are proposed [here](https://github.com/HabanaAI/Model-References/tree/1.13.0/PyTorch/nlp/DeepSpeedExamples/deepspeed-bert/scripts) by Habana. +Other examples of configurations for HPUs are proposed [here](https://github.com/HabanaAI/Model-References/tree/1.14.0/PyTorch/nlp/DeepSpeedExamples/deepspeed-bert/scripts) by Habana. The [Transformers documentation](https://huggingface.co/docs/transformers/main_classes/deepspeed#configuration) explains how to write a configuration from scratch very well. A more complete description of all configuration possibilities is available [here](https://www.deepspeed.ai/docs/config-json/). diff --git a/examples/audio-classification/README.md b/examples/audio-classification/README.md index e8167b65d9..071e7c7b58 100644 --- a/examples/audio-classification/README.md +++ b/examples/audio-classification/README.md @@ -99,7 +99,7 @@ On 8 HPUs, this script should run in ~12 minutes and yield an accuracy of **80.4 > You need to install DeepSpeed with: > ```bash -> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 > ``` DeepSpeed can be used with almost the same command as for a multi-card run: diff --git a/examples/gaudi_spawn.py b/examples/gaudi_spawn.py index c0c8014076..6e1aa53a5d 100644 --- a/examples/gaudi_spawn.py +++ b/examples/gaudi_spawn.py @@ -85,7 +85,7 @@ def main(): if not is_deepspeed_available(): raise ImportError( "--use_deepspeed requires deepspeed: `pip install" - " git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0`." + " git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0`." ) # Patch sys.argv diff --git a/examples/multi-node-training/EFA/Dockerfile b/examples/multi-node-training/EFA/Dockerfile index a4565c1c36..a3d4d3ca99 100644 --- a/examples/multi-node-training/EFA/Dockerfile +++ b/examples/multi-node-training/EFA/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest +FROM vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest # Installs pdsh and upgrade pip RUN apt-get update && apt-get install -y pdsh && \ @@ -18,7 +18,7 @@ RUN sed -i 's/#Port 22/Port 3022/g' /etc/ssh/sshd_config && \ # Installs Optimum Habana and Habana's fork of DeepSpeed RUN pip install optimum[habana] && \ - pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 + pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 CMD ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa && \ chmod 600 ~/.ssh/id_rsa && \ diff --git a/examples/multi-node-training/GaudiNIC/Dockerfile b/examples/multi-node-training/GaudiNIC/Dockerfile index 76ab657140..9e73a4528f 100644 --- a/examples/multi-node-training/GaudiNIC/Dockerfile +++ b/examples/multi-node-training/GaudiNIC/Dockerfile @@ -1,4 +1,4 @@ -FROM vault.habana.ai/gaudi-docker/1.13.0/ubuntu20.04/habanalabs/pytorch-installer-2.1.0:latest +FROM vault.habana.ai/gaudi-docker/1.14.0/ubuntu22.04/habanalabs/pytorch-installer-2.1.1:latest # Installs pdsh and upgrade pip RUN apt-get update && apt-get install -y pdsh && \ @@ -12,7 +12,7 @@ RUN sed -i 's/#Port 22/Port 3022/g' /etc/ssh/sshd_config && \ # Installs Optimum Habana and Habana's fork of DeepSpeed RUN pip install optimum[habana] && \ - pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 + pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 CMD ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa && \ chmod 600 ~/.ssh/id_rsa && \ diff --git a/examples/multi-node-training/README.md b/examples/multi-node-training/README.md index 643084e7a5..eb1192d05a 100644 --- a/examples/multi-node-training/README.md +++ b/examples/multi-node-training/README.md @@ -34,7 +34,7 @@ We provide two `Dockerfile` to easily start your multi-node runs: - A `Dockerfile` provided [here](https://github.com/huggingface/optimum-habana/tree/main.examples/multi-node-training/GaudiNIC/Dockerfile) for multi-node runs using GaudiNIC. -The Dockerfile is based on an image compatible with Ubuntu 20.04 but you can easily adapt it to another OS. +The Dockerfile is based on an image compatible with Ubuntu 22.04 but you can easily adapt it to another OS. To build the Docker image, run: ```bash diff --git a/examples/speech-recognition/README.md b/examples/speech-recognition/README.md index a4c0fab57b..d9dd850a5c 100644 --- a/examples/speech-recognition/README.md +++ b/examples/speech-recognition/README.md @@ -119,7 +119,7 @@ On 8 HPUs, this script should run in *ca.* 49 minutes and yield a CTC loss of ** > You need to install DeepSpeed with: > ```bash -> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +> pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 > ``` DeepSpeed can be used with almost the same command as for a multi-card run: diff --git a/examples/text-generation/README.md b/examples/text-generation/README.md index e966ed84cc..5732e684a4 100644 --- a/examples/text-generation/README.md +++ b/examples/text-generation/README.md @@ -28,7 +28,7 @@ pip install -r requirements.txt Then, if you plan to use [DeepSpeed-inference](https://docs.habana.ai/en/latest/PyTorch/DeepSpeed/Inference_Using_DeepSpeed.html) (e.g. to use BLOOM/BLOOMZ), you should install DeepSpeed as follows: ```bash -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 ``` diff --git a/examples/text-generation/text-generation-pipeline/README.md b/examples/text-generation/text-generation-pipeline/README.md index 2fc93a6ca2..39aa462384 100644 --- a/examples/text-generation/text-generation-pipeline/README.md +++ b/examples/text-generation/text-generation-pipeline/README.md @@ -28,7 +28,7 @@ export PYTHONPATH=${PYTHONPATH}:${OPTIMUM_HABANA_PATH}/examples/text-generation If you plan to use [DeepSpeed-inference](https://docs.habana.ai/en/latest/PyTorch/DeepSpeed/Inference_Using_DeepSpeed.html), you should install DeepSpeed as follows: ```bash -pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0 +pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0 ``` ## Usage diff --git a/examples/text-generation/utils.py b/examples/text-generation/utils.py index 53e4c3bab6..9b66de8128 100644 --- a/examples/text-generation/utils.py +++ b/examples/text-generation/utils.py @@ -36,7 +36,7 @@ model_on_meta, write_checkpoints_json, ) -from optimum.habana.utils import check_habana_frameworks_version, check_optimum_habana_min_version, set_seed +from optimum.habana.utils import check_optimum_habana_min_version, set_seed def adjust_batch(batch, size): @@ -174,10 +174,7 @@ def setup_model(args, model_dtype, model_kwargs, logger): if args.use_hpu_graphs: from habana_frameworks.torch.hpu import wrap_in_hpu_graph - if check_habana_frameworks_version("1.13.0") and model.config.model_type == "falcon": - model = wrap_in_hpu_graph(model, hash_with_views=False) - else: - model = wrap_in_hpu_graph(model) + model = wrap_in_hpu_graph(model) if args.torch_compile and model.config.model_type == "llama": model = get_torch_compiled_model(model) diff --git a/notebooks/AI_HW_Summit_2022.ipynb b/notebooks/AI_HW_Summit_2022.ipynb index 7259408da9..37075fd3ef 100644 --- a/notebooks/AI_HW_Summit_2022.ipynb +++ b/notebooks/AI_HW_Summit_2022.ipynb @@ -261,7 +261,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0" + "!pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0" ] }, { diff --git a/optimum/habana/accelerate/accelerator.py b/optimum/habana/accelerate/accelerator.py index 56e9a63e23..cb21e15fe9 100644 --- a/optimum/habana/accelerate/accelerator.py +++ b/optimum/habana/accelerate/accelerator.py @@ -140,7 +140,7 @@ def __init__( if deepspeed_plugin: if not is_deepspeed_available(): raise ImportError( - "DeepSpeed is not installed => run `pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0`." + "DeepSpeed is not installed => run `pip install git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0`." ) mixed_precision = ( diff --git a/optimum/habana/accelerate/state.py b/optimum/habana/accelerate/state.py index 5c81e06a52..6b3c0b20d5 100644 --- a/optimum/habana/accelerate/state.py +++ b/optimum/habana/accelerate/state.py @@ -55,7 +55,7 @@ def __init__(self, cpu: bool = False, **kwargs): if not is_deepspeed_available(): raise ImportError( "DeepSpeed is not available, install it with: `pip install" - " git+https://github.com/HabanaAI/DeepSpeed.git@1.13.0`." + " git+https://github.com/HabanaAI/DeepSpeed.git@1.14.0`." ) self.distributed_type = GaudiDistributedType.DEEPSPEED import deepspeed diff --git a/optimum/habana/transformers/modeling_utils.py b/optimum/habana/transformers/modeling_utils.py index 8dca660732..a17e67bd4e 100644 --- a/optimum/habana/transformers/modeling_utils.py +++ b/optimum/habana/transformers/modeling_utils.py @@ -115,9 +115,6 @@ def adapt_transformers_to_gaudi(): transformers.models.vit.modeling_vit.ViTSelfAttention.forward = gaudi_vit_self_attention_forward # Optimization tweak for Wav2Vec2 - # TODO: enable _gaudi_wav2vec2_compute_mask_indices, _gaudi_wav2vec2_mask_hidden_states and - # gaudi_wav2vec2_encoder_forward when SynapseAI v1.13 is released - # They are disabled for now due to accuracy issues transformers.models.wav2vec2.modeling_wav2vec2._compute_mask_indices = _gaudi_wav2vec2_compute_mask_indices # transformers.models.wav2vec2.modeling_wav2vec2._sample_negative_indices = _gaudi_wav2vec2_sample_negative_indices transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2Model._mask_hidden_states = ( diff --git a/optimum/habana/utils.py b/optimum/habana/utils.py index 59b27ac984..6a92a42036 100644 --- a/optimum/habana/utils.py +++ b/optimum/habana/utils.py @@ -31,7 +31,7 @@ logger = logging.get_logger(__name__) -CURRENTLY_VALIDATED_SYNAPSE_VERSION = version.parse("1.13.0") +CURRENTLY_VALIDATED_SYNAPSE_VERSION = version.parse("1.14.0") def to_device_dtype(my_input: Any, target_device: torch.device = None, target_dtype: torch.dtype = None): diff --git a/tests/baselines/albert_large_v2.json b/tests/baselines/albert_large_v2.json index 246ec2e0af..3e0ff3cedf 100644 --- a/tests/baselines/albert_large_v2.json +++ b/tests/baselines/albert_large_v2.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 6e-5, "train_batch_size": 32, - "eval_f1": 91.7937, - "train_runtime": 3439.7776, - "train_samples_per_second": 52.388, + "eval_f1": 92.0109, + "train_runtime": 3246.7928, + "train_samples_per_second": 55.517, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -18,9 +18,9 @@ "multi_card": { "learning_rate": 6e-5, "train_batch_size": 32, - "eval_f1": 92.6602, - "train_runtime": 529.2307, - "train_samples_per_second": 422.912, + "eval_f1": 92.8155, + "train_runtime": 497.1048, + "train_samples_per_second": 449.321, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" diff --git a/tests/baselines/albert_xxlarge_v1.json b/tests/baselines/albert_xxlarge_v1.json index 6d1b84d500..a62153c717 100644 --- a/tests/baselines/albert_xxlarge_v1.json +++ b/tests/baselines/albert_xxlarge_v1.json @@ -19,8 +19,8 @@ "learning_rate": 5e-5, "train_batch_size": 12, "eval_f1": 95.1629, - "train_runtime": 1311.7084, - "train_samples_per_second": 75.513, + "train_runtime": 1308.2465, + "train_samples_per_second": 75.506, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" diff --git a/tests/baselines/bert_large_uncased_whole_word_masking.json b/tests/baselines/bert_large_uncased_whole_word_masking.json index 0587a45afe..a9b3da10d7 100644 --- a/tests/baselines/bert_large_uncased_whole_word_masking.json +++ b/tests/baselines/bert_large_uncased_whole_word_masking.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 3e-5, "train_batch_size": 24, - "eval_f1": 93.3328, - "train_runtime": 1718.8101, - "train_samples_per_second": 52.704, + "eval_f1": 93.2812, + "train_runtime": 1719.9389, + "train_samples_per_second": 52.696, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -18,9 +18,9 @@ "multi_card": { "learning_rate": 7e-5, "train_batch_size": 24, - "eval_f1": 93.0761, - "train_runtime": 303.7404, - "train_samples_per_second": 397.945, + "eval_f1": 93.2092, + "train_runtime": 306.8871, + "train_samples_per_second": 397.041, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -35,9 +35,9 @@ "single_card": { "learning_rate": 3e-5, "train_batch_size": 32, - "eval_f1": 0.906, - "train_runtime": 88.3426, - "train_samples_per_second": 171.262, + "eval_f1": 0.8968, + "train_runtime": 88.7004, + "train_samples_per_second": 171.252, "extra_arguments": [ "--max_seq_length 128", "--use_hpu_graphs_for_inference" @@ -46,9 +46,9 @@ "multi_card": { "learning_rate": 3e-5, "train_batch_size": 16, - "eval_f1": 0.8793, - "train_runtime": 60.0117, - "train_samples_per_second": 886.223, + "eval_f1": 0.8885, + "train_runtime": 61.8181, + "train_samples_per_second": 893.265, "extra_arguments": [ "--max_seq_length 128", "--use_hpu_graphs_for_inference" diff --git a/tests/baselines/bloom_7b1.json b/tests/baselines/bloom_7b1.json index 5615007880..37251e8651 100644 --- a/tests/baselines/bloom_7b1.json +++ b/tests/baselines/bloom_7b1.json @@ -7,8 +7,8 @@ "deepspeed": { "learning_rate": 1e-4, "train_batch_size": 8, - "train_runtime": 1572.712, - "train_samples_per_second": 4.705, + "train_runtime": 1556.481, + "train_samples_per_second": 4.757, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_cache False", diff --git a/tests/baselines/clip_roberta.json b/tests/baselines/clip_roberta.json index 2afa4630f3..50105645f1 100644 --- a/tests/baselines/clip_roberta.json +++ b/tests/baselines/clip_roberta.json @@ -7,8 +7,8 @@ "multi_card": { "learning_rate": 5e-5, "train_batch_size": 64, - "train_runtime": 304.5456, - "train_samples_per_second": 2565.686, + "train_runtime": 304.18, + "train_samples_per_second": 2582.676, "extra_arguments": [ "--data_dir $PWD/", "--dataset_config_name 2017", diff --git a/tests/baselines/distilbert_base_uncased.json b/tests/baselines/distilbert_base_uncased.json index b1c1f18e22..65427c7759 100644 --- a/tests/baselines/distilbert_base_uncased.json +++ b/tests/baselines/distilbert_base_uncased.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 1e-4, "train_batch_size": 48, - "eval_f1": 84.2412, - "train_runtime": 271.4795, - "train_samples_per_second": 334.268, + "eval_f1": 84.7137, + "train_runtime": 271.2751, + "train_samples_per_second": 334.792, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -18,9 +18,9 @@ "multi_card": { "learning_rate": 3e-4, "train_batch_size": 48, - "eval_f1": 82.918, - "train_runtime": 53.8298, - "train_samples_per_second": 2495.828, + "eval_f1": 82.8831, + "train_runtime": 54.0269, + "train_samples_per_second": 2500.721, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" diff --git a/tests/baselines/gpt2.json b/tests/baselines/gpt2.json index 531b368c12..53dd257a14 100644 --- a/tests/baselines/gpt2.json +++ b/tests/baselines/gpt2.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 5e-5, "train_batch_size": 4, - "perplexity": 22.218811031228753, - "train_runtime": 292.7255, - "train_samples_per_second": 16.265, + "perplexity": 22.2641, + "train_runtime": 236.4595, + "train_samples_per_second": 20.24, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_hpu_graphs_for_inference", @@ -19,9 +19,9 @@ "multi_card": { "learning_rate": 4e-4, "train_batch_size": 4, - "perplexity": 22.209473717474346, - "train_runtime": 80.3361, - "train_samples_per_second": 121.647, + "perplexity": 22.2696, + "train_runtime": 72.1582, + "train_samples_per_second": 150.303, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_hpu_graphs_for_inference", diff --git a/tests/baselines/gpt2_xl.json b/tests/baselines/gpt2_xl.json index ec0d22e03f..4e26eef50b 100644 --- a/tests/baselines/gpt2_xl.json +++ b/tests/baselines/gpt2_xl.json @@ -7,9 +7,9 @@ "deepspeed": { "learning_rate": 5e-5, "train_batch_size": 2, - "perplexity": 12.6678, - "train_runtime": 378.1958, - "train_samples_per_second": 15.898, + "perplexity": 12.6711, + "train_runtime": 380.1311, + "train_samples_per_second": 16.045, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_hpu_graphs_for_inference", diff --git a/tests/baselines/llama_7b.json b/tests/baselines/llama_7b.json index 4331faace5..d14260f6f6 100644 --- a/tests/baselines/llama_7b.json +++ b/tests/baselines/llama_7b.json @@ -8,8 +8,8 @@ "learning_rate": 1e-4, "train_batch_size": 2, "perplexity": 2.7362, - "train_runtime": 516.4597, - "train_samples_per_second": 20.55, + "train_runtime": 538.1199, + "train_samples_per_second": 20.37, "extra_arguments": [ "--bf16", "--gradient_accumulation_steps 4", diff --git a/tests/baselines/roberta_base.json b/tests/baselines/roberta_base.json index 6b4e781df5..581bf7a767 100644 --- a/tests/baselines/roberta_base.json +++ b/tests/baselines/roberta_base.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 3e-5, "train_batch_size": 12, - "eval_f1": 91.9001, - "train_runtime": 608.3774, - "train_samples_per_second": 147.518, + "eval_f1": 91.8466, + "train_runtime": 610.6291, + "train_samples_per_second": 147.028, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -18,9 +18,9 @@ "multi_card": { "learning_rate": 8e-5, "train_batch_size": 12, - "eval_f1": 91.6451, - "train_runtime": 102.6389, - "train_samples_per_second": 1084.287, + "eval_f1": 91.7635, + "train_runtime": 102.8332, + "train_samples_per_second": 1081.823, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -35,9 +35,9 @@ "multi_card": { "learning_rate": 5e-5, "train_batch_size": 24, - "perplexity": 3.5972, - "train_runtime": 41.1445, - "train_samples_per_second": 553.783, + "perplexity": 3.5988, + "train_runtime": 41.6183, + "train_samples_per_second": 553.572, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_hpu_graphs_for_inference", diff --git a/tests/baselines/roberta_large.json b/tests/baselines/roberta_large.json index 4c4d5bfd54..836a5fac3a 100644 --- a/tests/baselines/roberta_large.json +++ b/tests/baselines/roberta_large.json @@ -7,9 +7,9 @@ "single_card": { "learning_rate": 3e-5, "train_batch_size": 12, - "eval_f1": 94.5151, - "train_runtime": 1798.7457, - "train_samples_per_second": 49.955, + "eval_f1": 94.3634, + "train_runtime": 1801.8127, + "train_samples_per_second": 49.895, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -18,9 +18,9 @@ "multi_card": { "learning_rate": 8e-5, "train_batch_size": 12, - "eval_f1": 94.2585, - "train_runtime": 299.259, - "train_samples_per_second": 365.395, + "eval_f1": 94.0942, + "train_runtime": 299.6718, + "train_samples_per_second": 364.947, "extra_arguments": [ "--max_seq_length 384", "--use_hpu_graphs_for_inference" @@ -35,9 +35,9 @@ "multi_card": { "learning_rate": 5e-5, "train_batch_size": 8, - "perplexity": 2.7817, - "train_runtime": 71.9316, - "train_samples_per_second": 216.464, + "perplexity": 2.7851, + "train_runtime": 72.0278, + "train_samples_per_second": 217.107, "extra_arguments": [ "--dataset_config_name wikitext-2-raw-v1", "--use_hpu_graphs_for_inference", diff --git a/tests/baselines/swin_base_patch4_window7_224_in22k.json b/tests/baselines/swin_base_patch4_window7_224_in22k.json index 94b9934137..6d49238b5d 100644 --- a/tests/baselines/swin_base_patch4_window7_224_in22k.json +++ b/tests/baselines/swin_base_patch4_window7_224_in22k.json @@ -8,8 +8,8 @@ "learning_rate": 3e-5, "train_batch_size": 64, "eval_accuracy": 0.9851, - "train_runtime": 250.4023, - "train_samples_per_second": 203.619, + "train_runtime": 249.7865, + "train_samples_per_second": 203.94, "extra_arguments": [ "--remove_unused_columns False", "--image_column_name img", @@ -25,8 +25,8 @@ "learning_rate": 2e-4, "train_batch_size": 64, "eval_accuracy": 0.9836, - "train_runtime": 110.8333, - "train_samples_per_second": 1679.61, + "train_runtime": 113.9324, + "train_samples_per_second": 1691.705, "extra_arguments": [ "--remove_unused_columns False", "--image_column_name img", diff --git a/tests/baselines/t5_small.json b/tests/baselines/t5_small.json index 557522dc2f..c25cba6716 100644 --- a/tests/baselines/t5_small.json +++ b/tests/baselines/t5_small.json @@ -7,10 +7,10 @@ "multi_card": { "learning_rate": 5e-5, "train_batch_size": 4, - "eval_rougeLsum": 38.4574, - "train_runtime": 1088.8385, - "train_samples_per_second": 269.432, - "eval_samples_per_second": 19.774, + "eval_rougeLsum": 38.6197, + "train_runtime": 1087.1076, + "train_samples_per_second": 268.231, + "eval_samples_per_second": 68.222, "extra_arguments": [ "--dataset_config \"3.0.0\"", "--source_prefix \"summarize: \"", @@ -30,9 +30,9 @@ "multi_card": { "learning_rate": 2e-4, "train_batch_size": 16, - "eval_f1": 64.4393, - "train_runtime": 236.8974, - "train_samples_per_second": 1242.793, + "eval_f1": 64.8034, + "train_runtime": 228.8655, + "train_samples_per_second": 1246.527, "extra_arguments": [ "--context_column context", "--question_column question", diff --git a/tests/baselines/vit_base_patch16_224_in21k.json b/tests/baselines/vit_base_patch16_224_in21k.json index 96c945bfc0..09bb543c11 100644 --- a/tests/baselines/vit_base_patch16_224_in21k.json +++ b/tests/baselines/vit_base_patch16_224_in21k.json @@ -8,8 +8,8 @@ "learning_rate": 5e-5, "train_batch_size": 64, "eval_accuracy": 0.9828, - "train_runtime": 139.2985, - "train_samples_per_second": 349.875, + "train_runtime": 139.4456, + "train_samples_per_second": 349.624, "extra_arguments": [ "--remove_unused_columns False", "--image_column_name img", @@ -24,8 +24,8 @@ "learning_rate": 2e-4, "train_batch_size": 64, "eval_accuracy": 0.98, - "train_runtime": 63.2587, - "train_samples_per_second": 2509.027, + "train_runtime": 58.345, + "train_samples_per_second": 2509.51, "extra_arguments": [ "--remove_unused_columns False", "--image_column_name img", diff --git a/tests/baselines/wav2vec2_base.json b/tests/baselines/wav2vec2_base.json index cc26f61dde..1696e4ff1d 100644 --- a/tests/baselines/wav2vec2_base.json +++ b/tests/baselines/wav2vec2_base.json @@ -8,9 +8,9 @@ "learning_rate": 5e-4, "train_batch_size": 32, "eval_accuracy": 0.8045, - "train_runtime": 361.08, - "train_samples_per_second": 714.447, - "eval_samples_per_second": 294.293, + "train_runtime": 363.7165, + "train_samples_per_second": 715.004, + "eval_samples_per_second": 312.719, "extra_arguments": [ "--audio_column_name audio", "--label_column_name language", @@ -55,4 +55,4 @@ } } } -} +} \ No newline at end of file diff --git a/tests/baselines/wav2vec2_large_lv60.json b/tests/baselines/wav2vec2_large_lv60.json index 47505a80cb..e3473420da 100644 --- a/tests/baselines/wav2vec2_large_lv60.json +++ b/tests/baselines/wav2vec2_large_lv60.json @@ -7,10 +7,10 @@ "multi_card": { "learning_rate": 6e-4, "train_batch_size": 8, - "eval_wer": 0.0543, - "train_runtime": 935.2505, - "train_samples_per_second": 66.563, - "eval_samples_per_second": 67.677, + "eval_wer": 0.0555, + "train_runtime": 889.0079, + "train_samples_per_second": 70.036, + "eval_samples_per_second": 57.302, "extra_arguments": [ "--dataset_config_name clean", "--train_split_name train.100", diff --git a/tests/ci/slow_tests_diffusers.sh b/tests/ci/slow_tests_diffusers.sh index 6e4293764c..ab776092a5 100644 --- a/tests/ci/slow_tests_diffusers.sh +++ b/tests/ci/slow_tests_diffusers.sh @@ -2,5 +2,6 @@ python -m pip install --upgrade pip export RUN_SLOW=true +make test_installs CUSTOM_BF16_OPS=1 python -m pytest tests/test_diffusers.py -v -s -k "test_no_throughput_regression_autocast" make slow_tests_diffusers diff --git a/tests/test_diffusers.py b/tests/test_diffusers.py index 9545e021fa..23ead63913 100644 --- a/tests/test_diffusers.py +++ b/tests/test_diffusers.py @@ -51,11 +51,11 @@ THROUGHPUT_BASELINE_BF16 = 1.019 THROUGHPUT_BASELINE_AUTOCAST = 0.389 else: - THROUGHPUT_BASELINE_BF16 = 0.301 - THROUGHPUT_BASELINE_AUTOCAST = 0.108 + THROUGHPUT_BASELINE_BF16 = 0.309 + THROUGHPUT_BASELINE_AUTOCAST = 0.114 -TEXTUAL_INVERSION_THROUGHPUT = 58.16156989437878 -TEXTUAL_INVERSION_RUNTIME = 206.32180358597543 +TEXTUAL_INVERSION_THROUGHPUT = 59.13010439968039 +TEXTUAL_INVERSION_RUNTIME = 202.94231038199996 _run_custom_bf16_ops_test_ = parse_flag_from_env("CUSTOM_BF16_OPS", default=False) diff --git a/tests/test_encoder_decoder_text_summarization.py b/tests/test_encoder_decoder_text_summarization.py index 197cc37e7a..fed5fade78 100644 --- a/tests/test_encoder_decoder_text_summarization.py +++ b/tests/test_encoder_decoder_text_summarization.py @@ -22,8 +22,8 @@ # Gaudi1 CI baselines MODELS_TO_TEST = { "bf16": [ - ("facebook/bart-large-cnn", "Habana/bart", 2.588, 26.0688, 2, 1), - ("t5-3b", "Habana/t5", 0.98, 21.56, 2, 1), + ("facebook/bart-large-cnn", "Habana/bart", 2.574, 26.5069, 2, 1), + ("t5-3b", "Habana/t5", 0.987, 21.3831, 2, 1), ], } diff --git a/tests/test_text_generation_example.py b/tests/test_text_generation_example.py index 5c5dc63fc2..17f8c8acc6 100644 --- a/tests/test_text_generation_example.py +++ b/tests/test_text_generation_example.py @@ -38,20 +38,20 @@ # Gaudi1 CI baselines MODELS_TO_TEST = { "bf16": [ - ("bigscience/bloomz-7b1", 41.93942748147396), - ("gpt2-xl", 126.6292071377241), + ("bigscience/bloomz-7b1", 41.51855420676164), + ("gpt2-xl", 137.159223188195), # TODO: fix OPT 6.7B # ("facebook/opt-6.7b", 0.0), - ("EleutherAI/gpt-j-6b", 37.14562499113717), - ("meta-llama/Llama-2-7b-hf", 43.951804139391925), - ("tiiuae/falcon-7b", 44.288602257903726), - ("bigcode/starcoder", 15.955986010526113), - ("Salesforce/codegen2-1B", 109.03016111561857), - ("mosaicml/mpt-7b", 44.888696119070424), - ("mistralai/Mistral-7B-v0.1", 40.0690067247771), + ("EleutherAI/gpt-j-6b", 50.66146537939035), + ("meta-llama/Llama-2-7b-hf", 44.29688546702468), + ("tiiuae/falcon-7b", 44.217408724737744), + ("bigcode/starcoder", 15.948143541091655), + ("Salesforce/codegen2-1B", 153.79670508220687), + ("mosaicml/mpt-7b", 44.80241777760578), + ("mistralai/Mistral-7B-v0.1", 40.00435417311187), ], "deepspeed": [ - ("bigscience/bloomz-7b1", 27.34439410425298), + ("bigscience/bloomz-7b1", 31.044523676681507), ], "torch_compile": [], }