Image decoder for data_pipeline #2
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) Meta Platforms, Inc. and affiliates. | ||
| # All rights reserved. | ||
| # | ||
| # This source code is licensed under the BSD-style license found in the | ||
| # LICENSE file in the root directory of this source tree. | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| torch: | ||
| type: string | ||
| required: true | ||
| py: | ||
| type: string | ||
| required: true | ||
| variant: | ||
| type: string | ||
| required: true | ||
| arch: | ||
| type: string | ||
| default: 'x86_64' | ||
| sanitizers: | ||
| type: string | ||
| default: 'nosan' | ||
| artifacts_name: | ||
| type: string | ||
| default: 'wheels' | ||
| version_override: | ||
| type: string | ||
| default: '' | ||
| label_version_with_variant: | ||
| type: boolean | ||
| default: true | ||
| run_integration_tests: | ||
| type: boolean | ||
| default: false | ||
| run_on_device: | ||
| type: boolean | ||
| default: false | ||
| jobs: | ||
| build: | ||
| name: Build | ||
| runs-on: | ||
| labels: 4-core-ubuntu | ||
| container: | ||
| image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:2-${{ inputs.variant }} | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| steps: | ||
| - name: Check-out the repository | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| submodules: recursive | ||
| - name: Create the Python virtual environment | ||
| run: | | ||
| python${{ inputs.py }} -m venv ~/venv | ||
| echo ~/venv/bin >> "$GITHUB_PATH" | ||
| - name: Install PyTorch | ||
| run: | | ||
| pip install --extra-index-url https://download.pytorch.org/whl/${{ inputs.variant }}\ | ||
| torch==${{ inputs.torch }} | ||
| - name: Install requirements | ||
| run: | | ||
| pip install --requirement fairseq2n/python/requirements-build.txt | ||
| - name: Override project version | ||
| if: inputs.version_override | ||
| run: | | ||
| tools/set-project-version.sh ${{ inputs.version_override }} | ||
| - name: Set build variant as local version label | ||
| if: inputs.label_version_with_variant | ||
| env: | ||
| VARIANT: ${{ inputs.variant }} | ||
| run: | | ||
| version=$(cat VERSION) | ||
| # If the version has already a local label, append the variant. | ||
| if [[ $version == *+* ]]; then | ||
| tools/set-project-version.sh $version.$VARIANT | ||
| else | ||
| tools/set-project-version.sh $version+$VARIANT | ||
| fi | ||
| - name: Build fairseq2n | ||
| working-directory: fairseq2n | ||
| env: | ||
| VARIANT: ${{ inputs.variant }} | ||
| SANITIZERS: ${{ inputs.sanitizers }} | ||
| run: | | ||
| # We build our CUDA kernels for Volta and Ampere. | ||
| if [[ $VARIANT == cu* ]]; then | ||
| cuda_archs="70-real;80-real;80-virtual" | ||
| cuda=ON | ||
| else | ||
| cuda=OFF | ||
| fi | ||
| # Since they do not play well together, we perform LTO only if no | ||
| # sanitizer is enabled. | ||
| if [[ $SANITIZERS == nosan ]]; then | ||
| build_type=Release | ||
| lto=ON | ||
| else | ||
| build_type=Debug | ||
| lto=OFF | ||
| fi | ||
| cmake\ | ||
| -GNinja\ | ||
| -DCMAKE_BUILD_TYPE=$build_type\ | ||
| -DCMAKE_CUDA_ARCHITECTURES=$cuda_archs\ | ||
| -DFAIRSEQ2N_PERFORM_LTO=$lto\ | ||
| -DFAIRSEQ2N_SANITIZERS="${SANITIZERS/_/;}"\ | ||
| -DFAIRSEQ2N_TREAT_WARNINGS_AS_ERRORS=ON\ | ||
| -DFAIRSEQ2N_USE_CUDA=$cuda\ | ||
| -DFAIRSEQ2N_PYTHON_DEVEL=OFF\ | ||
| -B build | ||
| cmake --build build | ||
| - name: Package fairseq2n | ||
| working-directory: fairseq2n/python | ||
| run: | | ||
| pip wheel\ | ||
| --use-pep517\ | ||
| --no-build-isolation\ | ||
| --no-deps\ | ||
| --config-settings "--build-option=--plat-name"\ | ||
| --config-settings "--build-option=manylinux2014_${{ inputs.arch }}"\ | ||
| --wheel-dir build/wheelhouse\ | ||
| . | ||
| - name: Package fairseq2 | ||
| run: | | ||
| pip wheel --no-deps --wheel-dir build/wheelhouse . | ||
| - name: Upload wheels and native tests to staging | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }} | ||
| path: | | ||
| build/wheelhouse/*.whl | ||
| fairseq2n/build/tests/run-tests | ||
| fairseq2n/python/build/wheelhouse/*.whl | ||
| retention-days: 1 | ||
| test: | ||
| name: Test | ||
| needs: build | ||
| runs-on: | ||
| labels: 4-core-ubuntu | ||
| container: | ||
| image: ghcr.io/facebookresearch/fairseq2-ci-manylinux_${{ inputs.arch }}:2-${{ inputs.variant }} | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| steps: | ||
| <<<<<<< HEAD | ||
| <<<<<<< HEAD | ||
| ======= | ||
| - name: Install libsndfile | ||
| run: | | ||
| yum --assumeyes install libsndfile-devel | ||
| - name: Install libpng-dev | ||
| run: | | ||
| yum --assumeyes install libpng-devel | ||
| <<<<<<< HEAD | ||
| <<<<<<< HEAD | ||
| <<<<<<< HEAD | ||
| >>>>>>> 18f3c8e9 (add png to build configs, support float32 tensor) | ||
| ======= | ||
| - name: Install libjpeg-dev | ||
| run: | | ||
| <<<<<<< HEAD | ||
| <<<<<<< HEAD | ||
| yum --assumeyes install libjpeg-devel | ||
| >>>>>>> 0587a6b7 (jpeg decoder) | ||
| ======= | ||
| - name: Install libjpeg-turbo | ||
| run: | | ||
| yum --assumeyes install libjpeg-turbo | ||
| >>>>>>> e0dcbd24 (change package name) | ||
| ======= | ||
| - name: Install libjpeg-dev | ||
| run: | | ||
| <<<<<<< HEAD | ||
| yum --assumeyes install libjpeg-dev | ||
| >>>>>>> b31b48e5 (change package name) | ||
| ======= | ||
| yum --assumeyes install libjpeg-devel | ||
| >>>>>>> 9be595fe (fix typo) | ||
| ======= | ||
| yum --assumeyes install libjpeg-devel-2.1.2 | ||
| >>>>>>> 9caad8cc (download libjpeg 1.5 and use const data_ptr) | ||
| ======= | ||
| yum --assumeyes install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/libjpeg-turbo-1.5.3-12.el8.x86_64.rpm | ||
| yum --assumeyes install https://rpmfind.net/linux/centos/8-stream/AppStream/x86_64/os/Packages/libjpeg-turbo-devel-1.5.3-12.el8.x86_64.rpm | ||
| >>>>>>> a06b73b1 (libjpeg 1.5) | ||
| ======= | ||
| >>>>>>> 7a2c1170 (fix lint) | ||
| - name: Download wheels and native tests from staging | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: ${{ inputs.artifacts_name }}-pt${{ inputs.torch }}-py${{ inputs.py }}-linux_${{ inputs.arch }}-${{ inputs.variant }}-${{ inputs.sanitizers }} | ||
| path: ~/artifacts/ | ||
| - name: Check-out the repository | ||
| uses: actions/checkout@v3 | ||
| - name: Create the Python virtual environment | ||
| run: | | ||
| python${{ inputs.py }} -m venv ~/venv | ||
| echo ~/venv/bin >> "$GITHUB_PATH" | ||
| - name: Install PyTorch | ||
| run: | | ||
| pip install --extra-index-url https://download.pytorch.org/whl/${{ inputs.variant }}\ | ||
| torch==${{ inputs.torch }} | ||
| - name: Install requirements | ||
| env: | ||
| ARCH: ${{ inputs.arch }} | ||
| run: | | ||
| pip install --requirement requirements-devel.txt | ||
| if [[ $ARCH == x86_64 ]]; then | ||
| # Our Docker images have glibc 2.17, but tbb versions later than | ||
| # 2021.9 are built against a newer glibc. | ||
| pip install tbb==2021.8 | ||
| fi | ||
| - name: Install fairseq2n | ||
| run: | | ||
| pip install --no-cache-dir ~/artifacts/fairseq2n/python/build/wheelhouse/*.whl | ||
| - name: Install fairseq2 | ||
| run: | | ||
| pip install --no-cache-dir ~/artifacts/build/wheelhouse/*.whl | ||
| - name: Set the sanitizer variables | ||
| if: inputs.sanitizers != 'nosan' | ||
| env: | ||
| SANITIZERS: ${{ inputs.sanitizers }} | ||
| run: | | ||
| { | ||
| # We can't enable container overflow checks due to false positives | ||
| # in non-instrumented dependencies. | ||
| echo ASAN_OPTIONS=detect_container_overflow=0 | ||
| echo LSAN_OPTIONS=suppressions=fairseq2n/LSan.supp,exitcode=0,log_path=$HOME/lsan.out | ||
| for sanitizer in ${SANITIZERS//_/ }; do | ||
| # LIBASAN and LIBTSAN environment variables are defined in the | ||
| # container image. | ||
| case $sanitizer in | ||
| asan) | ||
| echo SANITIZER_LIB=$(echo $LIBASAN) | ||
| break | ||
| ;; | ||
| tsan) | ||
| echo SANITIZER_LIB=$(echo $LIBTSAN) | ||
| break | ||
| ;; | ||
| esac | ||
| done | ||
| } >> "$GITHUB_ENV" | ||
| - name: Run native tests | ||
| run: | | ||
| chmod 755 ~/artifacts/fairseq2n/build/tests/run-tests | ||
| site_packages=~/venv/lib/python${{ inputs.py }}/site-packages | ||
| LD_LIBRARY_PATH=~/venv/lib:$site_packages/torch/lib:$site_packages/fairseq2n/lib:$LD_LIBRARY_PATH\ | ||
| ~/artifacts/fairseq2n/build/tests/run-tests | ||
| - name: Run Python tests | ||
| env: | ||
| RUN_INTEGRATION_TESTS: ${{ inputs.run_integration_tests }} | ||
| RUN_ON_DEVICE: ${{ inputs.run_on_device }} | ||
| run: | | ||
| if [[ $RUN_INTEGRATION_TESTS == true ]]; then | ||
| integration=--integration | ||
| fi | ||
| if [[ $RUN_ON_DEVICE != true ]]; then | ||
| dev=cpu | ||
| else | ||
| dev=cuda:0 | ||
| fi | ||
| LD_PRELOAD=${SANITIZER_LIB} pytest -rP --device $dev $integration --verbose | ||
| - name: Check the output of the leak sanitizer | ||
| if: env.LSAN_OPTIONS != '' | ||
| run: | | ||
| # Unfortunately Python leaks quite a bit of memory, so we cannot rely | ||
| # on the raw output of LSan. As a rudimentary workaround, we check if | ||
| # any stack frame has a symbol containing the word 'fairseq2'. | ||
| for f in ~/lsan.out.*; do | ||
| cat $f | ||
| if grep --silent 'fairseq2' $f; then | ||
| exit 1 | ||
| fi | ||
| done | ||
