viash-test.yml

name: viash test

on:
  pull_request:
  push:
    branches: [ main ]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ !contains(github.ref, 'main')}}

jobs:
  linting:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install ruff
      - name: Run Ruff
        run: ruff check --output-format=github .
      
      - uses: r-lib/actions/setup-r@v2
        with:
          use-public-rspm: true

      - uses: r-lib/actions/setup-r-dependencies@v2
        with:
          packages: any::lintr, any::styler, any::roxygen2
          needs: lint, styler

      - name: Lint
        run: lintr::lint_dir(path = ".")
        shell: Rscript {0}
        env:
          LINTR_ERROR_ON_LINT: true

      - name: Style
        run: styler::style_dir(dry = "off")
        shell: Rscript {0}
        

  # phase 1
  list:
    env:
      s3_bucket: s3://openpipelines-data/
    runs-on: ubuntu-latest

    outputs:
      matrix: ${{ steps.detect_changed_components.outputs.output_matrix }}
      cache_key: ${{ steps.cache.outputs.cache_key }}
      dest_paths: ${{ steps.cache.outputs.dest_paths }}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0 

    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
    - uses: data-intuitive/reclaim-the-bytes@v2

    - uses: viash-io/viash-actions/setup@v6

    - name: Check if all config can be parsed if there is no unicode support
      run: |
        LANG=C viash ns list > /dev/null
        JAVA_TOOL_OPTIONS=-Dfile.encoding=ascii viash ns build --parallel > /dev/null

    - uses: viash-io/viash-actions/project/sync-and-cache@v6
      id: cache

    - id: ns_list
      uses: viash-io/viash-actions/ns-list@v6
      with:
        engine: docker
        runner: executable
        format: json
        query_namespace: ^(?!workflows|test_workflows)

    - id: detect_changed_components
      uses: viash-io/viash-actions/project/detect-changed-components@v6
      with:
        input_file: "${{ steps.ns_list.outputs.output_file }}"

  # phase 2
  viash_test:
    needs: list
    if: ${{ needs.list.outputs.matrix != '[]' && needs.list.outputs.matrix != '' }}
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        component: ${{ fromJson(needs.list.outputs.matrix) }}

    steps:
    # Remove unnecessary files to free up space. Otherwise, we get 'no space left on device.'
    - uses: data-intuitive/reclaim-the-bytes@v2

    - uses: actions/checkout@v4

    - uses: viash-io/viash-actions/setup@v6
    - uses: viash-io/viash-actions/project/update-docker-engine@v6
    # use cache
    - name: Restore test resources cache
      if: ${{ needs.list.outputs.cache_key != '' }}
      uses: actions/cache/restore@v4
      timeout-minutes: 10
      with:
        path: ${{ needs.list.outputs.dest_paths }}
        key: ${{ needs.list.outputs.cache_key }}
        fail-on-cache-miss: true

    - name: Remove unused test resources to save space (only when restoring from cache)
      shell: bash
      run: |
        readarray -t resources < <(viash config view --format json "${{ matrix.component.config }}" | jq -r -c '(.build_info.config | capture("^(?<dir>.*\/)").dir) as $dir | .test_resources | map(select(.type == "file")) | map($dir + .path) | unique | .[]')
        to_not_remove=()
        for resource in "${resources[@]}"; do
          if [[ $resource == *"resources_test"* ]]; then
            relative_path=${resource#*resources_test/}
            relative_path_trailing_slash_removed=${relative_path%/}
            to_not_remove+=("-path" "./resources_test/$relative_path_trailing_slash_removed" "-prune" "-o")
          fi
        done
        # Remove last prune and -o
        if (( ${#errors[@]} )); then
          unset 'to_not_remove[${#to_not_remove[@]}-1]'
          unset 'to_not_remove[${#to_not_remove[@]}-1]'
          to_not_remove+=( "(" "${to_not_remove[@]}" ")" "-prune" "-o")
        fi
        find ./resources_test/ "${to_not_remove[@]}" -type f -exec rm {} +

    - name: Login to the nvidia container registry
      uses: docker/login-action@v3
      env: 
          NVIDIA_PASSWORD: ${{ secrets.NVIDIA_PASSWORD }}
      if: ${{ env.NVIDIA_PASSWORD != '' }}
      with:
        registry: nvcr.io
        username: $oauthtoken
        password: ${{ env.NVIDIA_PASSWORD }}

    - name: Run test
      timeout-minutes: 40
      run: |
        viash test \
          "${{ matrix.component.config }}" \
          --cpus 4 \
          --memory "14gb" \
          --engine docker \
          --runner executable