main.yml

name: main

on:
  push:
  pull_request:

concurrency:
  group: main-${{ github.head_ref }}
  cancel-in-progress: true

env:
  FORCE_COLOR: 3

jobs:
  get_python_version:
    runs-on: ubuntu-latest
    outputs:
      PYVERSION: ${{ steps.get_python_version.outputs.PYVERSION }}
    steps:
      - uses: actions/checkout@v4
      - name: Read the Python version from Makefile.envs
        id: get_python_version
        run: |
          echo "PYVERSION=$(git grep 'export PYVERSION ?=' Makefile.envs | cut -d' ' -f4)"  >> "$GITHUB_OUTPUT"
  test-python:
    runs-on: ubuntu-latest
    needs: get_python_version
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ needs.get_python_version.outputs.PYVERSION }}
      - name: Install requirements
        shell: bash -l {0}
        run: |
          mkdir test-results
          python3 -m pip install -r requirements.txt -r requirements-deploy.txt
          make pyodide_build
      - name: Run tests
        shell: bash -l {0}
        run: |
          PYODIDE_ROOT=. pytest \
              --junitxml=test-results/junit.xml \
              --verbose \
              --runtime=host \
              --cov=pyodide_build --cov=pyodide \
              src packages/_tests tools/
      - uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: false

  build-core:
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            pyodide_packages: "tag:core,numpy${{ needs.test-scipy-trigger.outputs.test-scipy == 'true' && ',scipy' || '' }}"
          - os: macos-14
            pyodide_packages: "tag:core,numpy"
    runs-on: ${{ matrix.os }}
    needs: [test-scipy-trigger]
    env:
      EMSDK_NUM_CORES: 3
      EMCC_CORES: 3
      PYODIDE_JOBS: 3
      CCACHE_DIR: /tmp/ccache

    steps:
      - uses: actions/checkout@v4

      - name: Cache ccache output
        uses: actions/cache@v4
        with:
          path: |
            /tmp/ccache
          key: ${{ hashFiles('Makefile.envs') }}-${{ runner.os }}-v20211025-

      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: pyodide-env
          environment-file: environment.yml
          channels: conda-forge

      - name: Check Versions
        shell: bash -l {0}
        run: |
          python --version
          node --version
          npm --version
          which python
          which node
          which npm

      - name: Install dependencies (ubuntu)
        shell: bash -l {0}
        if: ${{ contains(runner.os, 'ubuntu') }}
        run: |
          sudo apt install -y build-essential git xxd

      - name: Install dependencies (macos)
        shell: bash -l {0}
        if: ${{ contains(runner.os, 'macos') }}
        run: |
          brew install coreutils

      - name: Install dependencies (Python)
        shell: bash -l {0}
        run: |
          pip install -r requirements.txt

      - name: Build emsdk
        shell: bash -l {0}
        run: |
          which ccache

          ccache -z
          make -C emsdk
          ccache -s

      - name: Build Cpython
        shell: bash -l {0}
        run: |
          # This is necessary to use the ccache from emsdk
          source pyodide_env.sh

          which ccache

          ccache -z
          make -C cpython
          ccache -s

      - name: build Pyodide with packages ${{ matrix.pyodide_packages }}
        shell: bash -l {0}
        run: |
          # This is necessary to use the ccache from emsdk
          source pyodide_env.sh
          ccache -z
          PYODIDE_PACKAGES=${{ matrix.pyodide_packages }} make
          ccache -s

      - name: check-size
        shell: bash -l {0}
        run: |
          ls -lh dist/

          pip install brotli
          ./tools/check_compressed_size.py dist/pyodide.asm.* dist/python_stdlib*

      - name: Store artifacts build
        uses: actions/upload-artifact@v4
        with:
          name: core-build-${{ runner.os }}
          path: ./dist/
          retention-days: 60

  test-core:
    runs-on: ${{ matrix.os }}
    env:
      DISPLAY: :99

    needs: [build-core]
    strategy:
      fail-fast: false
      matrix:
        os: [macos-14]
        runner: [selenium]
        browser: [safari]

    steps:
      - uses: actions/checkout@v4

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: core-build-${{ runner.os }}
          path: ./dist/

      - uses: conda-incubator/setup-miniconda@v3
        with:
          activate-environment: pyodide-env
          python-version: "3.12"
          channels: conda-forge

      - name: install test requirements
        shell: bash -l {0}
        run: |
          pip install -r requirements.txt -r requirements-deploy.txt
          make pyodide_build

      - uses: pyodide/pyodide-actions/install-browser@v2
        with:
          runner: ${{ matrix.runner }}
          browser: ${{ matrix.browser }}

      - name: run core tests
        env:
          BROWSER: ${{ matrix.browser }}
          RUNNER: ${{ matrix.runner }}
        shell: bash -l {0}
        run: |
          ls -lh
          ls -lh dist/
          tools/pytest_wrapper.py src packages/micropip/ tools/ \
            -v \
            -k "not webworker" \
            --runtime="${BROWSER}-no-host" \
            --runner "${RUNNER}" \
            --durations 50 \
            --junitxml=test-results/core_test.xml

      - name: run package tests
        env:
          BROWSER: ${{ matrix.browser }}
          RUNNER: ${{ matrix.runner }}
        shell: bash -l {0}
        run: |
          ls -lh
          ls -lh dist/
          tools/pytest_wrapper.py packages/*/test* \
            -v \
            -k "numpy and not joblib" \
            --runtime="${BROWSER}-no-host" \
            --runner "${RUNNER}" \
            --durations 50 \
            --junitxml=test-results/packages_test.xml

      - name: Test Summary
        uses: test-summary/action@v2
        with:
          paths: "test-results/*.xml"
        if: always()

  test-scipy-trigger:
    name: test-scipy-trigger
    runs-on: ubuntu-latest
    outputs:
      test-scipy: ${{ steps.check-build-trigger.outputs.trigger }}

    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Check build trigger
        id: check-build-trigger
        shell: bash
        run: |
          set -e -x

          COMMIT_MSG=$(git log --no-merges -1 --oneline)

          # The scipy tests will be triggered on push or on pull_request when the commit
          # message contains "[scipy]"
          if [[ "$GITHUB_EVENT_NAME" == push ||
                "$COMMIT_MSG" =~ \[scipy\] ]]; then
              echo "trigger=true" >> "$GITHUB_OUTPUT"
          fi

  test-scipy:
    runs-on: ${{ matrix.os }}
    needs: [test-scipy-trigger, build-core]
    if: needs.test-scipy-trigger.outputs.test-scipy
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v4

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: core-build-${{ runner.os }}
          path: ./dist/

      - name: run scipy tests inside node
        shell: bash -l {0}
        run: |
          npm install pyodide
          cp -f dist/* node_modules/pyodide

          # This uses conftest.py on the current working directory to
          # skip/xfail tests in scipy
          cd packages/scipy
          # XXX for some unknown reason adding a conftest.py in the repo throws off the
          # other tests trying to import from conftest they find the scipy one ...
          mv scipy-conftest.py conftest.py

          node scipy-pytest.js --pyargs scipy -m 'not slow' -ra -v

  test-bun:
    runs-on: ${{ matrix.os }}

    needs: [build-core]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v4

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: core-build-${{ runner.os }}
          path: ./dist/

      - uses: oven-sh/setup-bun@v2

      - name: install test requirements
        working-directory: src/test-bun
        run: |
          bun --version
          bun install

      - name: run bun tests
        working-directory: src/test-bun
        run: bun smoke-test.js

  test-deno:
    runs-on: ${{ matrix.os }}

    needs: [build-core]
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]

    steps:
      - uses: actions/checkout@v4

      - name: Download build artifact
        uses: actions/download-artifact@v4
        with:
          name: core-build-${{ runner.os }}
          path: ./dist/

      - name: install test requirements
        shell: bash -l {0}
        run: |
          curl -fsSL https://deno.land/install.sh | sudo DENO_INSTALL=/usr/local sh -s v1.33.1
          deno --version

      - name: run deno tests
        shell: bash -l {0}
        run: cd src/test-deno && deno task test