diff --git a/.github/actions/workspace/action.yml b/.github/actions/workspace/action.yml index 77f0c6b5a2..9aa37949b7 100644 --- a/.github/actions/workspace/action.yml +++ b/.github/actions/workspace/action.yml @@ -16,7 +16,7 @@ inputs: default: workspace key_base: required: true - default: ${{ github.workflow }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}-${{ github.sha }} + default: ${{ github.workflow }}-${{ github.run_id }}-${{ github.run_number }}-${{ github.sha }} key_suffix: default: '' token: diff --git a/.github/workflows/unix.yml b/.github/workflows/tests.yml similarity index 64% rename from .github/workflows/unix.yml rename to .github/workflows/tests.yml index ec11117c0c..b901249c52 100644 --- a/.github/workflows/unix.yml +++ b/.github/workflows/tests.yml @@ -1,4 +1,4 @@ -name: Unix tests +name: Tests on: push: @@ -27,3 +27,15 @@ jobs: with: os: ${{ matrix.os }} build_type: ${{ matrix.build_type }} + + win_tests: + name: Windows tests + strategy: + matrix: + os: [windows-2019] + build_type: [release] + fail-fast: true + uses: ./.github/workflows/windows_impl.yml + with: + os: ${{ matrix.os }} + build_type: ${{ matrix.build_type }} diff --git a/.github/workflows/unix_impl.yml b/.github/workflows/unix_impl.yml index a8578a6336..8f615b26d1 100644 --- a/.github/workflows/unix_impl.yml +++ b/.github/workflows/unix_impl.yml @@ -75,9 +75,9 @@ jobs: - name: Run libmamba tests run: | unset CONDARC # Interferes with tests - cd build/libmamba/tests/ && ./test_libmamba + ./build/libmamba/tests/test_libmamba - libmambay_tests_unix: + libmambapy_tests_unix: name: Test libmamba Python bindings needs: ["build_shared_unix"] runs-on: ${{ inputs.os }} @@ -135,24 +135,3 @@ jobs: export TEST_MAMBA_EXE=$(pwd)/build/micromamba/micromamba unset CONDARC # Interferes with tests pytest -v --capture=tee-sys micromamba/tests/ - - name: micromamba local channel test - run: | - unset CONDARC # Interferes with tests - export TEST_MAMBA_EXE=$(pwd)/build/micromamba/micromamba - if [ "$RUNNER_OS" == "Linux" ]; then - mkdir -p $MAMBA_ROOT_PREFIX/conda-bld/linux-64 - wget -P $MAMBA_ROOT_PREFIX/conda-bld/linux-64 https://anaconda.org/conda-forge/xtensor/0.21.8/download/linux-64/xtensor-0.21.8-hc9558a2_0.tar.bz2 - wget -P $MAMBA_ROOT_PREFIX/conda-bld/linux-64 https://anaconda.org/conda-forge/xtl/0.6.21/download/linux-64/xtl-0.6.21-h0efe328_0.tar.bz2 - else - mkdir -p $MAMBA_ROOT_PREFIX/conda-bld/osx-64 - wget -P $MAMBA_ROOT_PREFIX/conda-bld/osx-64 https://anaconda.org/conda-forge/xtensor/0.21.8/download/osx-64/xtensor-0.21.8-h879752b_0.tar.bz2 - wget -P $MAMBA_ROOT_PREFIX/conda-bld/osx-64 https://anaconda.org/conda-forge/xtl/0.6.21/download/osx-64/xtl-0.6.21-h6516342_0.tar.bz2 - fi - conda index $MAMBA_ROOT_PREFIX/conda-bld - "${TEST_MAMBA_EXE}" create -n l_o_cal_test xtensor -c local -c conda-forge -y - - "${TEST_MAMBA_EXE}" list -n l_o_cal_test - "${TEST_MAMBA_EXE}" list -n l_o_cal_test | tail -n +3 > list.txt - if [ "$(grep -c "conda-bld" list.txt)" -ne 2 ]; then - exit 1 - fi diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml deleted file mode 100644 index c947ed2365..0000000000 --- a/.github/workflows/windows.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Windows tests - -on: - push: - branches: - - main - pull_request: - branches: - - main - paths-ignore: - - 'docs/**' - - '**.md' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - libmamba_tests_win: - name: libmamba tests - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-2019] - steps: - - uses: actions/checkout@v3 - - name: Create build environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ./dev/environment-dev.yml - environment-name: build_env - cache-environment: true - init-shell: bash cmd.exe - - uses: hendrikmuhs/ccache-action@main - with: - variant: sccache - key: ${{ github.job }}-${{ matrix.os }} - restore-keys: | - libmamba-${{ matrix.os }} - - name: Build libmamba - shell: cmd /C call {0} - run: | - cmake -B build/ ^ - -G Ninja ^ - -D CMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -D CMAKE_BUILD_TYPE="Release" ^ - -D CMAKE_MSVC_RUNTIME_LIBRARY="MultiThreadedDLL" ^ - -D BUILD_LIBMAMBA_TESTS=ON ^ - -D BUILD_LIBMAMBA=ON ^ - -D BUILD_LIBMAMBAPY=ON ^ - -D BUILD_SHARED=ON ^ - -D CMAKE_CXX_COMPILER_LAUNCHER=sccache ^ - -D CMAKE_C_COMPILER_LAUNCHER=sccache - if %errorlevel% neq 0 exit /b %errorlevel% - cmake --build build/ --parallel --target test_libmamba testing_libmamba_lock - if %errorlevel% neq 0 exit /b %errorlevel% - - name: Run libmamba tests - shell: cmd /C call {0} - run: | - cd build - ninja install - if %errorlevel% neq 0 exit /b %errorlevel% - @REM Interferes with tests - set CONDARC= - ninja test - if %errorlevel% neq 0 exit /b %errorlevel% - - name: Show build cache statistics - run: sccache --show-stats - - umamba_build_win: - name: Build micromamba - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-2019] - steps: - - uses: actions/checkout@v3 - - name: create build environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ./dev/environment-dev.yml - environment-name: build_env - cache-environment: true - init-shell: bash cmd.exe - - uses: hendrikmuhs/ccache-action@main - with: - variant: sccache - key: ${{ github.job }}-${{ matrix.os }} - restore-keys: | - libmamba-${{ matrix.os }} - - name: build micromamba - shell: cmd /C call {0} - run: | - cmake -B build/ ^ - -G Ninja ^ - -D CMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^ - -D CMAKE_BUILD_TYPE="Release" ^ - -D CMAKE_MSVC_RUNTIME_LIBRARY="MultiThreadedDLL" ^ - -D BUILD_MICROMAMBA=ON ^ - -D BUILD_LIBMAMBA=ON ^ - -D BUILD_SHARED=ON ^ - -D CMAKE_CXX_COMPILER_LAUNCHER=sccache ^ - -D CMAKE_C_COMPILER_LAUNCHER=sccache - if %errorlevel% neq 0 exit /b %errorlevel% - cmake --build build/ --parallel - if %errorlevel% neq 0 exit /b %errorlevel% - cmake --install build/ - - name: check that micromamba runs - shell: cmd /C call {0} - run: .\build\micromamba\micromamba.exe --help - - name: build cache statistics - run: sccache --show-stats - - name: tar micromamba artifact - shell: cmd /C call {0} - run: | - cp build/libmamba/libmamba.dll build/micromamba/ - tar -cvf umamba.tar build/micromamba/micromamba.exe build/micromamba/libmamba.dll - - uses: actions/upload-artifact@v3 - with: - name: _internal_micromamba_binary - path: umamba.tar - - umamba_integration_tests_win: - name: Micromamba integration tests with PWSH - needs: [umamba_build_win] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [windows-2019] - steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 - with: - name: _internal_micromamba_binary - - name: untar micromamba artifact - shell: bash -l -eo pipefail {0} - run: | - tar -xvf umamba.tar - - name: create build environment - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ./dev/environment-dev.yml - environment-name: build_env - create-args: >- - menuinst - conda-build - curl - init-shell: bash cmd.exe powershell - - name: micromamba python based tests with pwsh - shell: pwsh - run: | - $env:PYTHONIOENCODING='UTF-8' - $env:MAMBA_ROOT_PREFIX = Join-Path -Path $pwd -ChildPath 'mambaroot' - $env:TEST_MAMBA_EXE = Join-Path -Path $pwd -ChildPath 'build\micromamba\micromamba.exe' - $env:MAMBA_TEST_SHELL_TYPE='powershell' - Remove-Item -Path "env:CONDARC" - - pytest -v --capture=tee-sys micromamba/tests/ - - name: micromamba local channel checks - shell: bash -l {0} - run: | - mkdir -p $MAMBA_ROOT_PREFIX/conda-bld/win-64 - echo $PATH - micromamba list - micromamba info - curl --version - curl https://anaconda.org/conda-forge/xtensor/0.21.7/download/win-64/xtensor-0.21.7-h7ef1ec2_0.tar.bz2 -L -o $MAMBA_ROOT_PREFIX/conda-bld/win-64/xtensor-0.21.7-h7ef1ec2_0.tar.bz2 - curl https://anaconda.org/conda-forge/xtl/0.6.21/download/win-64/xtl-0.6.21-h5362a0b_0.tar.bz2 -L -o $MAMBA_ROOT_PREFIX/conda-bld/win-64/xtl-0.6.21-h5362a0b_0.tar.bz2 - conda index $MAMBA_ROOT_PREFIX/conda-bld - micromamba create -n l_o_cal_test xtensor -c local -c conda-forge -y - - micromamba list -n l_o_cal_test - # cut first couple of lines to remove prefix path - micromamba list -n l_o_cal_test | tail -n +3 > list.txt - if [ "$(grep -c "conda-bld" list.txt)" -ne 2 ]; then - exit 1 - fi diff --git a/.github/workflows/windows_impl.yml b/.github/workflows/windows_impl.yml new file mode 100644 index 0000000000..adb1162471 --- /dev/null +++ b/.github/workflows/windows_impl.yml @@ -0,0 +1,154 @@ +name: Windows tests impl + +on: + workflow_call: + inputs: + os: + required: true + type: string + build_type: + required: true + type: string + +defaults: + run: + # micromamba activation + shell: cmd /C call {0} + +jobs: + build_shared_win: + name: Build binaries + runs-on: ${{ inputs.os }} + steps: + - name: Checkout mamba repository + uses: actions/checkout@v4 + - name: Create build environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./dev/environment-dev.yml + environment-name: build_env + cache-environment: true + init-shell: cmd.exe + - uses: hendrikmuhs/ccache-action@main + with: + variant: sccache + key: ${{ github.job }}-${{ inputs.os }} + restore-keys: | + ccache-libmamba-${{ inputs.os }} + - name: Build mamba + run: | + cmake -B build/ -G Ninja ^ + --preset mamba-win-shared-${{ inputs.build_type }} ^ + -D CMAKE_MSVC_RUNTIME_LIBRARY="MultiThreadedDLL" ^ + -D CMAKE_CXX_COMPILER_LAUNCHER=sccache ^ + -D CMAKE_C_COMPILER_LAUNCHER=sccache + if %errorlevel% neq 0 exit /b %errorlevel% + cmake --build build/ --parallel + if %errorlevel% neq 0 exit /b %errorlevel% + - name: Show build cache statistics + run: sccache --show-stats + - name: Lock environment + run: micromamba env export --explicit > build/environment.lock + - name: Remove extra files before saving workspace + shell: bash -eo pipefail {0} + run: find build/ -type f -name '*.obj' -exec rm {} + + - name: Save workspace + uses: ./.github/actions/workspace + with: + action: save + path: build/ + key_suffix: ${{ inputs.os }}-${{ inputs.build_type }} + + libmamba_tests_win: + name: Test libmamba + needs: ["build_shared_win"] + runs-on: ${{ inputs.os }} + steps: + - name: Checkout mamba repository + uses: actions/checkout@v4 + - name: Restore workspace + uses: ./.github/actions/workspace + with: + action: restore + path: build/ + key_suffix: ${{ inputs.os }}-${{ inputs.build_type }} + - name: Create build environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./build/environment.lock + environment-name: build_env + init-shell: bash cmd.exe + - name: Run libmamba tests + shell: bash -elo pipefail {0} + run: | + unset CONDARC # Interferes with tests + cd ./build/libmamba && ./tests/test_libmamba + + # - name: Run libmamba tests + # run: | + # @REM Interferes with tests + # set CONDARC= + # @REM Move to directory of libmamba DLL for Windows to find it + # cd "build\libmamba" && .\tests\test_libmamba + + libmambapy_tests_win: + name: Test libmamba Python bindings + needs: ["build_shared_win"] + runs-on: ${{ inputs.os }} + steps: + - name: Checkout mamba repository + uses: actions/checkout@v4 + - name: Restore workspace + uses: ./.github/actions/workspace + with: + action: restore + path: build/ + key_suffix: ${{ inputs.os }}-${{ inputs.build_type }} + - name: Create build environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./build/environment.lock + environment-name: build_env + init-shell: bash cmd.exe + - name: Install libmambapy + shell: bash -elo pipefail {0} + run: | + ln build/libmambapy/bindings*.pyd libmambapy/libmambapy/ + cmake --install build/ --prefix "${CONDA_PREFIX}" + python -m pip install ./libmambapy/ + - name: Run libmamba Python bindings tests + run: | + python -m pytest libmambapy/tests/ + + umamba_integration_tests_win: + name: Test integration micromamba + needs: ["build_shared_win"] + runs-on: ${{ inputs.os }} + steps: + - name: Checkout mamba repository + uses: actions/checkout@v4 + - name: Restore workspace + uses: ./.github/actions/workspace + with: + action: restore + path: build/ + key_suffix: ${{ inputs.os }}-${{ inputs.build_type }} + - name: Create build environment + uses: mamba-org/setup-micromamba@v1 + with: + environment-file: ./build/environment.lock + environment-name: build_env + init-shell: bash cmd.exe powershell + - name: Install micromamba + shell: bash -elo pipefail {0} + run: | + cmake --install build/ --prefix local/ + - name: micromamba python based tests with pwsh + shell: pwsh + run: | + $env:PYTHONIOENCODING='UTF-8' + $env:MAMBA_ROOT_PREFIX = Join-Path -Path $pwd -ChildPath 'mambaroot' + $env:TEST_MAMBA_EXE = Join-Path -Path $pwd -ChildPath 'local\bin\micromamba.exe' + $env:MAMBA_TEST_SHELL_TYPE='powershell' + Remove-Item -Path "env:CONDARC" + python -m pytest -v --capture=tee-sys micromamba/tests/ diff --git a/dev/environment-dev.yml b/dev/environment-dev.yml index 28997334bf..43053d23f2 100644 --- a/dev/environment-dev.yml +++ b/dev/environment-dev.yml @@ -29,8 +29,10 @@ dependencies: - pytest-asyncio - pytest-lazy-fixture - pytest-xprocess + - requests - sel(win): pywin32 - - conda-build # Used in legacy CI tests for `conda index` + - sel(win): menuinst + - conda-index - conda-content-trust - conda-package-handling - cryptography<40.0 # Or breaks conda-content-trust diff --git a/micromamba/tests/conftest.py b/micromamba/tests/conftest.py index 8dee2da42d..4804d48995 100644 --- a/micromamba/tests/conftest.py +++ b/micromamba/tests/conftest.py @@ -76,11 +76,14 @@ def tmp_home( # Pytest would clean it automatically but this can be large (0.5 Gb for repodata) # We clean it explicitly + on_ci = "CI" in os.environ if not request.config.getoption("--no-eager-clean"): try: helpers.rmtree(new_home) - except PermissionError: - pass + # Silence possible cleanup exeptions on CI, weird things happening there + except Exception as e: + if not on_ci: + raise e @pytest.fixture @@ -148,9 +151,14 @@ def tmp_root_prefix( # Pytest would clean it automatically but this can be large (0.5 Gb for repodata) # We clean it explicitly + on_ci = "CI" in os.environ if not request.config.getoption("--no-eager-clean"): - if new_root_prefix.exists(): + try: helpers.rmtree(new_root_prefix) + # Silence possible cleanup exeptions on CI, weird things happening there + except Exception as e: + if not on_ci: + raise e # os.environ restored by tmp_clean_env and tmp_environ diff --git a/micromamba/tests/helpers.py b/micromamba/tests/helpers.py index 9ef2169f25..da5be2bfee 100644 --- a/micromamba/tests/helpers.py +++ b/micromamba/tests/helpers.py @@ -473,17 +473,21 @@ def recursive_chmod(path: Path, permission, is_root=True): def rmtree(path: Path): - p = Path(path) - recursive_chmod(p, 0o700) + path = Path(path) - def handleError(func, path, exc_info): - recursive_chmod(path, 0o700) - func(path) + if not path.exists(): + return - if p.is_dir(): - shutil.rmtree(p, onerror=handleError) + recursive_chmod(path, 0o700) + + def handleError(func, p, exc_info): + recursive_chmod(p, 0o700) + func(p) + + if path.is_dir(): + shutil.rmtree(path, onerror=handleError) else: - os.remove(p) + os.remove(path) def get_fake_activate(prefix): diff --git a/micromamba/tests/test_create.py b/micromamba/tests/test_create.py index 2902293a84..1a5e42fb83 100644 --- a/micromamba/tests/test_create.py +++ b/micromamba/tests/test_create.py @@ -4,7 +4,9 @@ import subprocess from pathlib import Path +import conda_index.index import pytest +import requests import yaml from . import helpers @@ -1149,3 +1151,47 @@ def test_create_with_unicode(tmp_home, tmp_root_prefix): assert res["actions"]["PREFIX"] == str(env_prefix) assert any(pkg["name"] == "xtensor" for pkg in res["actions"]["FETCH"]) assert any(pkg["name"] == "xtl" for pkg in res["actions"]["FETCH"]) + + +def download(url: str, out: Path): + response = requests.get(url) + if response.status_code == 200: + with open(out, "wb") as file: + file.write(response.content) + else: + raise Exception(f'Failed to download URL "{url}"') + + +@pytest.mark.skipif( + platform.system() == "Windows", + reason="Too many bugs runnning conda-index on Windows CI", +) +def test_create_local(tmp_home, tmp_root_prefix): + """The name "local" is a hard-coded custom multichannel.""" + name = "attrs" + plat = "noarch" + fn = "attrs-23.1.0-pyh71513ae_1.conda" + + conda_bld = (tmp_root_prefix / "conda-bld").expanduser().resolve() + (conda_bld / plat).mkdir(parents=True) + + download( + url=f"https://conda.anaconda.org/conda-forge/{plat}/{fn}", + out=conda_bld / plat / fn, + ) + + conda_index.index.ChannelIndex( + str(conda_bld), channel_name="conda-bld", threads=1 + ).index(None) + + res = helpers.create( + "-n", "myenv", "--override-channels", "-c", "local", "-y", name, "--json" + ) + + attrs_link_pkg = None + for pkg in res["actions"]["LINK"]: + if pkg["name"] == name: + attrs_link_pkg = pkg + + assert attrs_link_pkg["url"].startswith("file://") + assert attrs_link_pkg["url"].endswith(fn)