Merge pull request #6 from MannLabs/add_alphatesting

add the alphatesting worflow

.github/workflows/alphatesting.yml

@@ -0,0 +1,161 @@
+on:
+  workflow_dispatch:
+    inputs:
+      alphabase_ref:
+        description: "alphabase version: version of alphabase to use. Can be a commit hash, branch name or 'latest' for the latest release."
+        default: latest
+      alphatims_ref:
+        description: "alphatims version"
+        default: latest
+      alpharaw_ref:
+        description: "alpharaw version"
+        default: development  # TODO: change to latest once properly tagged release is out
+      alphapeptdeep_ref:
+        description: "alphapeptdeep version"
+        default: latest
+      directlfq_ref:
+        description: "alphatims version"
+        default: latest
+      alphadia_ref:
+        description: "alphadia version"
+        default: latest
+
+jobs:
+  build_image:
+    runs-on: AlphaDIA
+
+    steps:
+    - name: Print input parameters
+      run: echo "${{ toJSON(github.event.inputs) }}"
+
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+#    # avoid "no space left on device" error
+#    # https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+#    - name: cleanup
+#      run: |
+#        sudo rm -rf /usr/share/dotnet
+#        sudo rm -rf /opt/ghc
+#        sudo rm -rf "/usr/local/share/boost"
+#        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+    - name: Build and export
+      uses: docker/build-push-action@v6
+      with:
+        tags: alphax:latest
+        context: ./alphatesting
+        outputs: type=docker,dest=/tmp/alphax.tar
+        cache-from: type=gha
+        cache-to: type=gha,mode=max
+        build-args: |
+          ALPHABASE_REF=${{ inputs.alphabase_ref }}
+          ALPHATIMS_REF=${{ inputs.alphatims_ref }}
+          ALPHARAW_REF=${{ inputs.alpharaw_ref }}
+          ALPHAPEPTDEEP_REF=${{ inputs.alphapeptdeep_ref }}
+          DIRECTLFQ_REF=${{ inputs.directlfq_ref }}
+          ALPHADIA_REF=${{ inputs.alphadia_ref }}
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: alphax
+        path: /tmp/alphax.tar
+
+
+  alphabase_tests:
+    runs-on: AlphaDIA
+    needs: [ build_image ]
+    steps:
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: alphax
+        path: /tmp
+    - name: Load image
+      run: |
+        docker load --input /tmp/alphax.tar
+        docker image ls -a
+    - name: Print pip freeze
+      run: |
+        docker run alphax bash -c "pip freeze"
+    - name: alphabase tests
+      id: alphabase_tests
+      run: |
+        docker run alphax bash -c "cd alphabase/tests && chmod +x run_tests.sh && ./run_tests.sh"
+
+  alpharaw_tests:
+    runs-on: AlphaDIA
+    needs: [ build_image ]
+    steps:
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: alphax
+        path: /tmp
+    - name: Load image
+      run: |
+        docker load --input /tmp/alphax.tar
+        docker image ls -a
+    - name: Print pip freeze
+      run: |
+        docker run alphax bash -c "pip freeze"
+    - name: alpharaw tests
+      id: alpharaw_tests
+      run: |
+        docker run alphax bash -c "cd alpharaw/tests && ls && python -m download_test_data && chmod +x run_tests.sh && ./run_tests.sh"
+
+  alphapeptdeep_tests:
+    runs-on: AlphaDIA
+    needs: [ build_image ]
+    steps:
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: alphax
+        path: /tmp
+    - name: Load image
+      run: |
+        docker load --input /tmp/alphax.tar
+        docker image ls -a
+    - name: Print pip freeze
+      run: |
+        docker run alphax bash -c "pip freeze"
+    - name: alphapeptdeep tests
+      id: alphapeptdeep_tests
+      run: |
+        docker run alphax bash -c "cd alphapeptdeep/tests && chmod +x run_tests.sh && ./run_tests.sh"
+
+  alphadia_tests:
+    runs-on: AlphaDIA
+    needs: [ build_image ]
+    steps:
+    - name: Download artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: alphax
+        path: /tmp
+    - name: Load image
+      run: |
+        docker load --input /tmp/alphax.tar
+        docker image ls -a
+    - name: Print pip freeze
+      run: |
+        docker run alphax bash -c "pip freeze"
+    - name: alphadia unit tests ('not slow')
+      id: alphadia_unit_tests
+      run: |
+        docker run alphax bash -c "cd alphadia/tests &&  python -m pytest -k 'not slow'"
+    - name: alphadia unit tests ('slow')
+      id: alphadia_slow_unit_tests
+      run: |
+        docker run alphax bash -c "cd alphadia/tests &&  python -m pytest -k 'slow'"
+    - name: alphadia e2e test 'basic'
+      id: alphadia_e2e_tests
+      run: |
+        TEST_CASE_NAME=basic
+        docker run alphax bash -c "cd alphadia/tests/e2e_tests \
+          &&  python prepare_test_data.py $TEST_CASE_NAME \
+          &&  alphadia --config $TEST_CASE_NAME/config.yaml"    

README.md

@@ -1,10 +1,11 @@
 # alphashared
-Shared non-production code for AlphaX packages
+Shared non-production code for AlphaX packages.
 
 ## contents
 - `actions`: reusable github actions
 - `actions/get-code-review-input`: generates input for AI-assisted code review, see [README](actions/get-code-review-input/README.md)
 
-- `.github/workflows`: reusable github workflows
+- `.github/workflows`: github workflows
 - `.github/workflows/create_release.yml`: reusable workflow for creating releases, see [README](.github/workflows/README.md)
-- `.github/workflows/publish_on_pypi.yml`: reusable workflow for publishing on PyPi, see [README](.github/workflows/README.md)
+- `.github/workflows/publish_on_pypi.yml`: reusable workflow for publishing on PyPi, see [README](.github/workflows/README.md)
+- `.github/workflows/alphatesting.yml`: Cross-project tests for AlphaX, see [README](alphatesting/README.md)

alphatesting/Dockerfile

@@ -0,0 +1,102 @@
+# syntax=docker/dockerfile:1
+
+# A Dockerfile for stitiching together the different AlphaX packages with a defined version.
+
+# https://github.com/michaelosthege/pythonnet-docker
+FROM --platform=linux/amd64 mosthege/pythonnet:python3.10.10-mono6.12-pythonnet3.0.1
+
+# Prevents Python from writing pyc files.
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# Keeps Python from buffering stdout and stderr to avoid situations where
+# the application crashes without emitting any logs due to buffering.
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+COPY get_ref.sh .
+COPY replace_alphax.sh .
+COPY pip.conf /etc/pip.conf
+
+############################################################################
+ARG ALPHABASE_REF="development"
+RUN git clone https://github.com/MannLabs/alphabase.git \
+    && cd alphabase \
+    && git checkout $(../get_ref.sh $ALPHABASE_REF) \
+    && git status
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd alphabase && pip install "."
+
+
+############################################################################
+ARG ALPHATIMS_REF="develop"
+RUN git clone https://github.com/MannLabs/alphatims.git \
+    && cd alphatims \
+    && git checkout $(../get_ref.sh $ALPHATIMS_REF) \
+    && git status
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd alphatims && pip install "."
+
+
+############################################################################
+ARG ALPHARAW_REF="development"
+RUN git clone https://github.com/MannLabs/alpharaw.git \
+    && cd alpharaw \
+    && git checkout $(../get_ref.sh $ALPHARAW_REF) \
+    && ../replace_alphax.sh \
+    && git status
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd alpharaw && pip install "."
+
+
+############################################################################
+ARG ALPHAPEPTDEEP_REF="development"
+RUN git clone https://github.com/MannLabs/alphapeptdeep.git \
+    && cd alphapeptdeep \
+    && git checkout $(../get_ref.sh $ALPHAPEPTDEEP_REF) \
+    && ../replace_alphax.sh \
+    && git status
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd alphapeptdeep && pip install "."
+
+
+############################################################################
+ARG DIRECTLFQ_REF="development"
+RUN git clone https://github.com/MannLabs/directlfq.git \
+    && cd directlfq \
+    && git checkout $(../get_ref.sh $DIRECTLFQ_REF) \
+    && git status
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd directlfq && pip install "."
+
+############################################################################
+ARG ALPHADIA_REF="development"
+RUN git clone https://github.com/MannLabs/alphadia.git \
+    && cd alphadia \
+    && git checkout $(../get_ref.sh $ALPHADIA_REF) \
+    && git status
+
+# need to delete requirements in two files, otherwise the test comparing them will fail
+RUN cd alphadia \
+    && ../replace_alphax.sh ./requirements/requirements.txt \
+    && ../replace_alphax.sh ./requirements/requirements_loose.txt
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    cd alphadia && pip install ".[stable]"
+
+
+############################################################################
+# additional dependencies for testing
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install pytest nbmake
+
+# build:
+# docker build -f Dockerfile.e2e  --build-arg="ALPHABASE_REF=latest" --build-arg="ALPHATIMS_REF=latest" --build-arg="ALPHARAW_REF=f8a355a" --build-arg="ALPHAPEPTDEEP_REF=latest" --build-arg="DIRECTLFQ_REF=latest" -t alphax_e2e .
+
+# run bash:
+# docker run -v $DATA_FOLDER:/app/data/ -it alphax_e2e bashx

alphatesting/README.md

@@ -0,0 +1,65 @@
+# alphatesting
+Cross-project tests for AlphaX.
+These tests should help to detect breaking changes in the AlphaX projects
+before they are released.
+
+
+## Disclaimer
+This approach to catch breaking changes early relies on the assumption that the tests of the individual
+projects are comprehensive and cover all relevant aspects of the project. Breaking changes in code parts 
+that are not tested will not be detected.
+
+## Basic idea
+The following description differentiates "base packages"
+(no dependency on other AlphaX packages),
+ from "non-base packages" (one or more dependencies on other AlphaX packages).
+Examples for base packages are AlphaBase and directLFQ, examples for non-base packages are AlphaDIA and AlphaPeptDeep.
+
+The provided Dockerfile sequentially clones selected AlphaX projects, and checks out a user-defined version
+(git commit hash, branch name, latest release). Prior to installing of each non-base project,
+the dependencies to all base projects are removed from the requirements file, such that the 
+non-base projects use the already provided, user-defined versions of the base projects. 
+
+In this defined python environment, selected tests of base and non-base projects are run.
+
+## Maintenance
+Whenever a new project is added to the AlphaX family, that has dependencies to other projects, 
+add it to the `Dockerfile` and the workflow file. 
+
+Also, when an already supported AlphaX package gets a new dependency to another AlphaX package,
+in the `Dockerfile` adapt the replacements of dependencies in the `requirements` files .
+
+Use the other projects as a template.
+
+When a new class of tests is added, add it to the workflow file. Similarly, when a test run script is renamed,
+adapt the workflow file.
+
+
+## Usage
+### How to use this
+
+1. Run [this workflow](https://github.com/MannLabs/alphatesting/actions/workflows/alphatesting.yml)
+and optionally specify the git commit hashes or branch names of the supported AlphaX projects.
+The default is 'latest', i.e. the latest release.
+Note: tags are currently not supported
+
+2. The workflow will build a docker image with the specified versions of the AlphaX projects 
+and run the tests.
+
+Currently, the following tests are run:
+- `alphabase`
+- `alphawraw`
+- `alphapeptdeep`
+- `alphadia`
+
+If one or more tests failed, the workflow will fail. In this case, inspect the logs of the failed tests.
+
+### Example
+You want to release a new version of `alphabase` and want to make sure that the changes do not break the current releases
+of the other projects.
+
+In this case, identify the commit hash of the `alphabase` release candidate, and pass it to the
+workflow (alternatively, pass `development`, if this branch contains already the changes to be released. 
+Leave all other input values at their default.
+If all tests run successfully, you got some confidence that the changes in `alphabase` 
+do not break the other projects.

alphatesting/get_ref.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# if passed 'latest', return latest tag, otherwise return passed value
+# needs to be run in a folder containing a git repository
+
+set -u -e
+
+if [ "$1" == "latest" ]; then
+    latestTag=$(git describe --tags "$(git rev-list --tags --max-count=1)")
+    echo $latestTag
+    exit
+fi
+echo $1

alphatesting/pip.conf

@@ -0,0 +1,7 @@
+#https://stackoverflow.com/a/74616730
+
+[install]
+compile = no
+
+[global]
+no-cache-dir = True