Upgrade MHub to uv and python3.11 (#100)

- The base image now ships with uv 0.4.4
- The mhubio and segdb packages are now installed in a virtual
environment using python 3.11 (bumped from 3.8)
- All dependency versions are fixed now (and will be moved into package
dependencies with a future update)

- Some models require python 3.8, we created a new .venv38 environment
with python 3.8 and installed the model dependencies there. The model
logic is then outsourced into a cli script that is called from the
Runner module.

- Future base images likely will contain updated versions of uv since uv
is actively developed and receives frequent improvements and fixes.

base/bin/mhub.run

@@ -1,3 +1,4 @@
 #!/bin/bash
-PYTHON=$(which python3)
-$PYTHON -m mhubio.run "$@"
+# PYTHON=$(uv run which python3)
+# $PYTHON -m mhubio.run "$@"
+uv run python -m mhubio.run "$@"

base/bin/mhub.update

@@ -1,4 +1,3 @@
 #!/bin/bash
-PYTHON=$(which python3)
-$PYTHON -m pip uninstall -y mhubio segdb
-$PYTHON -m pip install git+https://github.com/MHubAI/mhubio.git git+https://github.com/MHubAI/segdb.git
+uv pip uninstall mhubio segdb
+uv pip install git+https://github.com/MHubAI/mhubio.git git+https://github.com/MHubAI/segdb.git

base/bin/mhub.version

@@ -3,10 +3,10 @@
 MODEL_FOLDER="/app/models/"
 
 # mhubio version
-MHUBIO=$(pip freeze | grep mhubio | cut -d"@" -f3)
+MHUBIO=$(uv pip freeze | grep mhubio | cut -d"@" -f3)
 
 # segdb version
-SEGDB=$(pip freeze | grep segdb | cut -d"@" -f3)
+SEGDB=$(uv pip freeze | grep segdb | cut -d"@" -f3)
 
 echo -e "mhubio==${MHUBIO}"
 echo -e "segdb==${SEGDB}"
@@ -37,21 +37,21 @@ fi
 # pip freeze without segdb and mhubio (already on top of the lists,
 # since for now they are commits). Ideally, this should return only pip versions
 # (although some package might be installed from git by contributors)
-pip freeze | grep -v "segdb" | grep -v "mhubio"
+uv pip freeze | grep -v "segdb" | grep -v "mhubio"
 
 # collect additional information on installed system dependencies.
 #  to allow contributors to include additional dependencies, we should use a environment variable or a file instead.
 
 # versions of python, pip, plastimatch, jq, git, libopenslide-dev, libvips-dev, dcm2niix, ffmpeg, libsm6, libxext6
 # echo -e "+++"
-# echo -e "python==$(python3 --version 2>&1)"
+# echo -e "python==$(python3 --version 2>&1 | cut -d" " -f2)"
 # echo -e "pip==$(pip --version 2>&1)"
-# echo -e "plastimatch==$(plastimatch --version 2>&1)"
+# echo -e "plastimatch==$(plastimatch --version 2>&1 | cut -d" " -f3)"
 # echo -e "jq==$(jq --version 2>&1)"
-# echo -e "git==$(git --version 2>&1)"
-# echo -e "libopenslide-dev==$(dpkg -s libopenslide-dev | grep Version)"
-# echo -e "libvips-dev==$(dpkg -s libvips-dev | grep Version)"
+# echo -e "git==$(git --version 2>&1 | cut -d" " -f3)"
+# echo -e "libopenslide-dev==$(dpkg -s libopenslide-dev | grep Version | cut -d" " -f2)"
+# echo -e "libvips-dev==$(dpkg -s libvips-dev | grep Version | cut -d" " -f2)"
 # echo -e "dcm2niix==$(dcm2niix -h | grep "dcm2niiX version" | cut -d"v" -f3)"
 # echo -e "ffmpeg==$(ffmpeg -version 2>&1 | grep ffmpeg | cut -d" " -f3)"
-# echo -e "libsm6==$(dpkg -s libsm6 | grep Version)"
-# echo -e "libxext6==$(dpkg -s libxext6 | grep Version)"
+# echo -e "libsm6==$(dpkg -s libsm6 | grep Version | cut -d" " -f2)"
+# echo -e "libxext6==$(dpkg -s libxext6 | grep Version | cut -d" " -f2)"

base/dockerfiles/Dockerfile

@@ -13,7 +13,8 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Create a working directory and set it as the working directory
 # Also create directories for input and output data (mounting points) in the same RUN to avoid creating intermediate layers
-RUN mkdir -p /app /app/data /app/data/input_data /app/data/output_data /app/utility/config /app/xmodules /app/xcollections
+RUN mkdir -p /app/data /app/data/input_data /app/data/output_data /app/data/reference_data \
+             /app/utility/config /app/xmodules /app/xcollections
 WORKDIR /app
 
 # Install system utilities and useful packages
@@ -25,42 +26,63 @@ RUN apt update && apt install -y --no-install-recommends \
   unzip \
   sudo \
   git \
-  python3 \
-  python3-pip
+  tree \
+  clang
 
 # Install core MHubIO modules requirements (python3-openslide and libvips-dev are dependencies of panimg)
 RUN apt update && apt install -y --no-install-recommends \
   libopenslide-dev \
   libvips-dev \
   plastimatch \
   ffmpeg libsm6 libxext6 \
+  ca-certificates \
   && rm -rf /var/lib/apt/lists/* 
 
+# Install uv (download installer, run & remove installer, add to PATH)
+ADD https://astral.sh/uv/0.4.4/install.sh /uv-installer.sh
+RUN sh /uv-installer.sh && rm /uv-installer.sh
+ENV PATH="/root/.cargo/bin/:$PATH"
+
+# install python3.11 via uv into .venv
+RUN uv venv -p 3.11
+
+# Add a link to pip3 for compatibility
+# NOTE: model implementations should install dependencies into separate virtual environments and use a cli script.
+# RUN uv pip install --no-cache pip && mkdir /mhub-compatibility && ln -s $(uv run which pip) /mhub-compatibility/pip3
+# ENV PATH="/mhub-compatibility:$PATH"
+RUN uv pip install --no-cache pip \
+ && ln -s $(uv run which pip) /usr/bin/pip3
+
 # Install general python utilities (specify version if necessary)
-RUN pip3 install --upgrade pip && pip3 install --no-cache-dir \
-  typing-extensions \
-  Pillow==9.5.0 \
-  h5py \
-  numpy \
-  pandas \ 
-  panimg \
-  pydicom \
-  pydicom-seg \
-  highdicom \
-  rt_utils \
-  PyYAML \
-  pyplastimatch \
-  SimpleITK==2.2.1 \
-  thedicomsort \
-  colorspacious \
-  jsonschema==4.21.1 \
-  dcmqi==0.2.0 \
-  dcm2niix==1.0.20220715
+# NOTE: these dependencies will be declared as part of mhubio in an upcoming release.
+# RUN uv pip install --no-cache typing-extensions Pillow h5py numpy pandas panimg pydicom pydicom-seg highdicom rt_utils PyYAML pyplastimatch SimpleITK thedicomsort colorspacious jsonschema dcmqi dcm2niix toml
+RUN uv pip install --no-cache \
+  "colorspacious~=1.1.2" \
+  "dcm2niix~=1.0.20220715" \
+  "dcmqi~=0.2.0" \
+  "h5py~=3.11.0" \
+  "highdicom~=0.22.0" \
+  "jsonschema~=3.2.0" \
+  "pandas~=2.2.2" \
+  "panimg~=0.13.2" \
+  "pillow~=10.4.0" \
+  "pydicom-seg~=0.4.1" \
+  "pydicom~=2.4.4" \
+  "pyplastimatch~=0.4.6" \
+  "pyyaml~=6.0.2" \
+  "rt-utils~=1.2.7" \
+  "simpleitk~=2.4.0" \
+  "thedicomsort~=1.0.1" \
+  "toml~=0.10.2" \
+  "typing-extensions~=4.12.2"
+
+# Install mhub dependencies and tools
+RUN uv pip install git+https://github.com/MHubAI/mhubio.git \
+  && uv pip install git+https://github.com/MHubAI/segdb.git  \
+  && uv tool install git+https://github.com/LennyN95/medcmp.git@pyproject_restructure_poetry
 
-# Install mhubio framework from git, pulls the utils scripts from GitHub
-RUN pip3 install git+https://github.com/MHubAI/mhubio.git \
-  && pip3 install git+https://github.com/MHubAI/segdb.git \
-  && git init \
+# Install the base image utilities
+RUN git init \
   && git sparse-checkout set "base/buildutils" "base/bin" "base/configs" "base/collections.json" \
   && git fetch https://github.com/MHubAI/models.git main \
   && git merge FETCH_HEAD \
@@ -76,5 +98,6 @@ RUN pip3 install git+https://github.com/MHubAI/mhubio.git \
 # Set PYTHONPATH to the /app folder
 ENV PYTHONPATH="/app"
 
-# FIXME: pass it as a command to the container in Slicer
-CMD ["echo", "MHub Ubuntu 20.04 base image. Visit http://mhub.ai to find out more!"]
+#CMD ["echo", "MHub Ubuntu 20.04 base image. Visit http://mhub.ai to find out more!"]
+ENTRYPOINT [ "mhub.run" ]
+CMD ["--utility"]

models/gc_lunglobes/dockerfiles/Dockerfile

@@ -6,14 +6,21 @@ LABEL authors="[email protected],[email protected],lnuernbe
 # Install system dependencies for OpenCV
 RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
 
+# create new virtual environment
+RUN uv venv --python-preference only-managed -p 3.8 .venv38
+
 # Install required dependencies for lobe segmentation (CUDA-enabled)
-RUN pip3 install --no-cache-dir \
-    opencv-python \
+#  SimpleITK==1.2.4 is required for legacy Resample::Execute operation
+RUN uv pip install -n -p .venv38 \
+    pydicom==2.4.4 packaging==24.1 psutil==6.0.0 \
+    opencv-python==4.10.0.84 \
     torch==2.0.1 torchvision==0.15.2 \
-    dgl==1.1.2 -f https://data.dgl.ai/wheels/cu118/repo.html
+    SimpleITK==1.2.4                        
 
-# SimpleITK downgrade required for legacy Resample::Execute operation
-RUN pip3 install --no-cache-dir --force-reinstall SimpleITK==1.2.4 
+# Install dgl (CUDA-enabled)
+# NOTE: uv pip install -f option doesn't work as intended
+RUN uv pip install -n -p .venv38 pip \
+ && uv run -p .venv38 pip install dgl==1.1.2 -f https://data.dgl.ai/wheels/cu118/repo.html
 
 # Import the MHub model definiton
 ARG MHUB_MODELS_REPO

models/gc_lunglobes/utils/LobeSegmentationRunner.py

@@ -10,14 +10,10 @@
 ------------------------------------------------------
 """
 
-from typing import List
-from mhubio.core import Instance, DataTypeQuery, InstanceData, IO, Module
+from mhubio.core import Instance, InstanceData, IO, Module
+from pathlib import Path
 
-import os
-import numpy as np
-import SimpleITK as sitk
-
-from src.test import segment_lobe, segment_lobe_init
+CLI_PATH = Path(__file__).parent / "run38.py"
 
 @IO.ConfigInput('in_data', 'nifti|nrrd|mha:mod=ct', the='supported datatypes for the lobes segmentation model')
 class LobeSegmentationRunner(Module):
@@ -28,30 +24,12 @@ class LobeSegmentationRunner(Module):
     def task(self, instance: Instance, in_data: InstanceData, out_data: InstanceData) -> None:
         # NOTE input data originally was specified for MHA/MHD and could be extended for DICOM
 
-        # read image
-        self.v(f"Reading image from {in_data.abspath}")
-        img_itk = sitk.ReadImage(in_data.abspath)
-        img_np = sitk.GetArrayFromImage(img_itk)
-
-        # apply lobe segmentation
-        origin = img_itk.GetOrigin()[::-1]
-        spacing = img_itk.GetSpacing()[::-1]
-        direction = np.asarray(img_itk.GetDirection()).reshape(3, 3)[::-1].flatten().tolist()
-        meta_dict =  {
-            "uid": os.path.basename(in_data.abspath),
-            "size": img_np.shape,
-            "spacing": spacing,
-            "origin": origin,
-            "original_spacing": spacing,
-            "original_size": img_np.shape,
-            "direction": direction
-        }
-
-        handle = segment_lobe_init()
-        seg_result_np = segment_lobe(handle, img_np, meta_dict)
-
-        # store image
-        self.v(f"Writing image to {out_data.abspath}")
-        seg_itk = sitk.GetImageFromArray(seg_result_np)
-        seg_itk.CopyInformation(img_itk)
-        sitk.WriteImage(seg_itk, out_data.abspath)
+        # Call the Xie2020 Lobe Segmentation in a separate python3.8 venv
+        cmd = [
+            "uv", "run", "-p", ".venv38", "python",
+            str(CLI_PATH),
+            in_data.abspath,
+            out_data.abspath
+        ]
+
+        self.subprocess(cmd, text=True)

models/gc_lunglobes/utils/run38.py

@@ -0,0 +1,43 @@
+import os
+import numpy as np 
+import SimpleITK as sitk
+
+from src.test import segment_lobe, segment_lobe_init
+
+def run(input_image_path: str, output_image_path: str):
+
+  img_itk = sitk.ReadImage(input_image_path)
+  img_np = sitk.GetArrayFromImage(img_itk)
+
+  # apply lobe segmentation
+  origin = img_itk.GetOrigin()[::-1]
+  spacing = img_itk.GetSpacing()[::-1]
+  direction = np.asarray(img_itk.GetDirection()).reshape(3, 3)[::-1].flatten().tolist()
+  meta_dict =  {
+      "uid": os.path.basename(input_image_path),
+      "size": img_np.shape,
+      "spacing": spacing,
+      "origin": origin,
+      "original_spacing": spacing,
+      "original_size": img_np.shape,
+      "direction": direction
+  }
+
+  handle = segment_lobe_init()
+  seg_result_np = segment_lobe(handle, img_np, meta_dict)
+
+  # store image
+  print(f"Writing image to {output_image_path}")
+  seg_itk = sitk.GetImageFromArray(seg_result_np)
+  seg_itk.CopyInformation(img_itk)
+  sitk.WriteImage(seg_itk, output_image_path)
+
+# cli
+if __name__ == "__main__":
+  import argparse
+  parser = argparse.ArgumentParser(description='Run Xie2020 Lobe Segmentation')
+  parser.add_argument('input_image_path', type=str, help='Path to input image')
+  parser.add_argument('output_image_path', type=str, help='Path to output image')
+  args = parser.parse_args()
+
+  run(args.input_image_path, args.output_image_path)

models/gc_tiger_lb2/dockerfiles/Dockerfile

@@ -3,12 +3,6 @@ FROM mhubai/base:latest
 # Specify/override authors label
 LABEL authors="[email protected]"
 
-# Install pipenv (for a custom Python/Pip environment for ASAP-2.1 and the other algorithm requirements)
-RUN pip3 install --no-cache-dir pipenv
-
-# Set environment variables for pipenv (installs into /app/.venv)
-ENV PIPENV_VENV_IN_PROJECT=1
-
 # Install ASAP 2.1
 RUN apt-get update && \
     apt-get -y install curl libpython3.8-dev && \
@@ -26,16 +20,17 @@ RUN git clone --depth 1 --branch 0.1.0 https://github.com/DIAGNijmegen/tiger_vun
     rm -rf /vuno/.git
 
 # Setup and install algorithm pipenv environment
-#   1. Ensure we configure a new empty pipenv for Python 3.8
+#   1. Ensure we configure a new empty virtual environment for Python 3.8
 #   2. Link ASAP libraries into our environment
 #   3. Install required torch and torchvision dependencies
 #   4. Install tiger LB2 dependencies from requirements.txt
 #   5. Upgrade version of numpy and numba to function correctly with ASAP
-RUN pipenv install --python 3.8 && \
-    echo "/opt/ASAP/bin" > /app/.venv/lib/python3.8/site-packages/asap.pth && \
-    pipenv run pip install --no-cache-dir torch==2.0.1+cu118 torchvision==0.15.2+cu118 -f https://download.pytorch.org/whl/torch_stable.html && \
-    pipenv run pip install --no-cache-dir -r /vuno/requirements.txt && \
-    pipenv run pip install --no-cache-dir --upgrade numpy==1.24.4 numba==0.58.1
+RUN uv venv --python 3.8 .venv38 \
+ && echo "/opt/ASAP/bin" > /app/.venv38/lib/python3.8/site-packages/asap.pth \
+ && uv pip install -n -p .venv38 -f https://download.pytorch.org/whl/torch_stable.html \
+    torch==2.0.1+cu118 torchvision==0.15.2+cu118 \
+ && uv pip install -n -p .venv38 -r /vuno/requirements.txt \
+ && uv pip install -n -p .venv38 --upgrade numpy==1.24.4 numba==0.58.1
 
 # Download and install model weights file from zenodo
 RUN rm -rf /vuno/pretrained_weights && \

models/gc_tiger_lb2/utils/TigerLB2Runner.py

@@ -36,8 +36,7 @@ def task(self, instance: Instance, in_data: InstanceData, out_data: InstanceData
         # Execute the Tiger LB2 Algorithm through a Python subprocess and associated pipenv environment
         self.subprocess(
             [
-                "pipenv",
-                "run",
+                "uv", "run", "-p", ".venv38",
                 "python",
                 str(self.CLI_SCRIPT_PATH),
                 in_data.abspath,

models/gc_wsi_bgseg/dockerfiles/Dockerfile

@@ -3,15 +3,9 @@ FROM mhubai/base:latest
 # Update authors label
 LABEL authors="[email protected]"
 
-# Install pipenv (for a custom Python/Pip environment for ASAP-2.1 and the other algorithm requirements)
-RUN pip3 install --no-cache-dir pipenv
-
-# Set environment variables for pipenv (installs into /app/.venv)
-ENV PIPENV_VENV_IN_PROJECT=1
-
 # Install ASAP 2.1
 RUN apt-get update && \
-    apt-get -y install curl libpython3.8-dev && \
+    apt-get -y install libpython3.8-dev && \
     curl --remote-name --location "https://github.com/computationalpathologygroup/ASAP/releases/download/ASAP-2.1/ASAP-2.1-py38-Ubuntu2004.deb" && \
     dpkg --install ASAP-2.1-py38-Ubuntu2004.deb || true && \
     apt-get -f install --fix-missing --fix-broken --assume-yes && \
@@ -25,15 +19,15 @@ RUN apt-get update && \
 #   3. Upgrade pip
 #   4. Upgrade version of numpy and numba to function correctly with ASAP
 #   5. Install required dependencies for algorithm
-RUN pipenv install --python 3.8 && \
-    echo "/opt/ASAP/bin" > /app/.venv/lib/python3.8/site-packages/asap.pth && \
-    pipenv run pip install --no-cache-dir --upgrade pip && \
-    pipenv run pip install --no-cache-dir --upgrade numpy==1.24.4 numba==0.58.1 && \
-    pipenv run pip install --no-cache-dir scipy==1.10.1 scikit-image==0.21.0 h5py==3.11.0
+RUN uv venv --python 3.8 .venv38 && \
+    echo "/opt/ASAP/bin" > /app/.venv38/lib/python3.8/site-packages/asap.pth && \
+    uv pip install -n -p .venv38 --upgrade numpy==1.24.4 numba==0.58.1 && \
+    uv pip install -n -p .venv38 scipy==1.10.1 scikit-image==0.21.0 h5py==3.11.0
 
 # build/install Tensorflow 2.11.0 with GPU support (without conda), with CUDA 11 toolkit and cudnn 8
 # tensorflow-2.11.0 Python 3.7-3.10 cuDNN >= 8.1 CUDA >= 11.2
-RUN pipenv run pip install --no-cache-dir \
+RUN uv pip install -n -p .venv38 pip \
+ && uv run -p .venv38 pip install --no-cache-dir \
     nvidia-cuda-runtime-cu11 \
     nvidia-cusolver-cu11 \
     nvidia-curand-cu11 \
@@ -46,8 +40,8 @@ RUN pipenv run pip install --no-cache-dir \
     --extra-index-url https://pypi.ngc.nvidia.com
 
 # Configure required paths for tensorflow with GPU support
-ENV NVIDIA_DIR /app/.venv/lib/python3.8/site-packages/nvidia
-ENV LD_LIBRARY_PATH /app/.venv/lib/python3.8/site-packages/tensorrt:$NVIDIA_DIR/cublas/lib:$NVIDIA_DIR/cuda_runtime/lib:$NVIDIA_DIR/cudnn/lib:$NVIDIA_DIR/cufft/lib:$NVIDIA_DIR/curand/lib:$NVIDIA_DIR/cusolver/lib:$NVIDIA_DIR/cusparse/lib
+ENV NVIDIA_DIR /app/.venv38/lib/python3.8/site-packages/nvidia
+ENV LD_LIBRARY_PATH /app/.venv38/lib/python3.8/site-packages/tensorrt:$NVIDIA_DIR/cublas/lib:$NVIDIA_DIR/cuda_runtime/lib:$NVIDIA_DIR/cudnn/lib:$NVIDIA_DIR/cufft/lib:$NVIDIA_DIR/curand/lib:$NVIDIA_DIR/cusolver/lib:$NVIDIA_DIR/cusparse/lib
 
 # Import the MHub model definiton
 ARG MHUB_MODELS_REPO